History

Merge tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Pull bpf updates from Alexei Starovoitov: - Add BPF uprobe session support (Jiri Olsa) - Optimize uprobe performance (Andrii Nakryiko) - Add bpf_fastcall support to helpers and kfuncs (Eduard Zingerman) - Avoid calling free_htab_elem() under hash map bucket lock (Hou Tao) - Prevent tailcall infinite loop caused by freplace (Leon Hwang) - Mark raw_tracepoint arguments as nullable (Kumar Kartikeya Dwivedi) - Introduce uptr support in the task local storage map (Martin KaFai Lau) - Stringify errno log messages in libbpf (Mykyta Yatsenko) - Add kmem_cache BPF iterator for perf's lock profiling (Namhyung Kim) - Support BPF objects of either endianness in libbpf (Tony Ambardar) - Add ksym to struct_ops trampoline to fix stack trace (Xu Kuohai) - Introduce private stack for eligible BPF programs (Yonghong Song) - Migrate samples/bpf tests to selftests/bpf test_progs (Daniel T. Lee) - Migrate test_sock to selftests/bpf test_progs (Jordan Rife) * tag 'bpf-next-6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (152 commits) libbpf: Change hash_combine parameters from long to unsigned long selftests/bpf: Fix build error with llvm 19 libbpf: Fix memory leak in bpf_program__attach_uprobe_multi bpf: use common instruction history across all states bpf: Add necessary migrate_disable to range_tree. bpf: Do not alloc arena on unsupported arches selftests/bpf: Set test path for token/obj_priv_implicit_token_envvar selftests/bpf: Add a test for arena range tree algorithm bpf: Introduce range_tree data structure and use it in bpf arena samples/bpf: Remove unused variable in xdp2skb_meta_kern.c samples/bpf: Remove unused variables in tc_l2_redirect_kern.c bpftool: Cast variable `var` to long long bpf, x86: Propagate tailcall info only for subprogs bpf: Add kernel symbol for struct_ops trampoline bpf: Use function pointers count as struct_ops links count bpf: Remove unused member rcu from bpf_struct_ops_map selftests/bpf: Add struct_ops prog private stack tests bpf: Support private stack for struct_ops progs selftests/bpf: Add tracing prog private stack tests bpf, x86: Support private stack in jit ...

Linus Torvalds committed 1 year ago

Commit 6e95ef02 parents 43fb83c 2c8b09a211 files · +6961 −3473Browse files

Unified · Split

211 files
  1. Documentation/bpf/btf.rst +76 −1
  2. Documentation/bpf/verifier.rst +2 −2
  3. arch/arm64/net/bpf_jit_comp.c +31 −16
  4. arch/x86/net/bpf_jit_comp.c +144 −5
  5. include/linux/bpf.h +54 −9
  6. include/linux/bpf_local_storage.h +9 −3
  7. include/linux/bpf_verifier.h +31 −36
  8. include/linux/btf.h +11 −11
  9. include/linux/btf_ids.h +1 −0
  10. include/linux/filter.h +1 −0
  11. include/uapi/linux/bpf.h +4 −5
  12. kernel/bpf/Makefile +2 −1
  13. kernel/bpf/arena.c +24 −14
  14. kernel/bpf/arraymap.c +24 −2
  15. kernel/bpf/bpf_cgrp_storage.c +2 −2
  16. kernel/bpf/bpf_inode_storage.c +2 −2
  17. kernel/bpf/bpf_local_storage.c +63 −16
  18. kernel/bpf/bpf_struct_ops.c +103 −12
  19. kernel/bpf/bpf_task_storage.c +5 −2
  20. kernel/bpf/btf.c +42 −15
  21. kernel/bpf/core.c +6 −0
  22. kernel/bpf/dispatcher.c +2 −1
  23. kernel/bpf/hashtab.c +39 −17
  24. kernel/bpf/helpers.c +27 −2
  25. kernel/bpf/kmem_cache_iter.c +238 −0
  26. kernel/bpf/memalloc.c +1 −4
  27. kernel/bpf/range_tree.c +272 −0
  28. kernel/bpf/range_tree.h +21 −0
  29. kernel/bpf/syscall.c +115 −9
  30. kernel/bpf/trampoline.c +50 −10
  31. kernel/bpf/verifier.c +418 −179
  32. kernel/trace/bpf_trace.c +86 −24
  33. mm/slab_common.c +19 −0
  34. net/core/bpf_sk_storage.c +3 −3
  35. samples/bpf/Makefile +0 −25
  36. samples/bpf/sock_flags.bpf.c +0 −47
  37. samples/bpf/syscall_nrs.c +5 −0
  38. samples/bpf/tc_l2_redirect_kern.c +0 −6
  39. samples/bpf/test_cgrp2_array_pin.c +0 −106
  40. samples/bpf/test_cgrp2_attach.c +0 −177
  41. samples/bpf/test_cgrp2_sock.c +0 −294
  42. samples/bpf/test_cgrp2_sock.sh +0 −137
  43. samples/bpf/test_cgrp2_sock2.c +0 −95
  44. samples/bpf/test_cgrp2_sock2.sh +0 −103
  45. samples/bpf/test_cgrp2_tc.bpf.c +0 −56
  46. samples/bpf/test_cgrp2_tc.sh +0 −187
  47. samples/bpf/test_current_task_under_cgroup.bpf.c +0 −43
  48. samples/bpf/test_current_task_under_cgroup_user.c +0 −115
  49. samples/bpf/test_overhead_kprobe.bpf.c +0 −41
  50. samples/bpf/test_overhead_raw_tp.bpf.c +0 −17
  51. samples/bpf/test_overhead_tp.bpf.c +0 −23
  52. samples/bpf/test_overhead_user.c +0 −225
  53. samples/bpf/test_override_return.sh +0 −16
  54. samples/bpf/test_probe_write_user.bpf.c +0 −52
  55. samples/bpf/test_probe_write_user_user.c +0 −108
  56. samples/bpf/tracex7.bpf.c +0 −15
  57. samples/bpf/tracex7_user.c +0 −56
  58. samples/bpf/xdp2skb_meta_kern.c +1 −1
  59. samples/bpf/xdp_adjust_tail_kern.c +1 −0
  60. scripts/Makefile.btf +4 −2
  61. scripts/bpf_doc.py +50 −3
  62. tools/bpf/bpf_jit_disasm.c +1 −1
  63. tools/bpf/bpftool/Makefile +5 −1
  64. tools/bpf/bpftool/btf.c +89 −11
  65. tools/bpf/bpftool/jit_disasm.c +29 −11
  66. tools/bpf/resolve_btfids/main.c +2 −2
  67. tools/bpf/runqslower/runqslower.bpf.c +0 −1
  68. tools/include/uapi/linux/bpf.h +4 −5
  69. tools/lib/bpf/Makefile +2 −1
  70. tools/lib/bpf/bpf.c +1 −0
  71. tools/lib/bpf/bpf_gen_internal.h +1 −0
  72. tools/lib/bpf/bpf_helpers.h +1 −0
  73. tools/lib/bpf/btf.c +236 −72
  74. tools/lib/bpf/btf.h +3 −0
  75. tools/lib/bpf/btf_dump.c +4 −3
  76. tools/lib/bpf/btf_relocate.c +1 −1
  77. tools/lib/bpf/elf.c +1 −3
  78. tools/lib/bpf/features.c +6 −9
  79. tools/lib/bpf/gen_loader.c +137 −53
  80. tools/lib/bpf/hashmap.h +10 −10
  81. tools/lib/bpf/libbpf.c +266 −260
  82. tools/lib/bpf/libbpf.h +3 −1
  83. tools/lib/bpf/libbpf.map +5 −0
  84. tools/lib/bpf/libbpf_internal.h +41 −2
  85. tools/lib/bpf/libbpf_version.h +1 −1
  86. tools/lib/bpf/linker.c +80 −25
  87. tools/lib/bpf/relo_core.c +1 −1
  88. tools/lib/bpf/ringbuf.c +18 −16
  89. tools/lib/bpf/skel_internal.h +2 −1
  90. tools/lib/bpf/str_error.c +71 −0
  91. tools/lib/bpf/str_error.h +7 −0
  92. tools/lib/bpf/usdt.c +17 −15
  93. tools/lib/bpf/zip.c +1 −1
  94. tools/lib/subcmd/parse-options.c +1 −1
  95. tools/testing/selftests/bpf/.gitignore +0 −1
  96. tools/testing/selftests/bpf/Makefile +38 −22
  97. tools/testing/selftests/bpf/benchs/bench_trigger.c +2 −1
  98. tools/testing/selftests/bpf/bpf_experimental.h +6 −0
  99. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h +8 −0
  100. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +106 −0
  101. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h +5 −0
  102. tools/testing/selftests/bpf/bpf_util.h +12 −0
  103. tools/testing/selftests/bpf/config.vm +5 −2
  104. tools/testing/selftests/bpf/io_helpers.c +21 −0
  105. tools/testing/selftests/bpf/io_helpers.h +7 −0
  106. tools/testing/selftests/bpf/map_tests/task_storage_map.c +2 −1
  107. tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +1 −1
  108. tools/testing/selftests/bpf/prog_tests/bpf_iter.c +7 −7
  109. tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +0 −4
  110. tools/testing/selftests/bpf/prog_tests/cb_refs.c +2 −2
  111. tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c +5 −5
  112. tools/testing/selftests/bpf/prog_tests/core_reloc.c +1 −1
  113. tools/testing/selftests/bpf/prog_tests/iters.c +2 −2
  114. tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c +126 −0
  115. tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c +2 −0
  116. tools/testing/selftests/bpf/prog_tests/linked_funcs.c +1 −1
  117. tools/testing/selftests/bpf/prog_tests/log_buf.c +0 −3
  118. tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c +0 −4
  119. tools/testing/selftests/bpf/prog_tests/map_in_map.c +131 −1
  120. tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c +1 −1
  121. tools/testing/selftests/bpf/prog_tests/perf_link.c +13 −2
  122. tools/testing/selftests/bpf/prog_tests/raw_tp_null.c +25 −0
  123. tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c +2 −2
  124. tools/testing/selftests/bpf/prog_tests/send_signal.c +111 −35
  125. tools/testing/selftests/bpf/prog_tests/sock_addr.c +0 −4
  126. tools/testing/selftests/bpf/prog_tests/sock_create.c +348 −0
  127. tools/testing/selftests/bpf/prog_tests/sock_post_bind.c +62 −192
  128. tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c +106 −0
  129. tools/testing/selftests/bpf/prog_tests/subskeleton.c +75 −1
  130. tools/testing/selftests/bpf/prog_tests/tailcalls.c +114 −14
  131. tools/testing/selftests/bpf/prog_tests/task_kfunc.c +80 −0
  132. tools/testing/selftests/bpf/prog_tests/task_local_storage.c +282 −4
  133. tools/testing/selftests/bpf/prog_tests/timer_lockup.c +6 −0
  134. tools/testing/selftests/bpf/prog_tests/token.c +12 −7
  135. tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c +311 −50
  136. tools/testing/selftests/bpf/prog_tests/verifier.c +2 −0
  137. tools/testing/selftests/bpf/progs/bpf_iter.h +0 −167
  138. tools/testing/selftests/bpf/progs/bpf_iter_bpf_array_map.c +1 −1
  139. tools/testing/selftests/bpf/progs/bpf_iter_bpf_hash_map.c +1 −1
  140. tools/testing/selftests/bpf/progs/bpf_iter_bpf_link.c +1 −1
  141. tools/testing/selftests/bpf/progs/bpf_iter_bpf_map.c +1 −1
  142. tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_array_map.c +1 −1
  143. tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_hash_map.c +1 −1
  144. tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_helpers.c +1 −1
  145. tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_map.c +1 −1
  146. tools/testing/selftests/bpf/progs/bpf_iter_ipv6_route.c +1 −1
  147. tools/testing/selftests/bpf/progs/bpf_iter_ksym.c +1 −1
  148. tools/testing/selftests/bpf/progs/bpf_iter_netlink.c +1 −1
  149. tools/testing/selftests/bpf/progs/bpf_iter_setsockopt.c +1 −1
  150. tools/testing/selftests/bpf/progs/bpf_iter_setsockopt_unix.c +1 −1
  151. tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c +1 −1
  152. tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c +1 −1
  153. tools/testing/selftests/bpf/progs/bpf_iter_task_file.c +1 −1
  154. tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c +1 −1
  155. tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c +1 −1
  156. tools/testing/selftests/bpf/progs/bpf_iter_tasks.c +1 −1
  157. tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c +1 −1
  158. tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c +1 −1
  159. tools/testing/selftests/bpf/progs/bpf_iter_test_kern3.c +1 −1
  160. tools/testing/selftests/bpf/progs/bpf_iter_test_kern4.c +1 −1
  161. tools/testing/selftests/bpf/progs/bpf_iter_test_kern5.c +1 −1
  162. tools/testing/selftests/bpf/progs/bpf_iter_test_kern6.c +1 −1
  163. tools/testing/selftests/bpf/progs/bpf_iter_test_kern_common.h +1 −1
  164. tools/testing/selftests/bpf/progs/bpf_iter_udp4.c +1 −1
  165. tools/testing/selftests/bpf/progs/bpf_iter_udp6.c +1 −1
  166. tools/testing/selftests/bpf/progs/bpf_iter_unix.c +1 −1
  167. tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c +1 −1
  168. tools/testing/selftests/bpf/progs/cgroup_iter.c +1 −2
  169. tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c +1 −2
  170. tools/testing/selftests/bpf/progs/exceptions_fail.c +2 −2
  171. tools/testing/selftests/bpf/progs/kmem_cache_iter.c +108 −0
  172. tools/testing/selftests/bpf/progs/kprobe_multi_verifier.c +31 −0
  173. tools/testing/selftests/bpf/progs/linked_funcs1.c +8 −0
  174. tools/testing/selftests/bpf/progs/linked_funcs2.c +8 −0
  175. tools/testing/selftests/bpf/progs/preempt_lock.c +7 −7
  176. tools/testing/selftests/bpf/progs/raw_tp_null.c +32 −0
  177. tools/testing/selftests/bpf/progs/struct_ops_detach.c +12 −0
  178. tools/testing/selftests/bpf/progs/struct_ops_private_stack.c +62 −0
  179. tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c +62 −0
  180. tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c +50 −0
  181. tools/testing/selftests/bpf/progs/tailcall_fail.c +64 −0
  182. tools/testing/selftests/bpf/progs/task_kfunc_common.h +1 −0
  183. tools/testing/selftests/bpf/progs/task_kfunc_failure.c +14 −0
  184. tools/testing/selftests/bpf/progs/task_kfunc_success.c +51 −0
  185. tools/testing/selftests/bpf/progs/task_ls_uptr.c +63 −0
  186. tools/testing/selftests/bpf/progs/tc_bpf2bpf.c +3 −2
  187. tools/testing/selftests/bpf/progs/test_send_signal_kern.c +29 −6
  188. tools/testing/selftests/bpf/progs/test_spin_lock_fail.c +2 −2
  189. tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c +5 −1
  190. tools/testing/selftests/bpf/progs/update_map_in_htab.c +30 −0
  191. tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c +3 −3
  192. tools/testing/selftests/bpf/progs/uprobe_multi_session.c +71 −0
  193. tools/testing/selftests/bpf/progs/uprobe_multi_session_cookie.c +48 −0
  194. tools/testing/selftests/bpf/progs/uprobe_multi_session_recursive.c +44 −0
  195. tools/testing/selftests/bpf/progs/uprobe_multi_session_single.c +44 −0
  196. tools/testing/selftests/bpf/progs/uprobe_multi_verifier.c +31 −0
  197. tools/testing/selftests/bpf/progs/uptr_failure.c +105 −0
  198. tools/testing/selftests/bpf/progs/uptr_map_failure.c +27 −0
  199. tools/testing/selftests/bpf/progs/uptr_update_failure.c +42 −0
  200. tools/testing/selftests/bpf/progs/verifier_arena_large.c +108 −2
  201. tools/testing/selftests/bpf/progs/verifier_private_stack.c +272 −0
  202. tools/testing/selftests/bpf/progs/verifier_ref_tracking.c +2 −2
  203. tools/testing/selftests/bpf/progs/verifier_sock.c +60 −0
  204. tools/testing/selftests/bpf/progs/verifier_spin_lock.c +1 −1
  205. tools/testing/selftests/bpf/test_maps.c +0 −4
  206. tools/testing/selftests/bpf/test_progs.c +111 −3
  207. tools/testing/selftests/bpf/test_progs.h +14 −0
  208. tools/testing/selftests/bpf/test_verifier.c +0 −4
  209. tools/testing/selftests/bpf/uprobe_multi.c +4 −0
  210. tools/testing/selftests/bpf/uptr_test_common.h +63 −0
  211. tools/testing/selftests/bpf/veristat.c +159 −2
Documentation/bpf/btf.rst+76−1
@@ -835,7 +835,7 @@ section named by ``btf_ext_info_sec->sec_name_off``. · 834 unmodified lines
835See :ref:`Documentation/bpf/llvm_reloc.rst <btf-co-re-relocations>`835See :ref:`Documentation/bpf/llvm_reloc.rst <btf-co-re-relocations>`
836for more information on CO-RE relocations.836for more information on CO-RE relocations.
837 837
8384.2 .BTF_ids section8384.3 .BTF_ids section
839--------------------839--------------------
840 840
841The .BTF_ids section encodes BTF ID values that are used within the kernel.841The .BTF_ids section encodes BTF ID values that are used within the kernel.
@@ -896,6 +896,81 @@ and is used as a filter when resolving the BTF ID value. · 54 unmodified lines
896All the BTF ID lists and sets are compiled in the .BTF_ids section and896All the BTF ID lists and sets are compiled in the .BTF_ids section and
897resolved during the linking phase of kernel build by ``resolve_btfids`` tool.897resolved during the linking phase of kernel build by ``resolve_btfids`` tool.
898 898
8994.4 .BTF.base section
900---------------------
901Split BTF - where the .BTF section only contains types not in the associated
902base .BTF section - is an extremely efficient way to encode type information
903for kernel modules, since they generally consist of a few module-specific
904types along with a large set of shared kernel types. The former are encoded
905in split BTF, while the latter are encoded in base BTF, resulting in more
906compact representations. A type in split BTF that refers to a type in
907base BTF refers to it using its base BTF ID, and split BTF IDs start
908at last_base_BTF_ID + 1.
909
910The downside of this approach however is that this makes the split BTF
911somewhat brittle - when the base BTF changes, base BTF ID references are
912no longer valid and the split BTF itself becomes useless. The role of the
913.BTF.base section is to make split BTF more resilient for cases where
914the base BTF may change, as is the case for kernel modules not built every
915time the kernel is for example. .BTF.base contains named base types; INTs,
916FLOATs, STRUCTs, UNIONs, ENUM[64]s and FWDs. INTs and FLOATs are fully
917described in .BTF.base sections, while composite types like structs
918and unions are not fully defined - the .BTF.base type simply serves as
919a description of the type the split BTF referred to, so structs/unions
920have 0 members in the .BTF.base section. ENUM[64]s are similarly recorded
921with 0 members. Any other types are added to the split BTF. This
922distillation process then leaves us with a .BTF.base section with
923such minimal descriptions of base types and .BTF split section which refers
924to those base types. Later, we can relocate the split BTF using both the
925information stored in the .BTF.base section and the new .BTF base; the type
926information in the .BTF.base section allows us to update the split BTF
927references to point at the corresponding new base BTF IDs.
928
929BTF relocation happens on kernel module load when a kernel module has a
930.BTF.base section, and libbpf also provides a btf__relocate() API to
931accomplish this.
932
933As an example consider the following base BTF::
934
935 [1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
936 [2] STRUCT 'foo' size=8 vlen=2
937 'f1' type_id=1 bits_offset=0
938 'f2' type_id=1 bits_offset=32
939
940...and associated split BTF::
941
942 [3] PTR '(anon)' type_id=2
943
944i.e. split BTF describes a pointer to struct foo { int f1; int f2 };
945
946.BTF.base will consist of::
947
948 [1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
949 [2] STRUCT 'foo' size=8 vlen=0
950
951If we relocate the split BTF later using the following new base BTF::
952
953 [1] INT 'long unsigned int' size=8 bits_offset=0 nr_bits=64 encoding=(none)
954 [2] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
955 [3] STRUCT 'foo' size=8 vlen=2
956 'f1' type_id=2 bits_offset=0
957 'f2' type_id=2 bits_offset=32
958
959...we can use our .BTF.base description to know that the split BTF reference
960is to struct foo, and relocation results in new split BTF::
961
962 [4] PTR '(anon)' type_id=3
963
964Note that we had to update BTF ID and start BTF ID for the split BTF.
965
966So we see how .BTF.base plays the role of facilitating later relocation,
967leading to more resilient split BTF.
968
969.BTF.base sections will be generated automatically for out-of-tree kernel module
970builds - i.e. where KBUILD_EXTMOD is set (as it would be for "make M=path/2/mod"
971cases). .BTF.base generation requires pahole support for the "distilled_base"
972BTF feature; this is available in pahole v1.28 and later.
973
8995. Using BTF9745. Using BTF
900============975============
901 976
Documentation/bpf/verifier.rst+2−2
@@ -507,7 +507,7 @@ Notes: · 506 unmodified lines
507 from the parent state to the current state.507 from the parent state to the current state.
508 508
509* Details about REG_LIVE_READ32 are omitted.509* Details about REG_LIVE_READ32 are omitted.
510 510
511* Function ``propagate_liveness()`` (see section :ref:`read_marks_for_cache_hits`)511* Function ``propagate_liveness()`` (see section :ref:`read_marks_for_cache_hits`)
512 might override the first parent link. Please refer to the comments in the512 might override the first parent link. Please refer to the comments in the
513 ``propagate_liveness()`` and ``mark_reg_read()`` source code for further513 ``propagate_liveness()`` and ``mark_reg_read()`` source code for further
@@ -571,7 +571,7 @@ works:: · 57 unmodified lines
571 are considered equivalent.571 are considered equivalent.
572 572
573.. _read_marks_for_cache_hits:573.. _read_marks_for_cache_hits:
574 574
575Read marks propagation for cache hits575Read marks propagation for cache hits
576~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~576~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
577 577
arch/arm64/net/bpf_jit_comp.c+31−16
@@ -2094,6 +2094,12 @@ static void restore_args(struct jit_ctx *ctx, int args_off, int nregs) · 2093 unmodified lines
2094 }2094 }
2095}2095}
2096 2096
2097static bool is_struct_ops_tramp(const struct bpf_tramp_links *fentry_links)
2098{
2099 return fentry_links->nr_links == 1 &&
2100 fentry_links->links[0]->link.type == BPF_LINK_TYPE_STRUCT_OPS;
2101}
2102
2097/* Based on the x86's implementation of arch_prepare_bpf_trampoline().2103/* Based on the x86's implementation of arch_prepare_bpf_trampoline().
2098 *2104 *
2099 * bpf prog and function entry before bpf trampoline hooked:2105 * bpf prog and function entry before bpf trampoline hooked:
@@ -2123,6 +2129,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 23 unmodified lines
2123 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];2129 struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
2124 bool save_ret;2130 bool save_ret;
2125 __le32 **branches = NULL;2131 __le32 **branches = NULL;
2132 bool is_struct_ops = is_struct_ops_tramp(fentry);
2126 2133
2127 /* trampoline stack layout:2134 /* trampoline stack layout:
2128 * [ parent ip ]2135 * [ parent ip ]
@@ -2191,11 +2198,14 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 62 unmodified lines
2191 */2198 */
2192 emit_bti(A64_BTI_JC, ctx);2199 emit_bti(A64_BTI_JC, ctx);
2193 2200
2194 /* frame for parent function */2201 /* x9 is not set for struct_ops */
2195 emit(A64_PUSH(A64_FP, A64_R(9), A64_SP), ctx);2202 if (!is_struct_ops) {
2196 emit(A64_MOV(1, A64_FP, A64_SP), ctx);2203 /* frame for parent function */
2204 emit(A64_PUSH(A64_FP, A64_R(9), A64_SP), ctx);
2205 emit(A64_MOV(1, A64_FP, A64_SP), ctx);
2206 }
2197 2207
2198 /* frame for patched function */2208 /* frame for patched function for tracing, or caller for struct_ops */
2199 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);2209 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
2200 emit(A64_MOV(1, A64_FP, A64_SP), ctx);2210 emit(A64_MOV(1, A64_FP, A64_SP), ctx);
2201 2211
@@ -2289,19 +2299,24 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, · 87 unmodified lines
2289 /* reset SP */2299 /* reset SP */
2290 emit(A64_MOV(1, A64_SP, A64_FP), ctx);2300 emit(A64_MOV(1, A64_SP, A64_FP), ctx);
2291 2301
2292 /* pop frames */2302 if (is_struct_ops) {
2293 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);2303 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
2294 emit(A64_POP(A64_FP, A64_R(9), A64_SP), ctx);2304 emit(A64_RET(A64_LR), ctx);
2295
2296 if (flags & BPF_TRAMP_F_SKIP_FRAME) {
2297 /* skip patched function, return to parent */
2298 emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2299 emit(A64_RET(A64_R(9)), ctx);
2300 } else {2305 } else {
2301 /* return to patched function */2306 /* pop frames */
2302 emit(A64_MOV(1, A64_R(10), A64_LR), ctx);2307 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
2303 emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);2308 emit(A64_POP(A64_FP, A64_R(9), A64_SP), ctx);
2304 emit(A64_RET(A64_R(10)), ctx);2309
2310 if (flags & BPF_TRAMP_F_SKIP_FRAME) {
2311 /* skip patched function, return to parent */
2312 emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2313 emit(A64_RET(A64_R(9)), ctx);
2314 } else {
2315 /* return to patched function */
2316 emit(A64_MOV(1, A64_R(10), A64_LR), ctx);
2317 emit(A64_MOV(1, A64_LR, A64_R(9)), ctx);
2318 emit(A64_RET(A64_R(10)), ctx);
2319 }
2305 }2320 }
2306 2321
2307 kfree(branches);2322 kfree(branches);
arch/x86/net/bpf_jit_comp.c+144−5
@@ -325,6 +325,22 @@ struct jit_context { · 324 unmodified lines
325/* Number of bytes that will be skipped on tailcall */325/* Number of bytes that will be skipped on tailcall */
326#define X86_TAIL_CALL_OFFSET (12 + ENDBR_INSN_SIZE)326#define X86_TAIL_CALL_OFFSET (12 + ENDBR_INSN_SIZE)
327 327
328static void push_r9(u8 **pprog)
329{
330 u8 *prog = *pprog;
331
332 EMIT2(0x41, 0x51); /* push r9 */
333 *pprog = prog;
334}
335
336static void pop_r9(u8 **pprog)
337{
338 u8 *prog = *pprog;
339
340 EMIT2(0x41, 0x59); /* pop r9 */
341 *pprog = prog;
342}
343
328static void push_r12(u8 **pprog)344static void push_r12(u8 **pprog)
329{345{
330 u8 *prog = *pprog;346 u8 *prog = *pprog;
@@ -1404,6 +1420,24 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) · 1073 unmodified lines
1404 *pprog = prog;1420 *pprog = prog;
1405}1421}
1406 1422
1423static void emit_priv_frame_ptr(u8 **pprog, void __percpu *priv_frame_ptr)
1424{
1425 u8 *prog = *pprog;
1426
1427 /* movabs r9, priv_frame_ptr */
1428 emit_mov_imm64(&prog, X86_REG_R9, (__force long) priv_frame_ptr >> 32,
1429 (u32) (__force long) priv_frame_ptr);
1430
1431#ifdef CONFIG_SMP
1432 /* add <r9>, gs:[<off>] */
1433 EMIT2(0x65, 0x4c);
1434 EMIT3(0x03, 0x0c, 0x25);
1435 EMIT((u32)(unsigned long)&this_cpu_off, 4);
1436#endif
1437
1438 *pprog = prog;
1439}
1440
1407#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))1441#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
1408 1442
1409#define __LOAD_TCC_PTR(off) \1443#define __LOAD_TCC_PTR(off) \
@@ -1412,6 +1446,10 @@ static void emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) · 2 unmodified lines
1412#define LOAD_TAIL_CALL_CNT_PTR(stack) \1446#define LOAD_TAIL_CALL_CNT_PTR(stack) \
1413 __LOAD_TCC_PTR(BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack))1447 __LOAD_TCC_PTR(BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack))
1414 1448
1449/* Memory size/value to protect private stack overflow/underflow */
1450#define PRIV_STACK_GUARD_SZ 8
1451#define PRIV_STACK_GUARD_VAL 0xEB9F12345678eb9fULL
1452
1415static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,1453static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
1416 int oldproglen, struct jit_context *ctx, bool jmp_padding)1454 int oldproglen, struct jit_context *ctx, bool jmp_padding)
1417{1455{
@@ -1421,18 +1459,28 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 3 unmodified lines
1421 int insn_cnt = bpf_prog->len;1459 int insn_cnt = bpf_prog->len;
1422 bool seen_exit = false;1460 bool seen_exit = false;
1423 u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];1461 u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
1462 void __percpu *priv_frame_ptr = NULL;
1424 u64 arena_vm_start, user_vm_start;1463 u64 arena_vm_start, user_vm_start;
1464 void __percpu *priv_stack_ptr;
1425 int i, excnt = 0;1465 int i, excnt = 0;
1426 int ilen, proglen = 0;1466 int ilen, proglen = 0;
1427 u8 *prog = temp;1467 u8 *prog = temp;
1468 u32 stack_depth;
1428 int err;1469 int err;
1429 1470
1471 stack_depth = bpf_prog->aux->stack_depth;
1472 priv_stack_ptr = bpf_prog->aux->priv_stack_ptr;
1473 if (priv_stack_ptr) {
1474 priv_frame_ptr = priv_stack_ptr + PRIV_STACK_GUARD_SZ + round_up(stack_depth, 8);
1475 stack_depth = 0;
1476 }
1477
1430 arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);1478 arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
1431 user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);1479 user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
1432 1480
1433 detect_reg_usage(insn, insn_cnt, callee_regs_used);1481 detect_reg_usage(insn, insn_cnt, callee_regs_used);
1434 1482
1435 emit_prologue(&prog, bpf_prog->aux->stack_depth,1483 emit_prologue(&prog, stack_depth,
1436 bpf_prog_was_classic(bpf_prog), tail_call_reachable,1484 bpf_prog_was_classic(bpf_prog), tail_call_reachable,
1437 bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);1485 bpf_is_subprog(bpf_prog), bpf_prog->aux->exception_cb);
1438 /* Exception callback will clobber callee regs for its own use, and1486 /* Exception callback will clobber callee regs for its own use, and
@@ -1454,6 +1502,9 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 15 unmodified lines
1454 emit_mov_imm64(&prog, X86_REG_R12,1502 emit_mov_imm64(&prog, X86_REG_R12,
1455 arena_vm_start >> 32, (u32) arena_vm_start);1503 arena_vm_start >> 32, (u32) arena_vm_start);
1456 1504
1505 if (priv_frame_ptr)
1506 emit_priv_frame_ptr(&prog, priv_frame_ptr);
1507
1457 ilen = prog - temp;1508 ilen = prog - temp;
1458 if (rw_image)1509 if (rw_image)
1459 memcpy(rw_image + proglen, temp, ilen);1510 memcpy(rw_image + proglen, temp, ilen);
@@ -1473,6 +1524,14 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 13 unmodified lines
1473 u8 *func;1524 u8 *func;
1474 int nops;1525 int nops;
1475 1526
1527 if (priv_frame_ptr) {
1528 if (src_reg == BPF_REG_FP)
1529 src_reg = X86_REG_R9;
1530
1531 if (dst_reg == BPF_REG_FP)
1532 dst_reg = X86_REG_R9;
1533 }
1534
1476 switch (insn->code) {1535 switch (insn->code) {
1477 /* ALU */1536 /* ALU */
1478 case BPF_ALU | BPF_ADD | BPF_X:1537 case BPF_ALU | BPF_ADD | BPF_X:
@@ -2127,15 +2186,21 @@ populate_extable: · 648 unmodified lines
2127 u8 *ip = image + addrs[i - 1];2186 u8 *ip = image + addrs[i - 1];
2128 2187
2129 func = (u8 *) __bpf_call_base + imm32;2188 func = (u8 *) __bpf_call_base + imm32;
2130 if (tail_call_reachable) {2189 if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
2131 LOAD_TAIL_CALL_CNT_PTR(bpf_prog->aux->stack_depth);2190 LOAD_TAIL_CALL_CNT_PTR(stack_depth);
2132 ip += 7;2191 ip += 7;
2133 }2192 }
2134 if (!imm32)2193 if (!imm32)
2135 return -EINVAL;2194 return -EINVAL;
2195 if (priv_frame_ptr) {
2196 push_r9(&prog);
2197 ip += 2;
2198 }
2136 ip += x86_call_depth_emit_accounting(&prog, func, ip);2199 ip += x86_call_depth_emit_accounting(&prog, func, ip);
2137 if (emit_call(&prog, func, ip))2200 if (emit_call(&prog, func, ip))
2138 return -EINVAL;2201 return -EINVAL;
2202 if (priv_frame_ptr)
2203 pop_r9(&prog);
2139 break;2204 break;
2140 }2205 }
2141 2206
@@ -2145,13 +2210,13 @@ populate_extable: · 3 unmodified lines
2145 &bpf_prog->aux->poke_tab[imm32 - 1],2210 &bpf_prog->aux->poke_tab[imm32 - 1],
2146 &prog, image + addrs[i - 1],2211 &prog, image + addrs[i - 1],
2147 callee_regs_used,2212 callee_regs_used,
2148 bpf_prog->aux->stack_depth,2213 stack_depth,
2149 ctx);2214 ctx);
2150 else2215 else
2151 emit_bpf_tail_call_indirect(bpf_prog,2216 emit_bpf_tail_call_indirect(bpf_prog,
2152 &prog,2217 &prog,
2153 callee_regs_used,2218 callee_regs_used,
2154 bpf_prog->aux->stack_depth,2219 stack_depth,
2155 image + addrs[i - 1],2220 image + addrs[i - 1],
2156 ctx);2221 ctx);
2157 break;2222 break;
@@ -3303,6 +3368,42 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func · 1145 unmodified lines
3303 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf);3368 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf);
3304}3369}
3305 3370
3371static const char *bpf_get_prog_name(struct bpf_prog *prog)
3372{
3373 if (prog->aux->ksym.prog)
3374 return prog->aux->ksym.name;
3375 return prog->aux->name;
3376}
3377
3378static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
3379{
3380 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
3381 u64 *stack_ptr;
3382
3383 for_each_possible_cpu(cpu) {
3384 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
3385 stack_ptr[0] = PRIV_STACK_GUARD_VAL;
3386 stack_ptr[underflow_idx] = PRIV_STACK_GUARD_VAL;
3387 }
3388}
3389
3390static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size,
3391 struct bpf_prog *prog)
3392{
3393 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
3394 u64 *stack_ptr;
3395
3396 for_each_possible_cpu(cpu) {
3397 stack_ptr = per_cpu_ptr(priv_stack_ptr, cpu);
3398 if (stack_ptr[0] != PRIV_STACK_GUARD_VAL ||
3399 stack_ptr[underflow_idx] != PRIV_STACK_GUARD_VAL) {
3400 pr_err("BPF private stack overflow/underflow detected for prog %sx\n",
3401 bpf_get_prog_name(prog));
3402 break;
3403 }
3404 }
3405}
3406
3306struct x64_jit_data {3407struct x64_jit_data {
3307 struct bpf_binary_header *rw_header;3408 struct bpf_binary_header *rw_header;
3308 struct bpf_binary_header *header;3409 struct bpf_binary_header *header;
@@ -3320,7 +3421,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 11 unmodified lines
3320 struct bpf_binary_header *rw_header = NULL;3421 struct bpf_binary_header *rw_header = NULL;
3321 struct bpf_binary_header *header = NULL;3422 struct bpf_binary_header *header = NULL;
3322 struct bpf_prog *tmp, *orig_prog = prog;3423 struct bpf_prog *tmp, *orig_prog = prog;
3424 void __percpu *priv_stack_ptr = NULL;
3323 struct x64_jit_data *jit_data;3425 struct x64_jit_data *jit_data;
3426 int priv_stack_alloc_sz;
3324 int proglen, oldproglen = 0;3427 int proglen, oldproglen = 0;
3325 struct jit_context ctx = {};3428 struct jit_context ctx = {};
3326 bool tmp_blinded = false;3429 bool tmp_blinded = false;
@@ -3356,6 +3459,23 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 29 unmodified lines
3356 }3459 }
3357 prog->aux->jit_data = jit_data;3460 prog->aux->jit_data = jit_data;
3358 }3461 }
3462 priv_stack_ptr = prog->aux->priv_stack_ptr;
3463 if (!priv_stack_ptr && prog->aux->jits_use_priv_stack) {
3464 /* Allocate actual private stack size with verifier-calculated
3465 * stack size plus two memory guards to protect overflow and
3466 * underflow.
3467 */
3468 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
3469 2 * PRIV_STACK_GUARD_SZ;
3470 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL);
3471 if (!priv_stack_ptr) {
3472 prog = orig_prog;
3473 goto out_priv_stack;
3474 }
3475
3476 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
3477 prog->aux->priv_stack_ptr = priv_stack_ptr;
3478 }
3359 addrs = jit_data->addrs;3479 addrs = jit_data->addrs;
3360 if (addrs) {3480 if (addrs) {
3361 ctx = jit_data->ctx;3481 ctx = jit_data->ctx;
@@ -3491,6 +3611,11 @@ out_image: · 129 unmodified lines
3491 bpf_prog_fill_jited_linfo(prog, addrs + 1);3611 bpf_prog_fill_jited_linfo(prog, addrs + 1);
3492out_addrs:3612out_addrs:
3493 kvfree(addrs);3613 kvfree(addrs);
3614 if (!image && priv_stack_ptr) {
3615 free_percpu(priv_stack_ptr);
3616 prog->aux->priv_stack_ptr = NULL;
3617 }
3618out_priv_stack:
3494 kfree(jit_data);3619 kfree(jit_data);
3495 prog->aux->jit_data = NULL;3620 prog->aux->jit_data = NULL;
3496 }3621 }
@@ -3529,6 +3654,8 @@ void bpf_jit_free(struct bpf_prog *prog) · 32 unmodified lines
3529 if (prog->jited) {3654 if (prog->jited) {
3530 struct x64_jit_data *jit_data = prog->aux->jit_data;3655 struct x64_jit_data *jit_data = prog->aux->jit_data;
3531 struct bpf_binary_header *hdr;3656 struct bpf_binary_header *hdr;
3657 void __percpu *priv_stack_ptr;
3658 int priv_stack_alloc_sz;
3532 3659
3533 /*3660 /*
3534 * If we fail the final pass of JIT (from jit_subprogs),3661 * If we fail the final pass of JIT (from jit_subprogs),
@@ -3544,6 +3671,13 @@ void bpf_jit_free(struct bpf_prog *prog) · 9 unmodified lines
3544 prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset();3671 prog->bpf_func = (void *)prog->bpf_func - cfi_get_offset();
3545 hdr = bpf_jit_binary_pack_hdr(prog);3672 hdr = bpf_jit_binary_pack_hdr(prog);
3546 bpf_jit_binary_pack_free(hdr, NULL);3673 bpf_jit_binary_pack_free(hdr, NULL);
3674 priv_stack_ptr = prog->aux->priv_stack_ptr;
3675 if (priv_stack_ptr) {
3676 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
3677 2 * PRIV_STACK_GUARD_SZ;
3678 priv_stack_check_guard(priv_stack_ptr, priv_stack_alloc_sz, prog);
3679 free_percpu(prog->aux->priv_stack_ptr);
3680 }
3547 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));3681 WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
3548 }3682 }
3549 3683
@@ -3559,6 +3693,11 @@ bool bpf_jit_supports_exceptions(void) · 9 unmodified lines
3559 return IS_ENABLED(CONFIG_UNWINDER_ORC);3693 return IS_ENABLED(CONFIG_UNWINDER_ORC);
3560}3694}
3561 3695
3696bool bpf_jit_supports_private_stack(void)
3697{
3698 return true;
3699}
3700
3562void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)3701void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
3563{3702{
3564#if defined(CONFIG_UNWINDER_ORC)3703#if defined(CONFIG_UNWINDER_ORC)
include/linux/bpf.h+54−9
@@ -203,6 +203,7 @@ enum btf_field_type { · 202 unmodified lines
203 BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,203 BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD,
204 BPF_REFCOUNT = (1 << 9),204 BPF_REFCOUNT = (1 << 9),
205 BPF_WORKQUEUE = (1 << 10),205 BPF_WORKQUEUE = (1 << 10),
206 BPF_UPTR = (1 << 11),
206};207};
207 208
208typedef void (*btf_dtor_kfunc_t)(void *);209typedef void (*btf_dtor_kfunc_t)(void *);
@@ -322,6 +323,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) · 113 unmodified lines
322 return "kptr";323 return "kptr";
323 case BPF_KPTR_PERCPU:324 case BPF_KPTR_PERCPU:
324 return "percpu_kptr";325 return "percpu_kptr";
326 case BPF_UPTR:
327 return "uptr";
325 case BPF_LIST_HEAD:328 case BPF_LIST_HEAD:
326 return "bpf_list_head";329 return "bpf_list_head";
327 case BPF_LIST_NODE:330 case BPF_LIST_NODE:
@@ -350,6 +353,7 @@ static inline u32 btf_field_type_size(enum btf_field_type type) · 22 unmodified lines
350 case BPF_KPTR_UNREF:353 case BPF_KPTR_UNREF:
351 case BPF_KPTR_REF:354 case BPF_KPTR_REF:
352 case BPF_KPTR_PERCPU:355 case BPF_KPTR_PERCPU:
356 case BPF_UPTR:
353 return sizeof(u64);357 return sizeof(u64);
354 case BPF_LIST_HEAD:358 case BPF_LIST_HEAD:
355 return sizeof(struct bpf_list_head);359 return sizeof(struct bpf_list_head);
@@ -379,6 +383,7 @@ static inline u32 btf_field_type_align(enum btf_field_type type) · 23 unmodified lines
379 case BPF_KPTR_UNREF:383 case BPF_KPTR_UNREF:
380 case BPF_KPTR_REF:384 case BPF_KPTR_REF:
381 case BPF_KPTR_PERCPU:385 case BPF_KPTR_PERCPU:
386 case BPF_UPTR:
382 return __alignof__(u64);387 return __alignof__(u64);
383 case BPF_LIST_HEAD:388 case BPF_LIST_HEAD:
384 return __alignof__(struct bpf_list_head);389 return __alignof__(struct bpf_list_head);
@@ -419,6 +424,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) · 34 unmodified lines
419 case BPF_KPTR_UNREF:424 case BPF_KPTR_UNREF:
420 case BPF_KPTR_REF:425 case BPF_KPTR_REF:
421 case BPF_KPTR_PERCPU:426 case BPF_KPTR_PERCPU:
427 case BPF_UPTR:
422 break;428 break;
423 default:429 default:
424 WARN_ON_ONCE(1);430 WARN_ON_ONCE(1);
@@ -507,6 +513,25 @@ static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src · 82 unmodified lines
507 bpf_obj_memcpy(map->record, dst, src, map->value_size, true);513 bpf_obj_memcpy(map->record, dst, src, map->value_size, true);
508}514}
509 515
516static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src)
517{
518 unsigned long *src_uptr, *dst_uptr;
519 const struct btf_field *field;
520 int i;
521
522 if (!btf_record_has_field(rec, BPF_UPTR))
523 return;
524
525 for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
526 if (field->type != BPF_UPTR)
527 continue;
528
529 src_uptr = src + field->offset;
530 dst_uptr = dst + field->offset;
531 swap(*src_uptr, *dst_uptr);
532 }
533}
534
510static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)535static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size)
511{536{
512 u32 curr_off = 0;537 u32 curr_off = 0;
@@ -907,10 +932,6 @@ enum bpf_reg_type { · 394 unmodified lines
907 * additional context, assume the value is non-null.932 * additional context, assume the value is non-null.
908 */933 */
909 PTR_TO_BTF_ID,934 PTR_TO_BTF_ID,
910 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
911 * been checked for null. Used primarily to inform the verifier
912 * an explicit null check is required for this struct.
913 */
914 PTR_TO_MEM, /* reg points to valid memory region */935 PTR_TO_MEM, /* reg points to valid memory region */
915 PTR_TO_ARENA,936 PTR_TO_ARENA,
916 PTR_TO_BUF, /* reg points to a read/write buffer */937 PTR_TO_BUF, /* reg points to a read/write buffer */
@@ -923,6 +944,10 @@ enum bpf_reg_type { · 6 unmodified lines
923 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,944 PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET,
924 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,945 PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON,
925 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,946 PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK,
947 /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
948 * been checked for null. Used primarily to inform the verifier
949 * an explicit null check is required for this struct.
950 */
926 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,951 PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID,
927 952
928 /* This must be the last entry. Its purpose is to ensure the enum is953 /* This must be the last entry. Its purpose is to ensure the enum is
@@ -1300,8 +1325,12 @@ void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len); · 371 unmodified lines
1300bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);1325bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr);
1301 1326
1302#ifdef CONFIG_BPF_JIT1327#ifdef CONFIG_BPF_JIT
1303int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);1328int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1304int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr);1329 struct bpf_trampoline *tr,
1330 struct bpf_prog *tgt_prog);
1331int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1332 struct bpf_trampoline *tr,
1333 struct bpf_prog *tgt_prog);
1305struct bpf_trampoline *bpf_trampoline_get(u64 key,1334struct bpf_trampoline *bpf_trampoline_get(u64 key,
1306 struct bpf_attach_target_info *tgt_info);1335 struct bpf_attach_target_info *tgt_info);
1307void bpf_trampoline_put(struct bpf_trampoline *tr);1336void bpf_trampoline_put(struct bpf_trampoline *tr);
@@ -1373,7 +1402,8 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func · 65 unmodified lines
1373void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,1402void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
1374 struct bpf_prog *to);1403 struct bpf_prog *to);
1375/* Called only from JIT-enabled code, so there's no need for stubs. */1404/* Called only from JIT-enabled code, so there's no need for stubs. */
1376void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym);1405void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym);
1406void bpf_image_ksym_add(struct bpf_ksym *ksym);
1377void bpf_image_ksym_del(struct bpf_ksym *ksym);1407void bpf_image_ksym_del(struct bpf_ksym *ksym);
1378void bpf_ksym_add(struct bpf_ksym *ksym);1408void bpf_ksym_add(struct bpf_ksym *ksym);
1379void bpf_ksym_del(struct bpf_ksym *ksym);1409void bpf_ksym_del(struct bpf_ksym *ksym);
@@ -1382,12 +1412,14 @@ void bpf_jit_uncharge_modmem(u32 size); · 2 unmodified lines
1382bool bpf_prog_has_trampoline(const struct bpf_prog *prog);1412bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1383#else1413#else
1384static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,1414static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1385 struct bpf_trampoline *tr)1415 struct bpf_trampoline *tr,
1416 struct bpf_prog *tgt_prog)
1386{1417{
1387 return -ENOTSUPP;1418 return -ENOTSUPP;
1388}1419}
1389static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,1420static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
1390 struct bpf_trampoline *tr)1421 struct bpf_trampoline *tr,
1422 struct bpf_prog *tgt_prog)
1391{1423{
1392 return -ENOTSUPP;1424 return -ENOTSUPP;
1393}1425}
@@ -1476,6 +1508,7 @@ struct bpf_prog_aux { · 82 unmodified lines
1476 u32 max_rdwr_access;1508 u32 max_rdwr_access;
1477 struct btf *attach_btf;1509 struct btf *attach_btf;
1478 const struct bpf_ctx_arg_aux *ctx_arg_info;1510 const struct bpf_ctx_arg_aux *ctx_arg_info;
1511 void __percpu *priv_stack_ptr;
1479 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */1512 struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
1480 struct bpf_prog *dst_prog;1513 struct bpf_prog *dst_prog;
1481 struct bpf_trampoline *dst_trampoline;1514 struct bpf_trampoline *dst_trampoline;
@@ -1491,7 +1524,13 @@ struct bpf_prog_aux { · 9 unmodified lines
1491 bool xdp_has_frags;1524 bool xdp_has_frags;
1492 bool exception_cb;1525 bool exception_cb;
1493 bool exception_boundary;1526 bool exception_boundary;
1527 bool is_extended; /* true if extended by freplace program */
1528 bool jits_use_priv_stack;
1529 bool priv_stack_requested;
1530 u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
1531 struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
1494 struct bpf_arena *arena;1532 struct bpf_arena *arena;
1533 void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */
1495 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */1534 /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
1496 const struct btf_type *attach_func_proto;1535 const struct btf_type *attach_func_proto;
1497 /* function name for valid attach_btf_id */1536 /* function name for valid attach_btf_id */
@@ -3461,4 +3500,10 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) · 1963 unmodified lines
3461 return prog->aux->func_idx != 0;3500 return prog->aux->func_idx != 0;
3462}3501}
3463 3502
3503static inline bool bpf_prog_is_raw_tp(const struct bpf_prog *prog)
3504{
3505 return prog->type == BPF_PROG_TYPE_TRACING &&
3506 prog->expected_attach_type == BPF_TRACE_RAW_TP;
3507}
3508
3464#endif /* _LINUX_BPF_H */3509#endif /* _LINUX_BPF_H */
include/linux/bpf_local_storage.h+9−3
@@ -77,7 +77,13 @@ struct bpf_local_storage_elem { · 76 unmodified lines
77 struct hlist_node map_node; /* Linked to bpf_local_storage_map */77 struct hlist_node map_node; /* Linked to bpf_local_storage_map */
78 struct hlist_node snode; /* Linked to bpf_local_storage */78 struct hlist_node snode; /* Linked to bpf_local_storage */
79 struct bpf_local_storage __rcu *local_storage;79 struct bpf_local_storage __rcu *local_storage;
80 struct rcu_head rcu;80 union {
81 struct rcu_head rcu;
82 struct hlist_node free_node; /* used to postpone
83 * bpf_selem_free
84 * after raw_spin_unlock
85 */
86 };
81 /* 8 bytes hole */87 /* 8 bytes hole */
82 /* The data is stored in another cacheline to minimize88 /* The data is stored in another cacheline to minimize
83 * the number of cachelines access during a cache hit.89 * the number of cachelines access during a cache hit.
@@ -181,7 +187,7 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap, · 97 unmodified lines
181 187
182struct bpf_local_storage_elem *188struct bpf_local_storage_elem *
183bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,189bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
184 bool charge_mem, gfp_t gfp_flags);190 bool charge_mem, bool swap_uptrs, gfp_t gfp_flags);
185 191
186void bpf_selem_free(struct bpf_local_storage_elem *selem,192void bpf_selem_free(struct bpf_local_storage_elem *selem,
187 struct bpf_local_storage_map *smap,193 struct bpf_local_storage_map *smap,
@@ -195,7 +201,7 @@ bpf_local_storage_alloc(void *owner, · 7 unmodified lines
195 201
196struct bpf_local_storage_data *202struct bpf_local_storage_data *
197bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,203bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
198 void *value, u64 map_flags, gfp_t gfp_flags);204 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags);
199 205
200u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);206u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map);
201 207
include/linux/bpf_verifier.h+31−36
@@ -48,22 +48,6 @@ enum bpf_reg_liveness { · 47 unmodified lines
48 REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */48 REG_LIVE_DONE = 0x8, /* liveness won't be updating this register anymore */
49};49};
50 50
51/* For every reg representing a map value or allocated object pointer,
52 * we consider the tuple of (ptr, id) for them to be unique in verifier
53 * context and conside them to not alias each other for the purposes of
54 * tracking lock state.
55 */
56struct bpf_active_lock {
57 /* This can either be reg->map_ptr or reg->btf. If ptr is NULL,
58 * there's no active lock held, and other fields have no
59 * meaning. If non-NULL, it indicates that a lock is held and
60 * id member has the reg->id of the register which can be >= 0.
61 */
62 void *ptr;
63 /* This will be reg->id */
64 u32 id;
65};
66
67#define ITER_PREFIX "bpf_iter_"51#define ITER_PREFIX "bpf_iter_"
68 52
69enum bpf_iter_state {53enum bpf_iter_state {
@@ -266,6 +250,13 @@ struct bpf_stack_state { · 196 unmodified lines
266};250};
267 251
268struct bpf_reference_state {252struct bpf_reference_state {
253 /* Each reference object has a type. Ensure REF_TYPE_PTR is zero to
254 * default to pointer reference on zero initialization of a state.
255 */
256 enum ref_state_type {
257 REF_TYPE_PTR = 0,
258 REF_TYPE_LOCK,
259 } type;
269 /* Track each reference created with a unique id, even if the same260 /* Track each reference created with a unique id, even if the same
270 * instruction creates the reference multiple times (eg, via CALL).261 * instruction creates the reference multiple times (eg, via CALL).
271 */262 */
@@ -274,17 +265,10 @@ struct bpf_reference_state { · 2 unmodified lines
274 * is used purely to inform the user of a reference leak.265 * is used purely to inform the user of a reference leak.
275 */266 */
276 int insn_idx;267 int insn_idx;
277 /* There can be a case like:268 /* Use to keep track of the source object of a lock, to ensure
278 * main (frame 0)269 * it matches on unlock.
279 * cb (frame 1)270 */
280 * func (frame 3)271 void *ptr;
281 * cb (frame 4)
282 * Hence for frame 4, if callback_ref just stored boolean, it would be
283 * impossible to distinguish nested callback refs. Hence store the
284 * frameno and compare that to callback_ref in check_reference_leak when
285 * exiting a callback function.
286 */
287 int callback_ref;
288};272};
289 273
290struct bpf_retval_range {274struct bpf_retval_range {
@@ -332,6 +316,7 @@ struct bpf_func_state { · 41 unmodified lines
332 316
333 /* The following fields should be last. See copy_func_state() */317 /* The following fields should be last. See copy_func_state() */
334 int acquired_refs;318 int acquired_refs;
319 int active_locks;
335 struct bpf_reference_state *refs;320 struct bpf_reference_state *refs;
336 /* The state of the stack. Each element of the array describes BPF_REG_SIZE321 /* The state of the stack. Each element of the array describes BPF_REG_SIZE
337 * (i.e. 8) bytes worth of stack memory.322 * (i.e. 8) bytes worth of stack memory.
@@ -349,7 +334,7 @@ struct bpf_func_state { · 11 unmodified lines
349 334
350#define MAX_CALL_FRAMES 8335#define MAX_CALL_FRAMES 8
351 336
352/* instruction history flags, used in bpf_jmp_history_entry.flags field */337/* instruction history flags, used in bpf_insn_hist_entry.flags field */
353enum {338enum {
354 /* instruction references stack slot through PTR_TO_STACK register;339 /* instruction references stack slot through PTR_TO_STACK register;
355 * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8)340 * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8)
@@ -367,7 +352,7 @@ enum { · 11 unmodified lines
367static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);352static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);
368static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8);353static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8);
369 354
370struct bpf_jmp_history_entry {355struct bpf_insn_hist_entry {
371 u32 idx;356 u32 idx;
372 /* insn idx can't be bigger than 1 million */357 /* insn idx can't be bigger than 1 million */
373 u32 prev_idx : 22;358 u32 prev_idx : 22;
@@ -434,7 +419,6 @@ struct bpf_verifier_state { · 60 unmodified lines
434 u32 insn_idx;419 u32 insn_idx;
435 u32 curframe;420 u32 curframe;
436 421
437 struct bpf_active_lock active_lock;
438 bool speculative;422 bool speculative;
439 bool active_rcu_lock;423 bool active_rcu_lock;
440 u32 active_preempt_lock;424 u32 active_preempt_lock;
@@ -458,13 +442,14 @@ struct bpf_verifier_state { · 17 unmodified lines
458 * See get_loop_entry() for more information.442 * See get_loop_entry() for more information.
459 */443 */
460 struct bpf_verifier_state *loop_entry;444 struct bpf_verifier_state *loop_entry;
461 /* jmp history recorded from first to last.445 /* Sub-range of env->insn_hist[] corresponding to this state's
462 * backtracking is using it to go from last to first.446 * instruction history.
463 * For most states jmp_history_cnt is [0-3].447 * Backtracking is using it to go from last to first.
448 * For most states instruction history is short, 0-3 instructions.
464 * For loops can go up to ~40.449 * For loops can go up to ~40.
465 */450 */
466 struct bpf_jmp_history_entry *jmp_history;451 u32 insn_hist_start;
467 u32 jmp_history_cnt;452 u32 insn_hist_end;
468 u32 dfs_depth;453 u32 dfs_depth;
469 u32 callback_unroll_depth;454 u32 callback_unroll_depth;
470 u32 may_goto_depth;455 u32 may_goto_depth;
@@ -649,6 +634,12 @@ struct bpf_subprog_arg_info { · 178 unmodified lines
649 };634 };
650};635};
651 636
637enum priv_stack_mode {
638 PRIV_STACK_UNKNOWN,
639 NO_PRIV_STACK,
640 PRIV_STACK_ADAPTIVE,
641};
642
652struct bpf_subprog_info {643struct bpf_subprog_info {
653 /* 'start' has to be the first field otherwise find_subprog() won't work */644 /* 'start' has to be the first field otherwise find_subprog() won't work */
654 u32 start; /* insn idx of function entry point */645 u32 start; /* insn idx of function entry point */
@@ -669,6 +660,7 @@ struct bpf_subprog_info { · 14 unmodified lines
669 /* true if bpf_fastcall stack region is used by functions that can't be inlined */660 /* true if bpf_fastcall stack region is used by functions that can't be inlined */
670 bool keep_fastcall_stack: 1;661 bool keep_fastcall_stack: 1;
671 662
663 enum priv_stack_mode priv_stack_mode;
672 u8 arg_cnt;664 u8 arg_cnt;
673 struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];665 struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
674};666};
@@ -747,7 +739,9 @@ struct bpf_verifier_env { · 72 unmodified lines
747 int cur_stack;739 int cur_stack;
748 } cfg;740 } cfg;
749 struct backtrack_state bt;741 struct backtrack_state bt;
750 struct bpf_jmp_history_entry *cur_hist_ent;742 struct bpf_insn_hist_entry *insn_hist;
743 struct bpf_insn_hist_entry *cur_hist_ent;
744 u32 insn_hist_cap;
751 u32 pass_cnt; /* number of times do_check() was called */745 u32 pass_cnt; /* number of times do_check() was called */
752 u32 subprog_cnt;746 u32 subprog_cnt;
753 /* number of instructions analyzed by the verifier */747 /* number of instructions analyzed by the verifier */
@@ -888,6 +882,7 @@ static inline bool bpf_prog_check_recur(const struct bpf_prog *prog) · 134 unmodified lines
888 case BPF_PROG_TYPE_TRACING:882 case BPF_PROG_TYPE_TRACING:
889 return prog->expected_attach_type != BPF_TRACE_ITER;883 return prog->expected_attach_type != BPF_TRACE_ITER;
890 case BPF_PROG_TYPE_STRUCT_OPS:884 case BPF_PROG_TYPE_STRUCT_OPS:
885 return prog->aux->jits_use_priv_stack;
891 case BPF_PROG_TYPE_LSM:886 case BPF_PROG_TYPE_LSM:
892 return false;887 return false;
893 default:888 default:
include/linux/btf.h+11−11
@@ -75,6 +75,7 @@ · 74 unmodified lines
75#define KF_ITER_NEXT (1 << 9) /* kfunc implements BPF iter next method */75#define KF_ITER_NEXT (1 << 9) /* kfunc implements BPF iter next method */
76#define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */76#define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
77#define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */77#define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
78#define KF_FASTCALL (1 << 12) /* kfunc supports bpf_fastcall protocol */
78 79
79/*80/*
80 * Tag marking a kernel function as a kfunc. This is meant to minimize the81 * Tag marking a kernel function as a kfunc. This is meant to minimize the
@@ -581,6 +582,16 @@ int get_kern_ctx_btf_id(struct bpf_verifier_log *log, enum bpf_prog_type prog_ty · 500 unmodified lines
581bool btf_types_are_same(const struct btf *btf1, u32 id1,582bool btf_types_are_same(const struct btf *btf1, u32 id1,
582 const struct btf *btf2, u32 id2);583 const struct btf *btf2, u32 id2);
583int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx);584int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx);
585
586static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t)
587{
588 if (!btf_type_is_ptr(t))
589 return false;
590
591 t = btf_type_skip_modifiers(btf, t->type, NULL);
592
593 return btf_type_is_struct(t);
594}
584#else595#else
585static inline const struct btf_type *btf_type_by_id(const struct btf *btf,596static inline const struct btf_type *btf_type_by_id(const struct btf *btf,
586 u32 type_id)597 u32 type_id)
@@ -660,15 +671,4 @@ static inline int btf_check_iter_arg(struct btf *btf, const struct btf_type *fun · 73 unmodified lines
660 return -EOPNOTSUPP;671 return -EOPNOTSUPP;
661}672}
662#endif673#endif
663
664static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t)
665{
666 if (!btf_type_is_ptr(t))
667 return false;
668
669 t = btf_type_skip_modifiers(btf, t->type, NULL);
670
671 return btf_type_is_struct(t);
672}
673
674#endif674#endif
include/linux/btf_ids.h+1−0
@@ -283,5 +283,6 @@ extern u32 btf_tracing_ids[]; · 282 unmodified lines
283extern u32 bpf_cgroup_btf_id[];283extern u32 bpf_cgroup_btf_id[];
284extern u32 bpf_local_storage_map_btf_id[];284extern u32 bpf_local_storage_map_btf_id[];
285extern u32 btf_bpf_map_id[];285extern u32 btf_bpf_map_id[];
286extern u32 bpf_kmem_cache_btf_id[];
286 287
287#endif288#endif
include/linux/filter.h+1−0
@@ -1119,6 +1119,7 @@ bool bpf_jit_supports_exceptions(void); · 1118 unmodified lines
1119bool bpf_jit_supports_ptr_xchg(void);1119bool bpf_jit_supports_ptr_xchg(void);
1120bool bpf_jit_supports_arena(void);1120bool bpf_jit_supports_arena(void);
1121bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);1121bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
1122bool bpf_jit_supports_private_stack(void);
1122u64 bpf_arch_uaddress_limit(void);1123u64 bpf_arch_uaddress_limit(void);
1123void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);1124void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
1124bool bpf_helper_changes_pkt_data(void *func);1125bool bpf_helper_changes_pkt_data(void *func);
include/uapi/linux/bpf.h+4−5
@@ -1116,6 +1116,7 @@ enum bpf_attach_type { · 1115 unmodified lines
1116 BPF_NETKIT_PRIMARY,1116 BPF_NETKIT_PRIMARY,
1117 BPF_NETKIT_PEER,1117 BPF_NETKIT_PEER,
1118 BPF_TRACE_KPROBE_SESSION,1118 BPF_TRACE_KPROBE_SESSION,
1119 BPF_TRACE_UPROBE_SESSION,
1119 __MAX_BPF_ATTACH_TYPE1120 __MAX_BPF_ATTACH_TYPE
1120};1121};
1121 1122
@@ -1973,6 +1974,8 @@ union bpf_attr { · 851 unmodified lines
1973 * program.1974 * program.
1974 * Return1975 * Return
1975 * The SMP id of the processor running the program.1976 * The SMP id of the processor running the program.
1977 * Attributes
1978 * __bpf_fastcall
1976 *1979 *
1977 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)1980 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
1978 * Description1981 * Description
@@ -3104,10 +3107,6 @@ union bpf_attr { · 1125 unmodified lines
3104 * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration3107 * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration
3105 * option, and in this case it only works on functions tagged with3108 * option, and in this case it only works on functions tagged with
3106 * **ALLOW_ERROR_INJECTION** in the kernel code.3109 * **ALLOW_ERROR_INJECTION** in the kernel code.
3107 *
3108 * Also, the helper is only available for the architectures having
3109 * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing,
3110 * x86 architecture is the only one to support this feature.
3111 * Return3110 * Return
3112 * 03111 * 0
3113 *3112 *
@@ -5372,7 +5371,7 @@ union bpf_attr { · 2258 unmodified lines
5372 * Currently, the **flags** must be 0. Currently, nr_loops is5371 * Currently, the **flags** must be 0. Currently, nr_loops is
5373 * limited to 1 << 23 (~8 million) loops.5372 * limited to 1 << 23 (~8 million) loops.
5374 *5373 *
5375 * long (\*callback_fn)(u32 index, void \*ctx);5374 * long (\*callback_fn)(u64 index, void \*ctx);
5376 *5375 *
5377 * where **index** is the current index in the loop. The index5376 * where **index** is the current index in the loop. The index
5378 * is zero-indexed.5377 * is zero-indexed.
kernel/bpf/Makefile+2−1
@@ -16,7 +16,7 @@ obj-$(CONFIG_BPF_SYSCALL) += disasm.o mprog.o · 15 unmodified lines
16obj-$(CONFIG_BPF_JIT) += trampoline.o16obj-$(CONFIG_BPF_JIT) += trampoline.o
17obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o17obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o
18ifeq ($(CONFIG_MMU)$(CONFIG_64BIT),yy)18ifeq ($(CONFIG_MMU)$(CONFIG_64BIT),yy)
19obj-$(CONFIG_BPF_SYSCALL) += arena.o19obj-$(CONFIG_BPF_SYSCALL) += arena.o range_tree.o
20endif20endif
21obj-$(CONFIG_BPF_JIT) += dispatcher.o21obj-$(CONFIG_BPF_JIT) += dispatcher.o
22ifeq ($(CONFIG_NET),y)22ifeq ($(CONFIG_NET),y)
@@ -52,3 +52,4 @@ obj-$(CONFIG_BPF_PRELOAD) += preload/ · 29 unmodified lines
52obj-$(CONFIG_BPF_SYSCALL) += relo_core.o52obj-$(CONFIG_BPF_SYSCALL) += relo_core.o
53obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o53obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o
54obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o54obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o
55obj-$(CONFIG_BPF_SYSCALL) += kmem_cache_iter.o
kernel/bpf/arena.c+24−14
@@ -3,9 +3,11 @@ · 2 unmodified lines
3#include <linux/bpf.h>3#include <linux/bpf.h>
4#include <linux/btf.h>4#include <linux/btf.h>
5#include <linux/err.h>5#include <linux/err.h>
6#include "linux/filter.h"
6#include <linux/btf_ids.h>7#include <linux/btf_ids.h>
7#include <linux/vmalloc.h>8#include <linux/vmalloc.h>
8#include <linux/pagemap.h>9#include <linux/pagemap.h>
10#include "range_tree.h"
9 11
10/*12/*
11 * bpf_arena is a sparsely populated shared memory region between bpf program and13 * bpf_arena is a sparsely populated shared memory region between bpf program and
@@ -45,7 +47,7 @@ struct bpf_arena { · 33 unmodified lines
45 u64 user_vm_start;47 u64 user_vm_start;
46 u64 user_vm_end;48 u64 user_vm_end;
47 struct vm_struct *kern_vm;49 struct vm_struct *kern_vm;
48 struct maple_tree mt;50 struct range_tree rt;
49 struct list_head vma_list;51 struct list_head vma_list;
50 struct mutex lock;52 struct mutex lock;
51};53};
@@ -98,6 +100,9 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) · 46 unmodified lines
98 u64 vm_range;100 u64 vm_range;
99 int err = -ENOMEM;101 int err = -ENOMEM;
100 102
103 if (!bpf_jit_supports_arena())
104 return ERR_PTR(-EOPNOTSUPP);
105
101 if (attr->key_size || attr->value_size || attr->max_entries == 0 ||106 if (attr->key_size || attr->value_size || attr->max_entries == 0 ||
102 /* BPF_F_MMAPABLE must be set */107 /* BPF_F_MMAPABLE must be set */
103 !(attr->map_flags & BPF_F_MMAPABLE) ||108 !(attr->map_flags & BPF_F_MMAPABLE) ||
@@ -132,7 +137,8 @@ static struct bpf_map *arena_map_alloc(union bpf_attr *attr) · 28 unmodified lines
132 137
133 INIT_LIST_HEAD(&arena->vma_list);138 INIT_LIST_HEAD(&arena->vma_list);
134 bpf_map_init_from_attr(&arena->map, attr);139 bpf_map_init_from_attr(&arena->map, attr);
135 mt_init_flags(&arena->mt, MT_FLAGS_ALLOC_RANGE);140 range_tree_init(&arena->rt);
141 range_tree_set(&arena->rt, 0, attr->max_entries);
136 mutex_init(&arena->lock);142 mutex_init(&arena->lock);
137 143
138 return &arena->map;144 return &arena->map;
@@ -183,7 +189,7 @@ static void arena_map_free(struct bpf_map *map) · 44 unmodified lines
183 apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena),189 apply_to_existing_page_range(&init_mm, bpf_arena_get_kern_vm_start(arena),
184 KERN_VM_SZ - GUARD_SZ, existing_page_cb, NULL);190 KERN_VM_SZ - GUARD_SZ, existing_page_cb, NULL);
185 free_vm_area(arena->kern_vm);191 free_vm_area(arena->kern_vm);
186 mtree_destroy(&arena->mt);192 range_tree_destroy(&arena->rt);
187 bpf_map_area_free(arena);193 bpf_map_area_free(arena);
188}194}
189 195
@@ -274,20 +280,20 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) · 84 unmodified lines
274 /* User space requested to segfault when page is not allocated by bpf prog */280 /* User space requested to segfault when page is not allocated by bpf prog */
275 return VM_FAULT_SIGSEGV;281 return VM_FAULT_SIGSEGV;
276 282
277 ret = mtree_insert(&arena->mt, vmf->pgoff, MT_ENTRY, GFP_KERNEL);283 ret = range_tree_clear(&arena->rt, vmf->pgoff, 1);
278 if (ret)284 if (ret)
279 return VM_FAULT_SIGSEGV;285 return VM_FAULT_SIGSEGV;
280 286
281 /* Account into memcg of the process that created bpf_arena */287 /* Account into memcg of the process that created bpf_arena */
282 ret = bpf_map_alloc_pages(map, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 1, &page);288 ret = bpf_map_alloc_pages(map, GFP_KERNEL | __GFP_ZERO, NUMA_NO_NODE, 1, &page);
283 if (ret) {289 if (ret) {
284 mtree_erase(&arena->mt, vmf->pgoff);290 range_tree_set(&arena->rt, vmf->pgoff, 1);
285 return VM_FAULT_SIGSEGV;291 return VM_FAULT_SIGSEGV;
286 }292 }
287 293
288 ret = vm_area_map_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE, &page);294 ret = vm_area_map_pages(arena->kern_vm, kaddr, kaddr + PAGE_SIZE, &page);
289 if (ret) {295 if (ret) {
290 mtree_erase(&arena->mt, vmf->pgoff);296 range_tree_set(&arena->rt, vmf->pgoff, 1);
291 __free_page(page);297 __free_page(page);
292 return VM_FAULT_SIGSEGV;298 return VM_FAULT_SIGSEGV;
293 }299 }
@@ -444,12 +450,16 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt · 150 unmodified lines
444 450
445 guard(mutex)(&arena->lock);451 guard(mutex)(&arena->lock);
446 452
447 if (uaddr)453 if (uaddr) {
448 ret = mtree_insert_range(&arena->mt, pgoff, pgoff + page_cnt - 1,454 ret = is_range_tree_set(&arena->rt, pgoff, page_cnt);
449 MT_ENTRY, GFP_KERNEL);455 if (ret)
450 else456 goto out_free_pages;
451 ret = mtree_alloc_range(&arena->mt, &pgoff, MT_ENTRY,457 ret = range_tree_clear(&arena->rt, pgoff, page_cnt);
452 page_cnt, 0, page_cnt_max - 1, GFP_KERNEL);458 } else {
459 ret = pgoff = range_tree_find(&arena->rt, page_cnt);
460 if (pgoff >= 0)
461 ret = range_tree_clear(&arena->rt, pgoff, page_cnt);
462 }
453 if (ret)463 if (ret)
454 goto out_free_pages;464 goto out_free_pages;
455 465
@@ -476,7 +486,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt · 20 unmodified lines
476 kvfree(pages);486 kvfree(pages);
477 return clear_lo32(arena->user_vm_start) + uaddr32;487 return clear_lo32(arena->user_vm_start) + uaddr32;
478out:488out:
479 mtree_erase(&arena->mt, pgoff);489 range_tree_set(&arena->rt, pgoff, page_cnt);
480out_free_pages:490out_free_pages:
481 kvfree(pages);491 kvfree(pages);
482 return 0;492 return 0;
@@ -516,7 +526,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt) · 33 unmodified lines
516 526
517 pgoff = compute_pgoff(arena, uaddr);527 pgoff = compute_pgoff(arena, uaddr);
518 /* clear range */528 /* clear range */
519 mtree_store_range(&arena->mt, pgoff, pgoff + page_cnt - 1, NULL, GFP_KERNEL);529 range_tree_set(&arena->rt, pgoff, page_cnt);
520 530
521 if (page_cnt > 1)531 if (page_cnt > 1)
522 /* bulk zap if multiple pages being freed */532 /* bulk zap if multiple pages being freed */
kernel/bpf/arraymap.c+24−2
@@ -947,22 +947,44 @@ static void *prog_fd_array_get_ptr(struct bpf_map *map, · 946 unmodified lines
947 struct file *map_file, int fd)947 struct file *map_file, int fd)
948{948{
949 struct bpf_prog *prog = bpf_prog_get(fd);949 struct bpf_prog *prog = bpf_prog_get(fd);
950 bool is_extended;
950 951
951 if (IS_ERR(prog))952 if (IS_ERR(prog))
952 return prog;953 return prog;
953 954
954 if (!bpf_prog_map_compatible(map, prog)) {955 if (prog->type == BPF_PROG_TYPE_EXT ||
956 !bpf_prog_map_compatible(map, prog)) {
955 bpf_prog_put(prog);957 bpf_prog_put(prog);
956 return ERR_PTR(-EINVAL);958 return ERR_PTR(-EINVAL);
957 }959 }
958 960
961 mutex_lock(&prog->aux->ext_mutex);
962 is_extended = prog->aux->is_extended;
963 if (!is_extended)
964 prog->aux->prog_array_member_cnt++;
965 mutex_unlock(&prog->aux->ext_mutex);
966 if (is_extended) {
967 /* Extended prog can not be tail callee. It's to prevent a
968 * potential infinite loop like:
969 * tail callee prog entry -> tail callee prog subprog ->
970 * freplace prog entry --tailcall-> tail callee prog entry.
971 */
972 bpf_prog_put(prog);
973 return ERR_PTR(-EBUSY);
974 }
975
959 return prog;976 return prog;
960}977}
961 978
962static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)979static void prog_fd_array_put_ptr(struct bpf_map *map, void *ptr, bool need_defer)
963{980{
981 struct bpf_prog *prog = ptr;
982
983 mutex_lock(&prog->aux->ext_mutex);
984 prog->aux->prog_array_member_cnt--;
985 mutex_unlock(&prog->aux->ext_mutex);
964 /* bpf_prog is freed after one RCU or tasks trace grace period */986 /* bpf_prog is freed after one RCU or tasks trace grace period */
965 bpf_prog_put(ptr);987 bpf_prog_put(prog);
966}988}
967 989
968static u32 prog_fd_array_sys_lookup_elem(void *ptr)990static u32 prog_fd_array_sys_lookup_elem(void *ptr)
kernel/bpf/bpf_cgrp_storage.c+2−2
@@ -107,7 +107,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key, · 106 unmodified lines
107 107
108 bpf_cgrp_storage_lock();108 bpf_cgrp_storage_lock();
109 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,109 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
110 value, map_flags, GFP_ATOMIC);110 value, map_flags, false, GFP_ATOMIC);
111 bpf_cgrp_storage_unlock();111 bpf_cgrp_storage_unlock();
112 cgroup_put(cgroup);112 cgroup_put(cgroup);
113 return PTR_ERR_OR_ZERO(sdata);113 return PTR_ERR_OR_ZERO(sdata);
@@ -181,7 +181,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup, · 67 unmodified lines
181 if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&181 if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
182 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))182 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
183 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,183 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
184 value, BPF_NOEXIST, gfp_flags);184 value, BPF_NOEXIST, false, gfp_flags);
185 185
186unlock:186unlock:
187 bpf_cgrp_storage_unlock();187 bpf_cgrp_storage_unlock();
kernel/bpf/bpf_inode_storage.c+2−2
@@ -99,7 +99,7 @@ static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, · 98 unmodified lines
99 99
100 sdata = bpf_local_storage_update(file_inode(fd_file(f)),100 sdata = bpf_local_storage_update(file_inode(fd_file(f)),
101 (struct bpf_local_storage_map *)map,101 (struct bpf_local_storage_map *)map,
102 value, map_flags, GFP_ATOMIC);102 value, map_flags, false, GFP_ATOMIC);
103 return PTR_ERR_OR_ZERO(sdata);103 return PTR_ERR_OR_ZERO(sdata);
104}104}
105 105
@@ -153,7 +153,7 @@ BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, · 47 unmodified lines
153 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {153 if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
154 sdata = bpf_local_storage_update(154 sdata = bpf_local_storage_update(
155 inode, (struct bpf_local_storage_map *)map, value,155 inode, (struct bpf_local_storage_map *)map, value,
156 BPF_NOEXIST, gfp_flags);156 BPF_NOEXIST, false, gfp_flags);
157 return IS_ERR(sdata) ? (unsigned long)NULL :157 return IS_ERR(sdata) ? (unsigned long)NULL :
158 (unsigned long)sdata->data;158 (unsigned long)sdata->data;
159 }159 }
kernel/bpf/bpf_local_storage.c+63−16
@@ -73,7 +73,7 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem) · 72 unmodified lines
73 73
74struct bpf_local_storage_elem *74struct bpf_local_storage_elem *
75bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,75bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,
76 void *value, bool charge_mem, gfp_t gfp_flags)76 void *value, bool charge_mem, bool swap_uptrs, gfp_t gfp_flags)
77{77{
78 struct bpf_local_storage_elem *selem;78 struct bpf_local_storage_elem *selem;
79 79
@@ -99,9 +99,12 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, · 19 unmodified lines
99 }99 }
100 100
101 if (selem) {101 if (selem) {
102 if (value)102 if (value) {
103 /* No need to call check_and_init_map_value as memory is zero init */
103 copy_map_value(&smap->map, SDATA(selem)->data, value);104 copy_map_value(&smap->map, SDATA(selem)->data, value);
104 /* No need to call check_and_init_map_value as memory is zero init */105 if (swap_uptrs)
106 bpf_obj_swap_uptrs(smap->map.record, SDATA(selem)->data, value);
107 }
105 return selem;108 return selem;
106 }109 }
107 110
@@ -209,8 +212,12 @@ static void __bpf_selem_free(struct bpf_local_storage_elem *selem, · 101 unmodified lines
209static void bpf_selem_free_rcu(struct rcu_head *rcu)212static void bpf_selem_free_rcu(struct rcu_head *rcu)
210{213{
211 struct bpf_local_storage_elem *selem;214 struct bpf_local_storage_elem *selem;
215 struct bpf_local_storage_map *smap;
212 216
213 selem = container_of(rcu, struct bpf_local_storage_elem, rcu);217 selem = container_of(rcu, struct bpf_local_storage_elem, rcu);
218 /* The bpf_local_storage_map_free will wait for rcu_barrier */
219 smap = rcu_dereference_check(SDATA(selem)->smap, 1);
220 bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
214 bpf_mem_cache_raw_free(selem);221 bpf_mem_cache_raw_free(selem);
215}222}
216 223
@@ -226,16 +233,25 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, · 9 unmodified lines
226 struct bpf_local_storage_map *smap,233 struct bpf_local_storage_map *smap,
227 bool reuse_now)234 bool reuse_now)
228{235{
229 bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
230
231 if (!smap->bpf_ma) {236 if (!smap->bpf_ma) {
237 /* Only task storage has uptrs and task storage
238 * has moved to bpf_mem_alloc. Meaning smap->bpf_ma == true
239 * for task storage, so this bpf_obj_free_fields() won't unpin
240 * any uptr.
241 */
242 bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
232 __bpf_selem_free(selem, reuse_now);243 __bpf_selem_free(selem, reuse_now);
233 return;244 return;
234 }245 }
235 246
236 if (!reuse_now) {247 if (reuse_now) {
237 call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_trace_rcu);248 /* reuse_now == true only happens when the storage owner
238 } else {249 * (e.g. task_struct) is being destructed or the map itself
250 * is being destructed (ie map_free). In both cases,
251 * no bpf prog can have a hold on the selem. It is
252 * safe to unpin the uptrs and free the selem now.
253 */
254 bpf_obj_free_fields(smap->map.record, SDATA(selem)->data);
239 /* Instead of using the vanilla call_rcu(),255 /* Instead of using the vanilla call_rcu(),
240 * bpf_mem_cache_free will be able to reuse selem256 * bpf_mem_cache_free will be able to reuse selem
241 * immediately.257 * immediately.
@@ -243,6 +259,26 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, · 1 unmodified line
243 migrate_disable();259 migrate_disable();
244 bpf_mem_cache_free(&smap->selem_ma, selem);260 bpf_mem_cache_free(&smap->selem_ma, selem);
245 migrate_enable();261 migrate_enable();
262 return;
263 }
264
265 call_rcu_tasks_trace(&selem->rcu, bpf_selem_free_trace_rcu);
266}
267
268static void bpf_selem_free_list(struct hlist_head *list, bool reuse_now)
269{
270 struct bpf_local_storage_elem *selem;
271 struct bpf_local_storage_map *smap;
272 struct hlist_node *n;
273
274 /* The "_safe" iteration is needed.
275 * The loop is not removing the selem from the list
276 * but bpf_selem_free will use the selem->rcu_head
277 * which is union-ized with the selem->free_node.
278 */
279 hlist_for_each_entry_safe(selem, n, list, free_node) {
280 smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held());
281 bpf_selem_free(selem, smap, reuse_now);
246 }282 }
247}283}
248 284
@@ -252,7 +288,7 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, · 3 unmodified lines
252 */288 */
253static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,289static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
254 struct bpf_local_storage_elem *selem,290 struct bpf_local_storage_elem *selem,
255 bool uncharge_mem, bool reuse_now)291 bool uncharge_mem, struct hlist_head *free_selem_list)
256{292{
257 struct bpf_local_storage_map *smap;293 struct bpf_local_storage_map *smap;
258 bool free_local_storage;294 bool free_local_storage;
@@ -296,7 +332,7 @@ static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_stor · 37 unmodified lines
296 SDATA(selem))332 SDATA(selem))
297 RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);333 RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
298 334
299 bpf_selem_free(selem, smap, reuse_now);335 hlist_add_head(&selem->free_node, free_selem_list);
300 336
301 if (rcu_access_pointer(local_storage->smap) == smap)337 if (rcu_access_pointer(local_storage->smap) == smap)
302 RCU_INIT_POINTER(local_storage->smap, NULL);338 RCU_INIT_POINTER(local_storage->smap, NULL);
@@ -345,6 +381,7 @@ static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem, · 42 unmodified lines
345 struct bpf_local_storage_map *storage_smap;381 struct bpf_local_storage_map *storage_smap;
346 struct bpf_local_storage *local_storage;382 struct bpf_local_storage *local_storage;
347 bool bpf_ma, free_local_storage = false;383 bool bpf_ma, free_local_storage = false;
384 HLIST_HEAD(selem_free_list);
348 unsigned long flags;385 unsigned long flags;
349 386
350 if (unlikely(!selem_linked_to_storage_lockless(selem)))387 if (unlikely(!selem_linked_to_storage_lockless(selem)))
@@ -360,9 +397,11 @@ static void bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem, · 9 unmodified lines
360 raw_spin_lock_irqsave(&local_storage->lock, flags);397 raw_spin_lock_irqsave(&local_storage->lock, flags);
361 if (likely(selem_linked_to_storage(selem)))398 if (likely(selem_linked_to_storage(selem)))
362 free_local_storage = bpf_selem_unlink_storage_nolock(399 free_local_storage = bpf_selem_unlink_storage_nolock(
363 local_storage, selem, true, reuse_now);400 local_storage, selem, true, &selem_free_list);
364 raw_spin_unlock_irqrestore(&local_storage->lock, flags);401 raw_spin_unlock_irqrestore(&local_storage->lock, flags);
365 402
403 bpf_selem_free_list(&selem_free_list, reuse_now);
404
366 if (free_local_storage)405 if (free_local_storage)
367 bpf_local_storage_free(local_storage, storage_smap, bpf_ma, reuse_now);406 bpf_local_storage_free(local_storage, storage_smap, bpf_ma, reuse_now);
368}407}
@@ -524,11 +563,12 @@ uncharge: · 155 unmodified lines
524 */563 */
525struct bpf_local_storage_data *564struct bpf_local_storage_data *
526bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,565bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
527 void *value, u64 map_flags, gfp_t gfp_flags)566 void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags)
528{567{
529 struct bpf_local_storage_data *old_sdata = NULL;568 struct bpf_local_storage_data *old_sdata = NULL;
530 struct bpf_local_storage_elem *alloc_selem, *selem = NULL;569 struct bpf_local_storage_elem *alloc_selem, *selem = NULL;
531 struct bpf_local_storage *local_storage;570 struct bpf_local_storage *local_storage;
571 HLIST_HEAD(old_selem_free_list);
532 unsigned long flags;572 unsigned long flags;
533 int err;573 int err;
534 574
@@ -550,7 +590,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 15 unmodified lines
550 if (err)590 if (err)
551 return ERR_PTR(err);591 return ERR_PTR(err);
552 592
553 selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);593 selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);
554 if (!selem)594 if (!selem)
555 return ERR_PTR(-ENOMEM);595 return ERR_PTR(-ENOMEM);
556 596
@@ -584,7 +624,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 27 unmodified lines
584 /* A lookup has just been done before and concluded a new selem is624 /* A lookup has just been done before and concluded a new selem is
585 * needed. The chance of an unnecessary alloc is unlikely.625 * needed. The chance of an unnecessary alloc is unlikely.
586 */626 */
587 alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags);627 alloc_selem = selem = bpf_selem_alloc(smap, owner, value, true, swap_uptrs, gfp_flags);
588 if (!alloc_selem)628 if (!alloc_selem)
589 return ERR_PTR(-ENOMEM);629 return ERR_PTR(-ENOMEM);
590 630
@@ -624,11 +664,12 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, · 33 unmodified lines
624 if (old_sdata) {664 if (old_sdata) {
625 bpf_selem_unlink_map(SELEM(old_sdata));665 bpf_selem_unlink_map(SELEM(old_sdata));
626 bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),666 bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
627 true, false);667 true, &old_selem_free_list);
628 }668 }
629 669
630unlock:670unlock:
631 raw_spin_unlock_irqrestore(&local_storage->lock, flags);671 raw_spin_unlock_irqrestore(&local_storage->lock, flags);
672 bpf_selem_free_list(&old_selem_free_list, false);
632 if (alloc_selem) {673 if (alloc_selem) {
633 mem_uncharge(smap, owner, smap->elem_size);674 mem_uncharge(smap, owner, smap->elem_size);
634 bpf_selem_free(alloc_selem, smap, true);675 bpf_selem_free(alloc_selem, smap, true);
@@ -706,6 +747,7 @@ void bpf_local_storage_destroy(struct bpf_local_storage *local_storage) · 71 unmodified lines
706 struct bpf_local_storage_map *storage_smap;747 struct bpf_local_storage_map *storage_smap;
707 struct bpf_local_storage_elem *selem;748 struct bpf_local_storage_elem *selem;
708 bool bpf_ma, free_storage = false;749 bool bpf_ma, free_storage = false;
750 HLIST_HEAD(free_selem_list);
709 struct hlist_node *n;751 struct hlist_node *n;
710 unsigned long flags;752 unsigned long flags;
711 753
@@ -734,10 +776,12 @@ void bpf_local_storage_destroy(struct bpf_local_storage *local_storage) · 22 unmodified lines
734 * of the loop will set the free_cgroup_storage to true.776 * of the loop will set the free_cgroup_storage to true.
735 */777 */
736 free_storage = bpf_selem_unlink_storage_nolock(778 free_storage = bpf_selem_unlink_storage_nolock(
737 local_storage, selem, true, true);779 local_storage, selem, true, &free_selem_list);
738 }780 }
739 raw_spin_unlock_irqrestore(&local_storage->lock, flags);781 raw_spin_unlock_irqrestore(&local_storage->lock, flags);
740 782
783 bpf_selem_free_list(&free_selem_list, true);
784
741 if (free_storage)785 if (free_storage)
742 bpf_local_storage_free(local_storage, storage_smap, bpf_ma, true);786 bpf_local_storage_free(local_storage, storage_smap, bpf_ma, true);
743}787}
@@ -883,6 +927,9 @@ void bpf_local_storage_map_free(struct bpf_map *map, · 139 unmodified lines
883 synchronize_rcu();927 synchronize_rcu();
884 928
885 if (smap->bpf_ma) {929 if (smap->bpf_ma) {
930 rcu_barrier_tasks_trace();
931 if (!rcu_trace_implies_rcu_gp())
932 rcu_barrier();
886 bpf_mem_alloc_destroy(&smap->selem_ma);933 bpf_mem_alloc_destroy(&smap->selem_ma);
887 bpf_mem_alloc_destroy(&smap->storage_ma);934 bpf_mem_alloc_destroy(&smap->storage_ma);
888 }935 }
kernel/bpf/bpf_struct_ops.c+103−12
@@ -23,7 +23,6 @@ struct bpf_struct_ops_value { · 22 unmodified lines
23 23
24struct bpf_struct_ops_map {24struct bpf_struct_ops_map {
25 struct bpf_map map;25 struct bpf_map map;
26 struct rcu_head rcu;
27 const struct bpf_struct_ops_desc *st_ops_desc;26 const struct bpf_struct_ops_desc *st_ops_desc;
28 /* protect map_update */27 /* protect map_update */
29 struct mutex lock;28 struct mutex lock;
@@ -32,7 +31,9 @@ struct bpf_struct_ops_map { · 2 unmodified lines
32 * (in kvalue.data).31 * (in kvalue.data).
33 */32 */
34 struct bpf_link **links;33 struct bpf_link **links;
35 u32 links_cnt;34 /* ksyms for bpf trampolines */
35 struct bpf_ksym **ksyms;
36 u32 funcs_cnt;
36 u32 image_pages_cnt;37 u32 image_pages_cnt;
37 /* image_pages is an array of pages that has all the trampolines38 /* image_pages is an array of pages that has all the trampolines
38 * that stores the func args before calling the bpf_prog.39 * that stores the func args before calling the bpf_prog.
@@ -481,11 +482,11 @@ static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map) · 442 unmodified lines
481{482{
482 u32 i;483 u32 i;
483 484
484 for (i = 0; i < st_map->links_cnt; i++) {485 for (i = 0; i < st_map->funcs_cnt; i++) {
485 if (st_map->links[i]) {486 if (!st_map->links[i])
486 bpf_link_put(st_map->links[i]);487 break;
487 st_map->links[i] = NULL;488 bpf_link_put(st_map->links[i]);
488 }489 st_map->links[i] = NULL;
489 }490 }
490}491}
491 492
@@ -586,6 +587,49 @@ int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, · 94 unmodified lines
586 return 0;587 return 0;
587}588}
588 589
590static void bpf_struct_ops_ksym_init(const char *tname, const char *mname,
591 void *image, unsigned int size,
592 struct bpf_ksym *ksym)
593{
594 snprintf(ksym->name, KSYM_NAME_LEN, "bpf__%s_%s", tname, mname);
595 INIT_LIST_HEAD_RCU(&ksym->lnode);
596 bpf_image_ksym_init(image, size, ksym);
597}
598
599static void bpf_struct_ops_map_add_ksyms(struct bpf_struct_ops_map *st_map)
600{
601 u32 i;
602
603 for (i = 0; i < st_map->funcs_cnt; i++) {
604 if (!st_map->ksyms[i])
605 break;
606 bpf_image_ksym_add(st_map->ksyms[i]);
607 }
608}
609
610static void bpf_struct_ops_map_del_ksyms(struct bpf_struct_ops_map *st_map)
611{
612 u32 i;
613
614 for (i = 0; i < st_map->funcs_cnt; i++) {
615 if (!st_map->ksyms[i])
616 break;
617 bpf_image_ksym_del(st_map->ksyms[i]);
618 }
619}
620
621static void bpf_struct_ops_map_free_ksyms(struct bpf_struct_ops_map *st_map)
622{
623 u32 i;
624
625 for (i = 0; i < st_map->funcs_cnt; i++) {
626 if (!st_map->ksyms[i])
627 break;
628 kfree(st_map->ksyms[i]);
629 st_map->ksyms[i] = NULL;
630 }
631}
632
589static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,633static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key,
590 void *value, u64 flags)634 void *value, u64 flags)
591{635{
@@ -601,6 +645,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 9 unmodified lines
601 int prog_fd, err;645 int prog_fd, err;
602 u32 i, trampoline_start, image_off = 0;646 u32 i, trampoline_start, image_off = 0;
603 void *cur_image = NULL, *image = NULL;647 void *cur_image = NULL, *image = NULL;
648 struct bpf_link **plink;
649 struct bpf_ksym **pksym;
650 const char *tname, *mname;
604 651
605 if (flags)652 if (flags)
606 return -EINVAL;653 return -EINVAL;
@@ -639,14 +686,19 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 32 unmodified lines
639 udata = &uvalue->data;686 udata = &uvalue->data;
640 kdata = &kvalue->data;687 kdata = &kvalue->data;
641 688
689 plink = st_map->links;
690 pksym = st_map->ksyms;
691 tname = btf_name_by_offset(st_map->btf, t->name_off);
642 module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);692 module_type = btf_type_by_id(btf_vmlinux, st_ops_ids[IDX_MODULE_ID]);
643 for_each_member(i, t, member) {693 for_each_member(i, t, member) {
644 const struct btf_type *mtype, *ptype;694 const struct btf_type *mtype, *ptype;
645 struct bpf_prog *prog;695 struct bpf_prog *prog;
646 struct bpf_tramp_link *link;696 struct bpf_tramp_link *link;
697 struct bpf_ksym *ksym;
647 u32 moff;698 u32 moff;
648 699
649 moff = __btf_member_bit_offset(t, member) / 8;700 moff = __btf_member_bit_offset(t, member) / 8;
701 mname = btf_name_by_offset(st_map->btf, member->name_off);
650 ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);702 ptype = btf_type_resolve_ptr(st_map->btf, member->type, NULL);
651 if (ptype == module_type) {703 if (ptype == module_type) {
652 if (*(void **)(udata + moff))704 if (*(void **)(udata + moff))
@@ -714,7 +766,14 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 61 unmodified lines
714 }766 }
715 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,767 bpf_link_init(&link->link, BPF_LINK_TYPE_STRUCT_OPS,
716 &bpf_struct_ops_link_lops, prog);768 &bpf_struct_ops_link_lops, prog);
717 st_map->links[i] = &link->link;769 *plink++ = &link->link;
770
771 ksym = kzalloc(sizeof(*ksym), GFP_USER);
772 if (!ksym) {
773 err = -ENOMEM;
774 goto reset_unlock;
775 }
776 *pksym++ = ksym;
718 777
719 trampoline_start = image_off;778 trampoline_start = image_off;
720 err = bpf_struct_ops_prepare_trampoline(tlinks, link,779 err = bpf_struct_ops_prepare_trampoline(tlinks, link,
@@ -735,6 +794,12 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 14 unmodified lines
735 794
736 /* put prog_id to udata */795 /* put prog_id to udata */
737 *(unsigned long *)(udata + moff) = prog->aux->id;796 *(unsigned long *)(udata + moff) = prog->aux->id;
797
798 /* init ksym for this trampoline */
799 bpf_struct_ops_ksym_init(tname, mname,
800 image + trampoline_start,
801 image_off - trampoline_start,
802 ksym);
738 }803 }
739 804
740 if (st_ops->validate) {805 if (st_ops->validate) {
@@ -783,6 +848,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 42 unmodified lines
783 */848 */
784 849
785reset_unlock:850reset_unlock:
851 bpf_struct_ops_map_free_ksyms(st_map);
786 bpf_struct_ops_map_free_image(st_map);852 bpf_struct_ops_map_free_image(st_map);
787 bpf_struct_ops_map_put_progs(st_map);853 bpf_struct_ops_map_put_progs(st_map);
788 memset(uvalue, 0, map->value_size);854 memset(uvalue, 0, map->value_size);
@@ -790,6 +856,8 @@ reset_unlock: · 1 unmodified line
790unlock:856unlock:
791 kfree(tlinks);857 kfree(tlinks);
792 mutex_unlock(&st_map->lock);858 mutex_unlock(&st_map->lock);
859 if (!err)
860 bpf_struct_ops_map_add_ksyms(st_map);
793 return err;861 return err;
794}862}
795 863
@@ -849,7 +917,10 @@ static void __bpf_struct_ops_map_free(struct bpf_map *map) · 53 unmodified lines
849 917
850 if (st_map->links)918 if (st_map->links)
851 bpf_struct_ops_map_put_progs(st_map);919 bpf_struct_ops_map_put_progs(st_map);
920 if (st_map->ksyms)
921 bpf_struct_ops_map_free_ksyms(st_map);
852 bpf_map_area_free(st_map->links);922 bpf_map_area_free(st_map->links);
923 bpf_map_area_free(st_map->ksyms);
853 bpf_struct_ops_map_free_image(st_map);924 bpf_struct_ops_map_free_image(st_map);
854 bpf_map_area_free(st_map->uvalue);925 bpf_map_area_free(st_map->uvalue);
855 bpf_map_area_free(st_map);926 bpf_map_area_free(st_map);
@@ -866,6 +937,8 @@ static void bpf_struct_ops_map_free(struct bpf_map *map) · 10 unmodified lines
866 if (btf_is_module(st_map->btf))937 if (btf_is_module(st_map->btf))
867 module_put(st_map->st_ops_desc->st_ops->owner);938 module_put(st_map->st_ops_desc->st_ops->owner);
868 939
940 bpf_struct_ops_map_del_ksyms(st_map);
941
869 /* The struct_ops's function may switch to another struct_ops.942 /* The struct_ops's function may switch to another struct_ops.
870 *943 *
871 * For example, bpf_tcp_cc_x->init() may switch to944 * For example, bpf_tcp_cc_x->init() may switch to
@@ -895,6 +968,19 @@ static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr) · 23 unmodified lines
895 return 0;968 return 0;
896}969}
897 970
971static u32 count_func_ptrs(const struct btf *btf, const struct btf_type *t)
972{
973 int i;
974 u32 count;
975 const struct btf_member *member;
976
977 count = 0;
978 for_each_member(i, t, member)
979 if (btf_type_resolve_func_ptr(btf, member->type, NULL))
980 count++;
981 return count;
982}
983
898static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)984static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr)
899{985{
900 const struct bpf_struct_ops_desc *st_ops_desc;986 const struct bpf_struct_ops_desc *st_ops_desc;
@@ -961,11 +1047,15 @@ static struct bpf_map *bpf_struct_ops_map_alloc(union bpf_attr *attr) · 60 unmodified lines
961 map = &st_map->map;1047 map = &st_map->map;
962 1048
963 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);1049 st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
964 st_map->links_cnt = btf_type_vlen(t);1050 st_map->funcs_cnt = count_func_ptrs(btf, t);
965 st_map->links =1051 st_map->links =
966 bpf_map_area_alloc(st_map->links_cnt * sizeof(struct bpf_links *),1052 bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_link *),
1053 NUMA_NO_NODE);
1054
1055 st_map->ksyms =
1056 bpf_map_area_alloc(st_map->funcs_cnt * sizeof(struct bpf_ksym *),
967 NUMA_NO_NODE);1057 NUMA_NO_NODE);
968 if (!st_map->uvalue || !st_map->links) {1058 if (!st_map->uvalue || !st_map->links || !st_map->ksyms) {
969 ret = -ENOMEM;1059 ret = -ENOMEM;
970 goto errout_free;1060 goto errout_free;
971 }1061 }
@@ -994,7 +1084,8 @@ static u64 bpf_struct_ops_map_mem_usage(const struct bpf_map *map) · 22 unmodified lines
994 usage = sizeof(*st_map) +1084 usage = sizeof(*st_map) +
995 vt->size - sizeof(struct bpf_struct_ops_value);1085 vt->size - sizeof(struct bpf_struct_ops_value);
996 usage += vt->size;1086 usage += vt->size;
997 usage += btf_type_vlen(vt) * sizeof(struct bpf_links *);1087 usage += st_map->funcs_cnt * sizeof(struct bpf_link *);
1088 usage += st_map->funcs_cnt * sizeof(struct bpf_ksym *);
998 usage += PAGE_SIZE;1089 usage += PAGE_SIZE;
999 return usage;1090 return usage;
1000}1091}
kernel/bpf/bpf_task_storage.c+5−2
@@ -128,6 +128,9 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, · 127 unmodified lines
128 struct pid *pid;128 struct pid *pid;
129 int fd, err;129 int fd, err;
130 130
131 if ((map_flags & BPF_F_LOCK) && btf_record_has_field(map->record, BPF_UPTR))
132 return -EOPNOTSUPP;
133
131 fd = *(int *)key;134 fd = *(int *)key;
132 pid = pidfd_get_pid(fd, &f_flags);135 pid = pidfd_get_pid(fd, &f_flags);
133 if (IS_ERR(pid))136 if (IS_ERR(pid))
@@ -146,7 +149,7 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, · 12 unmodified lines
146 bpf_task_storage_lock();149 bpf_task_storage_lock();
147 sdata = bpf_local_storage_update(150 sdata = bpf_local_storage_update(
148 task, (struct bpf_local_storage_map *)map, value, map_flags,151 task, (struct bpf_local_storage_map *)map, value, map_flags,
149 GFP_ATOMIC);152 true, GFP_ATOMIC);
150 bpf_task_storage_unlock();153 bpf_task_storage_unlock();
151 154
152 err = PTR_ERR_OR_ZERO(sdata);155 err = PTR_ERR_OR_ZERO(sdata);
@@ -218,7 +221,7 @@ static void *__bpf_task_storage_get(struct bpf_map *map, · 65 unmodified lines
218 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) {221 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) {
219 sdata = bpf_local_storage_update(222 sdata = bpf_local_storage_update(
220 task, (struct bpf_local_storage_map *)map, value,223 task, (struct bpf_local_storage_map *)map, value,
221 BPF_NOEXIST, gfp_flags);224 BPF_NOEXIST, false, gfp_flags);
222 return IS_ERR(sdata) ? NULL : sdata->data;225 return IS_ERR(sdata) ? NULL : sdata->data;
223 }226 }
224 227
kernel/bpf/btf.c+42−15
@@ -2808,7 +2808,7 @@ static void btf_ref_type_log(struct btf_verifier_env *env, · 2807 unmodified lines
2808 btf_verifier_log(env, "type_id=%u", t->type);2808 btf_verifier_log(env, "type_id=%u", t->type);
2809}2809}
2810 2810
2811static struct btf_kind_operations modifier_ops = {2811static const struct btf_kind_operations modifier_ops = {
2812 .check_meta = btf_ref_type_check_meta,2812 .check_meta = btf_ref_type_check_meta,
2813 .resolve = btf_modifier_resolve,2813 .resolve = btf_modifier_resolve,
2814 .check_member = btf_modifier_check_member,2814 .check_member = btf_modifier_check_member,
@@ -2817,7 +2817,7 @@ static struct btf_kind_operations modifier_ops = { · 2 unmodified lines
2817 .show = btf_modifier_show,2817 .show = btf_modifier_show,
2818};2818};
2819 2819
2820static struct btf_kind_operations ptr_ops = {2820static const struct btf_kind_operations ptr_ops = {
2821 .check_meta = btf_ref_type_check_meta,2821 .check_meta = btf_ref_type_check_meta,
2822 .resolve = btf_ptr_resolve,2822 .resolve = btf_ptr_resolve,
2823 .check_member = btf_ptr_check_member,2823 .check_member = btf_ptr_check_member,
@@ -2858,7 +2858,7 @@ static void btf_fwd_type_log(struct btf_verifier_env *env, · 34 unmodified lines
2858 btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");2858 btf_verifier_log(env, "%s", btf_type_kflag(t) ? "union" : "struct");
2859}2859}
2860 2860
2861static struct btf_kind_operations fwd_ops = {2861static const struct btf_kind_operations fwd_ops = {
2862 .check_meta = btf_fwd_check_meta,2862 .check_meta = btf_fwd_check_meta,
2863 .resolve = btf_df_resolve,2863 .resolve = btf_df_resolve,
2864 .check_member = btf_df_check_member,2864 .check_member = btf_df_check_member,
@@ -3109,7 +3109,7 @@ static void btf_array_show(const struct btf *btf, const struct btf_type *t, · 244 unmodified lines
3109 __btf_array_show(btf, t, type_id, data, bits_offset, show);3109 __btf_array_show(btf, t, type_id, data, bits_offset, show);
3110}3110}
3111 3111
3112static struct btf_kind_operations array_ops = {3112static const struct btf_kind_operations array_ops = {
3113 .check_meta = btf_array_check_meta,3113 .check_meta = btf_array_check_meta,
3114 .resolve = btf_array_resolve,3114 .resolve = btf_array_resolve,
3115 .check_member = btf_array_check_member,3115 .check_member = btf_array_check_member,
@@ -3334,7 +3334,7 @@ static int btf_find_struct(const struct btf *btf, const struct btf_type *t, · 218 unmodified lines
3334}3334}
3335 3335
3336static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,3336static int btf_find_kptr(const struct btf *btf, const struct btf_type *t,
3337 u32 off, int sz, struct btf_field_info *info)3337 u32 off, int sz, struct btf_field_info *info, u32 field_mask)
3338{3338{
3339 enum btf_field_type type;3339 enum btf_field_type type;
3340 u32 res_id;3340 u32 res_id;
@@ -3358,9 +3358,14 @@ static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, · 17 unmodified lines
3358 type = BPF_KPTR_REF;3358 type = BPF_KPTR_REF;
3359 else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))3359 else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))
3360 type = BPF_KPTR_PERCPU;3360 type = BPF_KPTR_PERCPU;
3361 else if (!strcmp("uptr", __btf_name_by_offset(btf, t->name_off)))
3362 type = BPF_UPTR;
3361 else3363 else
3362 return -EINVAL;3364 return -EINVAL;
3363 3365
3366 if (!(type & field_mask))
3367 return BTF_FIELD_IGNORE;
3368
3364 /* Get the base type */3369 /* Get the base type */
3365 t = btf_type_skip_modifiers(btf, t->type, &res_id);3370 t = btf_type_skip_modifiers(btf, t->type, &res_id);
3366 /* Only pointer to struct is allowed */3371 /* Only pointer to struct is allowed */
@@ -3502,7 +3507,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ · 135 unmodified lines
3502 field_mask_test_name(BPF_REFCOUNT, "bpf_refcount");3507 field_mask_test_name(BPF_REFCOUNT, "bpf_refcount");
3503 3508
3504 /* Only return BPF_KPTR when all other types with matchable names fail */3509 /* Only return BPF_KPTR when all other types with matchable names fail */
3505 if (field_mask & BPF_KPTR && !__btf_type_is_struct(var_type)) {3510 if (field_mask & (BPF_KPTR | BPF_UPTR) && !__btf_type_is_struct(var_type)) {
3506 type = BPF_KPTR_REF;3511 type = BPF_KPTR_REF;
3507 goto end;3512 goto end;
3508 }3513 }
@@ -3535,6 +3540,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, · 26 unmodified lines
3535 case BPF_KPTR_UNREF:3540 case BPF_KPTR_UNREF:
3536 case BPF_KPTR_REF:3541 case BPF_KPTR_REF:
3537 case BPF_KPTR_PERCPU:3542 case BPF_KPTR_PERCPU:
3543 case BPF_UPTR:
3538 case BPF_LIST_HEAD:3544 case BPF_LIST_HEAD:
3539 case BPF_RB_ROOT:3545 case BPF_RB_ROOT:
3540 break;3546 break;
@@ -3667,8 +3673,9 @@ static int btf_find_field_one(const struct btf *btf, · 126 unmodified lines
3667 case BPF_KPTR_UNREF:3673 case BPF_KPTR_UNREF:
3668 case BPF_KPTR_REF:3674 case BPF_KPTR_REF:
3669 case BPF_KPTR_PERCPU:3675 case BPF_KPTR_PERCPU:
3676 case BPF_UPTR:
3670 ret = btf_find_kptr(btf, var_type, off, sz,3677 ret = btf_find_kptr(btf, var_type, off, sz,
3671 info_cnt ? &info[0] : &tmp);3678 info_cnt ? &info[0] : &tmp, field_mask);
3672 if (ret < 0)3679 if (ret < 0)
3673 return ret;3680 return ret;
3674 break;3681 break;
@@ -3991,6 +3998,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type · 316 unmodified lines
3991 case BPF_KPTR_UNREF:3998 case BPF_KPTR_UNREF:
3992 case BPF_KPTR_REF:3999 case BPF_KPTR_REF:
3993 case BPF_KPTR_PERCPU:4000 case BPF_KPTR_PERCPU:
4001 case BPF_UPTR:
3994 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]);4002 ret = btf_parse_kptr(btf, &rec->fields[i], &info_arr[i]);
3995 if (ret < 0)4003 if (ret < 0)
3996 goto end;4004 goto end;
@@ -4050,12 +4058,28 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec) · 53 unmodified lines
4050 * Hence we only need to ensure that bpf_{list_head,rb_root} ownership4058 * Hence we only need to ensure that bpf_{list_head,rb_root} ownership
4051 * does not form cycles.4059 * does not form cycles.
4052 */4060 */
4053 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & BPF_GRAPH_ROOT))4061 if (IS_ERR_OR_NULL(rec) || !(rec->field_mask & (BPF_GRAPH_ROOT | BPF_UPTR)))
4054 return 0;4062 return 0;
4055 for (i = 0; i < rec->cnt; i++) {4063 for (i = 0; i < rec->cnt; i++) {
4056 struct btf_struct_meta *meta;4064 struct btf_struct_meta *meta;
4065 const struct btf_type *t;
4057 u32 btf_id;4066 u32 btf_id;
4058 4067
4068 if (rec->fields[i].type == BPF_UPTR) {
4069 /* The uptr only supports pinning one page and cannot
4070 * point to a kernel struct
4071 */
4072 if (btf_is_kernel(rec->fields[i].kptr.btf))
4073 return -EINVAL;
4074 t = btf_type_by_id(rec->fields[i].kptr.btf,
4075 rec->fields[i].kptr.btf_id);
4076 if (!t->size)
4077 return -EINVAL;
4078 if (t->size > PAGE_SIZE)
4079 return -E2BIG;
4080 continue;
4081 }
4082
4059 if (!(rec->fields[i].type & BPF_GRAPH_ROOT))4083 if (!(rec->fields[i].type & BPF_GRAPH_ROOT))
4060 continue;4084 continue;
4061 btf_id = rec->fields[i].graph_root.value_btf_id;4085 btf_id = rec->fields[i].graph_root.value_btf_id;
@@ -4191,7 +4215,7 @@ static void btf_struct_show(const struct btf *btf, const struct btf_type *t, · 129 unmodified lines
4191 __btf_struct_show(btf, t, type_id, data, bits_offset, show);4215 __btf_struct_show(btf, t, type_id, data, bits_offset, show);
4192}4216}
4193 4217
4194static struct btf_kind_operations struct_ops = {4218static const struct btf_kind_operations struct_ops = {
4195 .check_meta = btf_struct_check_meta,4219 .check_meta = btf_struct_check_meta,
4196 .resolve = btf_struct_resolve,4220 .resolve = btf_struct_resolve,
4197 .check_member = btf_struct_check_member,4221 .check_member = btf_struct_check_member,
@@ -4359,7 +4383,7 @@ static void btf_enum_show(const struct btf *btf, const struct btf_type *t, · 161 unmodified lines
4359 btf_show_end_type(show);4383 btf_show_end_type(show);
4360}4384}
4361 4385
4362static struct btf_kind_operations enum_ops = {4386static const struct btf_kind_operations enum_ops = {
4363 .check_meta = btf_enum_check_meta,4387 .check_meta = btf_enum_check_meta,
4364 .resolve = btf_df_resolve,4388 .resolve = btf_df_resolve,
4365 .check_member = btf_enum_check_member,4389 .check_member = btf_enum_check_member,
@@ -4462,7 +4486,7 @@ static void btf_enum64_show(const struct btf *btf, const struct btf_type *t, · 96 unmodified lines
4462 btf_show_end_type(show);4486 btf_show_end_type(show);
4463}4487}
4464 4488
4465static struct btf_kind_operations enum64_ops = {4489static const struct btf_kind_operations enum64_ops = {
4466 .check_meta = btf_enum64_check_meta,4490 .check_meta = btf_enum64_check_meta,
4467 .resolve = btf_df_resolve,4491 .resolve = btf_df_resolve,
4468 .check_member = btf_enum_check_member,4492 .check_member = btf_enum_check_member,
@@ -4540,7 +4564,7 @@ done: · 71 unmodified lines
4540 btf_verifier_log(env, ")");4564 btf_verifier_log(env, ")");
4541}4565}
4542 4566
4543static struct btf_kind_operations func_proto_ops = {4567static const struct btf_kind_operations func_proto_ops = {
4544 .check_meta = btf_func_proto_check_meta,4568 .check_meta = btf_func_proto_check_meta,
4545 .resolve = btf_df_resolve,4569 .resolve = btf_df_resolve,
4546 /*4570 /*
@@ -4598,7 +4622,7 @@ static int btf_func_resolve(struct btf_verifier_env *env, · 51 unmodified lines
4598 return 0;4622 return 0;
4599}4623}
4600 4624
4601static struct btf_kind_operations func_ops = {4625static const struct btf_kind_operations func_ops = {
4602 .check_meta = btf_func_check_meta,4626 .check_meta = btf_func_check_meta,
4603 .resolve = btf_func_resolve,4627 .resolve = btf_func_resolve,
4604 .check_member = btf_df_check_member,4628 .check_member = btf_df_check_member,
@@ -5566,7 +5590,7 @@ btf_parse_struct_metas(struct bpf_verifier_log *log, struct btf *btf) · 961 unmodified lines
5566 goto free_aof;5590 goto free_aof;
5567 }5591 }
5568 5592
5569 ret = btf_find_kptr(btf, t, 0, 0, &tmp);5593 ret = btf_find_kptr(btf, t, 0, 0, &tmp, BPF_KPTR);
5570 if (ret != BTF_FIELD_FOUND)5594 if (ret != BTF_FIELD_FOUND)
5571 continue;5595 continue;
5572 5596
@@ -6564,7 +6588,10 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 991 unmodified lines
6564 if (prog_args_trusted(prog))6588 if (prog_args_trusted(prog))
6565 info->reg_type |= PTR_TRUSTED;6589 info->reg_type |= PTR_TRUSTED;
6566 6590
6567 if (btf_param_match_suffix(btf, &args[arg], "__nullable"))6591 /* Raw tracepoint arguments always get marked as maybe NULL */
6592 if (bpf_prog_is_raw_tp(prog))
6593 info->reg_type |= PTR_MAYBE_NULL;
6594 else if (btf_param_match_suffix(btf, &args[arg], "__nullable"))
6568 info->reg_type |= PTR_MAYBE_NULL;6595 info->reg_type |= PTR_MAYBE_NULL;
6569 6596
6570 if (tgt_prog) {6597 if (tgt_prog) {
kernel/bpf/core.c+6−0
@@ -131,6 +131,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag · 130 unmodified lines
131 INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode);131 INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode);
132#endif132#endif
133 mutex_init(&fp->aux->used_maps_mutex);133 mutex_init(&fp->aux->used_maps_mutex);
134 mutex_init(&fp->aux->ext_mutex);
134 mutex_init(&fp->aux->dst_mutex);135 mutex_init(&fp->aux->dst_mutex);
135 136
136 return fp;137 return fp;
@@ -3044,6 +3045,11 @@ bool __weak bpf_jit_supports_exceptions(void) · 2907 unmodified lines
3044 return false;3045 return false;
3045}3046}
3046 3047
3048bool __weak bpf_jit_supports_private_stack(void)
3049{
3050 return false;
3051}
3052
3047void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)3053void __weak arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie)
3048{3054{
3049}3055}
kernel/bpf/dispatcher.c+2−1
@@ -154,7 +154,8 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, · 153 unmodified lines
154 d->image = NULL;154 d->image = NULL;
155 goto out;155 goto out;
156 }156 }
157 bpf_image_ksym_add(d->image, PAGE_SIZE, &d->ksym);157 bpf_image_ksym_init(d->image, PAGE_SIZE, &d->ksym);
158 bpf_image_ksym_add(&d->ksym);
158 }159 }
159 160
160 prev_num_progs = d->num_progs;161 prev_num_progs = d->num_progs;
kernel/bpf/hashtab.c+39−17
@@ -896,9 +896,12 @@ find_first_elem: · 895 unmodified lines
896static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)896static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
897{897{
898 check_and_free_fields(htab, l);898 check_and_free_fields(htab, l);
899
900 migrate_disable();
899 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)901 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
900 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);902 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
901 bpf_mem_cache_free(&htab->ma, l);903 bpf_mem_cache_free(&htab->ma, l);
904 migrate_enable();
902}905}
903 906
904static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)907static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
@@ -948,7 +951,7 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) · 43 unmodified lines
948 if (htab_is_prealloc(htab)) {951 if (htab_is_prealloc(htab)) {
949 bpf_map_dec_elem_count(&htab->map);952 bpf_map_dec_elem_count(&htab->map);
950 check_and_free_fields(htab, l);953 check_and_free_fields(htab, l);
951 __pcpu_freelist_push(&htab->freelist, &l->fnode);954 pcpu_freelist_push(&htab->freelist, &l->fnode);
952 } else {955 } else {
953 dec_elem_count(htab);956 dec_elem_count(htab);
954 htab_elem_free(htab, l);957 htab_elem_free(htab, l);
@@ -1018,7 +1021,6 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, · 63 unmodified lines
1018 */1021 */
1019 pl_new = this_cpu_ptr(htab->extra_elems);1022 pl_new = this_cpu_ptr(htab->extra_elems);
1020 l_new = *pl_new;1023 l_new = *pl_new;
1021 htab_put_fd_value(htab, old_elem);
1022 *pl_new = old_elem;1024 *pl_new = old_elem;
1023 } else {1025 } else {
1024 struct pcpu_freelist_node *l;1026 struct pcpu_freelist_node *l;
@@ -1105,6 +1107,7 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 80 unmodified lines
1105 struct htab_elem *l_new = NULL, *l_old;1107 struct htab_elem *l_new = NULL, *l_old;
1106 struct hlist_nulls_head *head;1108 struct hlist_nulls_head *head;
1107 unsigned long flags;1109 unsigned long flags;
1110 void *old_map_ptr;
1108 struct bucket *b;1111 struct bucket *b;
1109 u32 key_size, hash;1112 u32 key_size, hash;
1110 int ret;1113 int ret;
@@ -1183,12 +1186,27 @@ static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, · 72 unmodified lines
1183 hlist_nulls_add_head_rcu(&l_new->hash_node, head);1186 hlist_nulls_add_head_rcu(&l_new->hash_node, head);
1184 if (l_old) {1187 if (l_old) {
1185 hlist_nulls_del_rcu(&l_old->hash_node);1188 hlist_nulls_del_rcu(&l_old->hash_node);
1189
1190 /* l_old has already been stashed in htab->extra_elems, free
1191 * its special fields before it is available for reuse. Also
1192 * save the old map pointer in htab of maps before unlock
1193 * and release it after unlock.
1194 */
1195 old_map_ptr = NULL;
1196 if (htab_is_prealloc(htab)) {
1197 if (map->ops->map_fd_put_ptr)
1198 old_map_ptr = fd_htab_map_get_ptr(map, l_old);
1199 check_and_free_fields(htab, l_old);
1200 }
1201 }
1202 htab_unlock_bucket(htab, b, hash, flags);
1203 if (l_old) {
1204 if (old_map_ptr)
1205 map->ops->map_fd_put_ptr(map, old_map_ptr, true);
1186 if (!htab_is_prealloc(htab))1206 if (!htab_is_prealloc(htab))
1187 free_htab_elem(htab, l_old);1207 free_htab_elem(htab, l_old);
1188 else
1189 check_and_free_fields(htab, l_old);
1190 }1208 }
1191 ret = 0;1209 return 0;
1192err:1210err:
1193 htab_unlock_bucket(htab, b, hash, flags);1211 htab_unlock_bucket(htab, b, hash, flags);
1194 return ret;1212 return ret;
@@ -1432,15 +1450,15 @@ static long htab_map_delete_elem(struct bpf_map *map, void *key) · 237 unmodified lines
1432 return ret;1450 return ret;
1433 1451
1434 l = lookup_elem_raw(head, hash, key, key_size);1452 l = lookup_elem_raw(head, hash, key, key_size);
1435 1453 if (l)
1436 if (l) {
1437 hlist_nulls_del_rcu(&l->hash_node);1454 hlist_nulls_del_rcu(&l->hash_node);
1438 free_htab_elem(htab, l);1455 else
1439 } else {
1440 ret = -ENOENT;1456 ret = -ENOENT;
1441 }
1442 1457
1443 htab_unlock_bucket(htab, b, hash, flags);1458 htab_unlock_bucket(htab, b, hash, flags);
1459
1460 if (l)
1461 free_htab_elem(htab, l);
1444 return ret;1462 return ret;
1445}1463}
1446 1464
@@ -1853,13 +1871,14 @@ again_nocopy: · 406 unmodified lines
1853 * may cause deadlock. See comments in function1871 * may cause deadlock. See comments in function
1854 * prealloc_lru_pop(). Let us do bpf_lru_push_free()1872 * prealloc_lru_pop(). Let us do bpf_lru_push_free()
1855 * after releasing the bucket lock.1873 * after releasing the bucket lock.
1874 *
1875 * For htab of maps, htab_put_fd_value() in
1876 * free_htab_elem() may acquire a spinlock with bucket
1877 * lock being held and it violates the lock rule, so
1878 * invoke free_htab_elem() after unlock as well.
1856 */1879 */
1857 if (is_lru_map) {1880 l->batch_flink = node_to_free;
1858 l->batch_flink = node_to_free;1881 node_to_free = l;
1859 node_to_free = l;
1860 } else {
1861 free_htab_elem(htab, l);
1862 }
1863 }1882 }
1864 dst_key += key_size;1883 dst_key += key_size;
1865 dst_val += value_size;1884 dst_val += value_size;
@@ -1871,7 +1890,10 @@ again_nocopy: · 5 unmodified lines
1871 while (node_to_free) {1890 while (node_to_free) {
1872 l = node_to_free;1891 l = node_to_free;
1873 node_to_free = node_to_free->batch_flink;1892 node_to_free = node_to_free->batch_flink;
1874 htab_lru_push_free(htab, l);1893 if (is_lru_map)
1894 htab_lru_push_free(htab, l);
1895 else
1896 free_htab_elem(htab, l);
1875 }1897 }
1876 1898
1877next_batch:1899next_batch:
kernel/bpf/helpers.c+27−2
@@ -2521,6 +2521,25 @@ __bpf_kfunc struct task_struct *bpf_task_from_pid(s32 pid) · 2520 unmodified lines
2521 return p;2521 return p;
2522}2522}
2523 2523
2524/**
2525 * bpf_task_from_vpid - Find a struct task_struct from its vpid by looking it up
2526 * in the pid namespace of the current task. If a task is returned, it must
2527 * either be stored in a map, or released with bpf_task_release().
2528 * @vpid: The vpid of the task being looked up.
2529 */
2530__bpf_kfunc struct task_struct *bpf_task_from_vpid(s32 vpid)
2531{
2532 struct task_struct *p;
2533
2534 rcu_read_lock();
2535 p = find_task_by_vpid(vpid);
2536 if (p)
2537 p = bpf_task_acquire(p);
2538 rcu_read_unlock();
2539
2540 return p;
2541}
2542
2524/**2543/**
2525 * bpf_dynptr_slice() - Obtain a read-only pointer to the dynptr data.2544 * bpf_dynptr_slice() - Obtain a read-only pointer to the dynptr data.
2526 * @p: The dynptr whose data slice to retrieve2545 * @p: The dynptr whose data slice to retrieve
@@ -3068,7 +3087,9 @@ BTF_ID_FLAGS(func, bpf_task_under_cgroup, KF_RCU) · 541 unmodified lines
3068BTF_ID_FLAGS(func, bpf_task_get_cgroup1, KF_ACQUIRE | KF_RCU | KF_RET_NULL)3087BTF_ID_FLAGS(func, bpf_task_get_cgroup1, KF_ACQUIRE | KF_RCU | KF_RET_NULL)
3069#endif3088#endif
3070BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL)3089BTF_ID_FLAGS(func, bpf_task_from_pid, KF_ACQUIRE | KF_RET_NULL)
3090BTF_ID_FLAGS(func, bpf_task_from_vpid, KF_ACQUIRE | KF_RET_NULL)
3071BTF_ID_FLAGS(func, bpf_throw)3091BTF_ID_FLAGS(func, bpf_throw)
3092BTF_ID_FLAGS(func, bpf_send_signal_task, KF_TRUSTED_ARGS)
3072BTF_KFUNCS_END(generic_btf_ids)3093BTF_KFUNCS_END(generic_btf_ids)
3073 3094
3074static const struct btf_kfunc_id_set generic_kfunc_set = {3095static const struct btf_kfunc_id_set generic_kfunc_set = {
@@ -3086,8 +3107,8 @@ BTF_ID(func, bpf_cgroup_release_dtor) · 11 unmodified lines
3086#endif3107#endif
3087 3108
3088BTF_KFUNCS_START(common_btf_ids)3109BTF_KFUNCS_START(common_btf_ids)
3089BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx)3110BTF_ID_FLAGS(func, bpf_cast_to_kern_ctx, KF_FASTCALL)
3090BTF_ID_FLAGS(func, bpf_rdonly_cast)3111BTF_ID_FLAGS(func, bpf_rdonly_cast, KF_FASTCALL)
3091BTF_ID_FLAGS(func, bpf_rcu_read_lock)3112BTF_ID_FLAGS(func, bpf_rcu_read_lock)
3092BTF_ID_FLAGS(func, bpf_rcu_read_unlock)3113BTF_ID_FLAGS(func, bpf_rcu_read_unlock)
3093BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL)3114BTF_ID_FLAGS(func, bpf_dynptr_slice, KF_RET_NULL)
@@ -3124,6 +3145,10 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) · 30 unmodified lines
3124BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL)3145BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT | KF_RET_NULL)
3125BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)3146BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)
3126BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)3147BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)
3148BTF_ID_FLAGS(func, bpf_get_kmem_cache)
3149BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW | KF_SLEEPABLE)
3150BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT | KF_RET_NULL | KF_SLEEPABLE)
3151BTF_ID_FLAGS(func, bpf_iter_kmem_cache_destroy, KF_ITER_DESTROY | KF_SLEEPABLE)
3127BTF_KFUNCS_END(common_btf_ids)3152BTF_KFUNCS_END(common_btf_ids)
3128 3153
3129static const struct btf_kfunc_id_set common_kfunc_set = {3154static const struct btf_kfunc_id_set common_kfunc_set = {
kernel/bpf/kmem_cache_iter.c+238−0
@@ -0,0 +1,238 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2024 Google */
3#include <linux/bpf.h>
4#include <linux/btf_ids.h>
5#include <linux/slab.h>
6#include <linux/kernel.h>
7#include <linux/seq_file.h>
8
9#include "../../mm/slab.h" /* kmem_cache, slab_caches and slab_mutex */
10
11/* open-coded version */
12struct bpf_iter_kmem_cache {
13 __u64 __opaque[1];
14} __attribute__((aligned(8)));
15
16struct bpf_iter_kmem_cache_kern {
17 struct kmem_cache *pos;
18} __attribute__((aligned(8)));
19
20#define KMEM_CACHE_POS_START ((void *)1L)
21
22__bpf_kfunc_start_defs();
23
24__bpf_kfunc int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it)
25{
26 struct bpf_iter_kmem_cache_kern *kit = (void *)it;
27
28 BUILD_BUG_ON(sizeof(*kit) > sizeof(*it));
29 BUILD_BUG_ON(__alignof__(*kit) != __alignof__(*it));
30
31 kit->pos = KMEM_CACHE_POS_START;
32 return 0;
33}
34
35__bpf_kfunc struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it)
36{
37 struct bpf_iter_kmem_cache_kern *kit = (void *)it;
38 struct kmem_cache *prev = kit->pos;
39 struct kmem_cache *next;
40 bool destroy = false;
41
42 if (!prev)
43 return NULL;
44
45 mutex_lock(&slab_mutex);
46
47 if (list_empty(&slab_caches)) {
48 mutex_unlock(&slab_mutex);
49 return NULL;
50 }
51
52 if (prev == KMEM_CACHE_POS_START)
53 next = list_first_entry(&slab_caches, struct kmem_cache, list);
54 else if (list_last_entry(&slab_caches, struct kmem_cache, list) == prev)
55 next = NULL;
56 else
57 next = list_next_entry(prev, list);
58
59 /* boot_caches have negative refcount, don't touch them */
60 if (next && next->refcount > 0)
61 next->refcount++;
62
63 /* Skip kmem_cache_destroy() for active entries */
64 if (prev && prev != KMEM_CACHE_POS_START) {
65 if (prev->refcount > 1)
66 prev->refcount--;
67 else if (prev->refcount == 1)
68 destroy = true;
69 }
70
71 mutex_unlock(&slab_mutex);
72
73 if (destroy)
74 kmem_cache_destroy(prev);
75
76 kit->pos = next;
77 return next;
78}
79
80__bpf_kfunc void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it)
81{
82 struct bpf_iter_kmem_cache_kern *kit = (void *)it;
83 struct kmem_cache *s = kit->pos;
84 bool destroy = false;
85
86 if (s == NULL || s == KMEM_CACHE_POS_START)
87 return;
88
89 mutex_lock(&slab_mutex);
90
91 /* Skip kmem_cache_destroy() for active entries */
92 if (s->refcount > 1)
93 s->refcount--;
94 else if (s->refcount == 1)
95 destroy = true;
96
97 mutex_unlock(&slab_mutex);
98
99 if (destroy)
100 kmem_cache_destroy(s);
101}
102
103__bpf_kfunc_end_defs();
104
105struct bpf_iter__kmem_cache {
106 __bpf_md_ptr(struct bpf_iter_meta *, meta);
107 __bpf_md_ptr(struct kmem_cache *, s);
108};
109
110union kmem_cache_iter_priv {
111 struct bpf_iter_kmem_cache it;
112 struct bpf_iter_kmem_cache_kern kit;
113};
114
115static void *kmem_cache_iter_seq_start(struct seq_file *seq, loff_t *pos)
116{
117 loff_t cnt = 0;
118 bool found = false;
119 struct kmem_cache *s;
120 union kmem_cache_iter_priv *p = seq->private;
121
122 mutex_lock(&slab_mutex);
123
124 /* Find an entry at the given position in the slab_caches list instead
125 * of keeping a reference (of the last visited entry, if any) out of
126 * slab_mutex. It might miss something if one is deleted in the middle
127 * while it releases the lock. But it should be rare and there's not
128 * much we can do about it.
129 */
130 list_for_each_entry(s, &slab_caches, list) {
131 if (cnt == *pos) {
132 /* Make sure this entry remains in the list by getting
133 * a new reference count. Note that boot_cache entries
134 * have a negative refcount, so don't touch them.
135 */
136 if (s->refcount > 0)
137 s->refcount++;
138 found = true;
139 break;
140 }
141 cnt++;
142 }
143 mutex_unlock(&slab_mutex);
144
145 if (!found)
146 s = NULL;
147
148 p->kit.pos = s;
149 return s;
150}
151
152static void kmem_cache_iter_seq_stop(struct seq_file *seq, void *v)
153{
154 struct bpf_iter_meta meta;
155 struct bpf_iter__kmem_cache ctx = {
156 .meta = &meta,
157 .s = v,
158 };
159 union kmem_cache_iter_priv *p = seq->private;
160 struct bpf_prog *prog;
161
162 meta.seq = seq;
163 prog = bpf_iter_get_info(&meta, true);
164 if (prog && !ctx.s)
165 bpf_iter_run_prog(prog, &ctx);
166
167 bpf_iter_kmem_cache_destroy(&p->it);
168}
169
170static void *kmem_cache_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos)
171{
172 union kmem_cache_iter_priv *p = seq->private;
173
174 ++*pos;
175
176 return bpf_iter_kmem_cache_next(&p->it);
177}
178
179static int kmem_cache_iter_seq_show(struct seq_file *seq, void *v)
180{
181 struct bpf_iter_meta meta;
182 struct bpf_iter__kmem_cache ctx = {
183 .meta = &meta,
184 .s = v,
185 };
186 struct bpf_prog *prog;
187 int ret = 0;
188
189 meta.seq = seq;
190 prog = bpf_iter_get_info(&meta, false);
191 if (prog)
192 ret = bpf_iter_run_prog(prog, &ctx);
193
194 return ret;
195}
196
197static const struct seq_operations kmem_cache_iter_seq_ops = {
198 .start = kmem_cache_iter_seq_start,
199 .next = kmem_cache_iter_seq_next,
200 .stop = kmem_cache_iter_seq_stop,
201 .show = kmem_cache_iter_seq_show,
202};
203
204BTF_ID_LIST_GLOBAL_SINGLE(bpf_kmem_cache_btf_id, struct, kmem_cache)
205
206static const struct bpf_iter_seq_info kmem_cache_iter_seq_info = {
207 .seq_ops = &kmem_cache_iter_seq_ops,
208 .seq_priv_size = sizeof(union kmem_cache_iter_priv),
209};
210
211static void bpf_iter_kmem_cache_show_fdinfo(const struct bpf_iter_aux_info *aux,
212 struct seq_file *seq)
213{
214 seq_puts(seq, "kmem_cache iter\n");
215}
216
217DEFINE_BPF_ITER_FUNC(kmem_cache, struct bpf_iter_meta *meta,
218 struct kmem_cache *s)
219
220static struct bpf_iter_reg bpf_kmem_cache_reg_info = {
221 .target = "kmem_cache",
222 .feature = BPF_ITER_RESCHED,
223 .show_fdinfo = bpf_iter_kmem_cache_show_fdinfo,
224 .ctx_arg_info_size = 1,
225 .ctx_arg_info = {
226 { offsetof(struct bpf_iter__kmem_cache, s),
227 PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
228 },
229 .seq_info = &kmem_cache_iter_seq_info,
230};
231
232static int __init bpf_kmem_cache_iter_init(void)
233{
234 bpf_kmem_cache_reg_info.ctx_arg_info[0].btf_id = bpf_kmem_cache_btf_id[0];
235 return bpf_iter_reg_target(&bpf_kmem_cache_reg_info);
236}
237
238late_initcall(bpf_kmem_cache_iter_init);
kernel/bpf/memalloc.c+1−4
@@ -254,11 +254,8 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) · 253 unmodified lines
254 254
255static void free_one(void *obj, bool percpu)255static void free_one(void *obj, bool percpu)
256{256{
257 if (percpu) {257 if (percpu)
258 free_percpu(((void __percpu **)obj)[1]);258 free_percpu(((void __percpu **)obj)[1]);
259 kfree(obj);
260 return;
261 }
262 259
263 kfree(obj);260 kfree(obj);
264}261}
kernel/bpf/range_tree.c+272−0
@@ -0,0 +1,272 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3#include <linux/interval_tree_generic.h>
4#include <linux/slab.h>
5#include <linux/bpf_mem_alloc.h>
6#include <linux/bpf.h>
7#include "range_tree.h"
8
9/*
10 * struct range_tree is a data structure used to allocate contiguous memory
11 * ranges in bpf arena. It's a large bitmap. The contiguous sequence of bits is
12 * represented by struct range_node or 'rn' for short.
13 * rn->rn_rbnode links it into an interval tree while
14 * rn->rb_range_size links it into a second rbtree sorted by size of the range.
15 * __find_range() performs binary search and best fit algorithm to find the
16 * range less or equal requested size.
17 * range_tree_clear/set() clears or sets a range of bits in this bitmap. The
18 * adjacent ranges are merged or split at the same time.
19 *
20 * The split/merge logic is based/borrowed from XFS's xbitmap32 added
21 * in commit 6772fcc8890a ("xfs: convert xbitmap to interval tree").
22 *
23 * The implementation relies on external lock to protect rbtree-s.
24 * The alloc/free of range_node-s is done via bpf_mem_alloc.
25 *
26 * bpf arena is using range_tree to represent unallocated slots.
27 * At init time:
28 * range_tree_set(rt, 0, max);
29 * Then:
30 * start = range_tree_find(rt, len);
31 * if (start >= 0)
32 * range_tree_clear(rt, start, len);
33 * to find free range and mark slots as allocated and later:
34 * range_tree_set(rt, start, len);
35 * to mark as unallocated after use.
36 */
37struct range_node {
38 struct rb_node rn_rbnode;
39 struct rb_node rb_range_size;
40 u32 rn_start;
41 u32 rn_last; /* inclusive */
42 u32 __rn_subtree_last;
43};
44
45static struct range_node *rb_to_range_node(struct rb_node *rb)
46{
47 return rb_entry(rb, struct range_node, rb_range_size);
48}
49
50static u32 rn_size(struct range_node *rn)
51{
52 return rn->rn_last - rn->rn_start + 1;
53}
54
55/* Find range that fits best to requested size */
56static inline struct range_node *__find_range(struct range_tree *rt, u32 len)
57{
58 struct rb_node *rb = rt->range_size_root.rb_root.rb_node;
59 struct range_node *best = NULL;
60
61 while (rb) {
62 struct range_node *rn = rb_to_range_node(rb);
63
64 if (len <= rn_size(rn)) {
65 best = rn;
66 rb = rb->rb_right;
67 } else {
68 rb = rb->rb_left;
69 }
70 }
71
72 return best;
73}
74
75s64 range_tree_find(struct range_tree *rt, u32 len)
76{
77 struct range_node *rn;
78
79 rn = __find_range(rt, len);
80 if (!rn)
81 return -ENOENT;
82 return rn->rn_start;
83}
84
85/* Insert the range into rbtree sorted by the range size */
86static inline void __range_size_insert(struct range_node *rn,
87 struct rb_root_cached *root)
88{
89 struct rb_node **link = &root->rb_root.rb_node, *rb = NULL;
90 u64 size = rn_size(rn);
91 bool leftmost = true;
92
93 while (*link) {
94 rb = *link;
95 if (size > rn_size(rb_to_range_node(rb))) {
96 link = &rb->rb_left;
97 } else {
98 link = &rb->rb_right;
99 leftmost = false;
100 }
101 }
102
103 rb_link_node(&rn->rb_range_size, rb, link);
104 rb_insert_color_cached(&rn->rb_range_size, root, leftmost);
105}
106
107#define START(node) ((node)->rn_start)
108#define LAST(node) ((node)->rn_last)
109
110INTERVAL_TREE_DEFINE(struct range_node, rn_rbnode, u32,
111 __rn_subtree_last, START, LAST,
112 static inline __maybe_unused,
113 __range_it)
114
115static inline __maybe_unused void
116range_it_insert(struct range_node *rn, struct range_tree *rt)
117{
118 __range_size_insert(rn, &rt->range_size_root);
119 __range_it_insert(rn, &rt->it_root);
120}
121
122static inline __maybe_unused void
123range_it_remove(struct range_node *rn, struct range_tree *rt)
124{
125 rb_erase_cached(&rn->rb_range_size, &rt->range_size_root);
126 RB_CLEAR_NODE(&rn->rb_range_size);
127 __range_it_remove(rn, &rt->it_root);
128}
129
130static inline __maybe_unused struct range_node *
131range_it_iter_first(struct range_tree *rt, u32 start, u32 last)
132{
133 return __range_it_iter_first(&rt->it_root, start, last);
134}
135
136/* Clear the range in this range tree */
137int range_tree_clear(struct range_tree *rt, u32 start, u32 len)
138{
139 u32 last = start + len - 1;
140 struct range_node *new_rn;
141 struct range_node *rn;
142
143 while ((rn = range_it_iter_first(rt, start, last))) {
144 if (rn->rn_start < start && rn->rn_last > last) {
145 u32 old_last = rn->rn_last;
146
147 /* Overlaps with the entire clearing range */
148 range_it_remove(rn, rt);
149 rn->rn_last = start - 1;
150 range_it_insert(rn, rt);
151
152 /* Add a range */
153 migrate_disable();
154 new_rn = bpf_mem_alloc(&bpf_global_ma, sizeof(struct range_node));
155 migrate_enable();
156 if (!new_rn)
157 return -ENOMEM;
158 new_rn->rn_start = last + 1;
159 new_rn->rn_last = old_last;
160 range_it_insert(new_rn, rt);
161 } else if (rn->rn_start < start) {
162 /* Overlaps with the left side of the clearing range */
163 range_it_remove(rn, rt);
164 rn->rn_last = start - 1;
165 range_it_insert(rn, rt);
166 } else if (rn->rn_last > last) {
167 /* Overlaps with the right side of the clearing range */
168 range_it_remove(rn, rt);
169 rn->rn_start = last + 1;
170 range_it_insert(rn, rt);
171 break;
172 } else {
173 /* in the middle of the clearing range */
174 range_it_remove(rn, rt);
175 migrate_disable();
176 bpf_mem_free(&bpf_global_ma, rn);
177 migrate_enable();
178 }
179 }
180 return 0;
181}
182
183/* Is the whole range set ? */
184int is_range_tree_set(struct range_tree *rt, u32 start, u32 len)
185{
186 u32 last = start + len - 1;
187 struct range_node *left;
188
189 /* Is this whole range set ? */
190 left = range_it_iter_first(rt, start, last);
191 if (left && left->rn_start <= start && left->rn_last >= last)
192 return 0;
193 return -ESRCH;
194}
195
196/* Set the range in this range tree */
197int range_tree_set(struct range_tree *rt, u32 start, u32 len)
198{
199 u32 last = start + len - 1;
200 struct range_node *right;
201 struct range_node *left;
202 int err;
203
204 /* Is this whole range already set ? */
205 left = range_it_iter_first(rt, start, last);
206 if (left && left->rn_start <= start && left->rn_last >= last)
207 return 0;
208
209 /* Clear out everything in the range we want to set. */
210 err = range_tree_clear(rt, start, len);
211 if (err)
212 return err;
213
214 /* Do we have a left-adjacent range ? */
215 left = range_it_iter_first(rt, start - 1, start - 1);
216 if (left && left->rn_last + 1 != start)
217 return -EFAULT;
218
219 /* Do we have a right-adjacent range ? */
220 right = range_it_iter_first(rt, last + 1, last + 1);
221 if (right && right->rn_start != last + 1)
222 return -EFAULT;
223
224 if (left && right) {
225 /* Combine left and right adjacent ranges */
226 range_it_remove(left, rt);
227 range_it_remove(right, rt);
228 left->rn_last = right->rn_last;
229 range_it_insert(left, rt);
230 migrate_disable();
231 bpf_mem_free(&bpf_global_ma, right);
232 migrate_enable();
233 } else if (left) {
234 /* Combine with the left range */
235 range_it_remove(left, rt);
236 left->rn_last = last;
237 range_it_insert(left, rt);
238 } else if (right) {
239 /* Combine with the right range */
240 range_it_remove(right, rt);
241 right->rn_start = start;
242 range_it_insert(right, rt);
243 } else {
244 migrate_disable();
245 left = bpf_mem_alloc(&bpf_global_ma, sizeof(struct range_node));
246 migrate_enable();
247 if (!left)
248 return -ENOMEM;
249 left->rn_start = start;
250 left->rn_last = last;
251 range_it_insert(left, rt);
252 }
253 return 0;
254}
255
256void range_tree_destroy(struct range_tree *rt)
257{
258 struct range_node *rn;
259
260 while ((rn = range_it_iter_first(rt, 0, -1U))) {
261 range_it_remove(rn, rt);
262 migrate_disable();
263 bpf_mem_free(&bpf_global_ma, rn);
264 migrate_enable();
265 }
266}
267
268void range_tree_init(struct range_tree *rt)
269{
270 rt->it_root = RB_ROOT_CACHED;
271 rt->range_size_root = RB_ROOT_CACHED;
272}
kernel/bpf/range_tree.h+21−0
@@ -0,0 +1,21 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3#ifndef _RANGE_TREE_H
4#define _RANGE_TREE_H 1
5
6struct range_tree {
7 /* root of interval tree */
8 struct rb_root_cached it_root;
9 /* root of rbtree of interval sizes */
10 struct rb_root_cached range_size_root;
11};
12
13void range_tree_init(struct range_tree *rt);
14void range_tree_destroy(struct range_tree *rt);
15
16int range_tree_clear(struct range_tree *rt, u32 start, u32 len);
17int range_tree_set(struct range_tree *rt, u32 start, u32 len);
18int is_range_tree_set(struct range_tree *rt, u32 start, u32 len);
19s64 range_tree_find(struct range_tree *rt, u32 len);
20
21#endif
kernel/bpf/syscall.c+115−9
@@ -155,6 +155,89 @@ static void maybe_wait_bpf_programs(struct bpf_map *map) · 154 unmodified lines
155 synchronize_rcu();155 synchronize_rcu();
156}156}
157 157
158static void unpin_uptr_kaddr(void *kaddr)
159{
160 if (kaddr)
161 unpin_user_page(virt_to_page(kaddr));
162}
163
164static void __bpf_obj_unpin_uptrs(struct btf_record *rec, u32 cnt, void *obj)
165{
166 const struct btf_field *field;
167 void **uptr_addr;
168 int i;
169
170 for (i = 0, field = rec->fields; i < cnt; i++, field++) {
171 if (field->type != BPF_UPTR)
172 continue;
173
174 uptr_addr = obj + field->offset;
175 unpin_uptr_kaddr(*uptr_addr);
176 }
177}
178
179static void bpf_obj_unpin_uptrs(struct btf_record *rec, void *obj)
180{
181 if (!btf_record_has_field(rec, BPF_UPTR))
182 return;
183
184 __bpf_obj_unpin_uptrs(rec, rec->cnt, obj);
185}
186
187static int bpf_obj_pin_uptrs(struct btf_record *rec, void *obj)
188{
189 const struct btf_field *field;
190 const struct btf_type *t;
191 unsigned long start, end;
192 struct page *page;
193 void **uptr_addr;
194 int i, err;
195
196 if (!btf_record_has_field(rec, BPF_UPTR))
197 return 0;
198
199 for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) {
200 if (field->type != BPF_UPTR)
201 continue;
202
203 uptr_addr = obj + field->offset;
204 start = *(unsigned long *)uptr_addr;
205 if (!start)
206 continue;
207
208 t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
209 /* t->size was checked for zero before */
210 if (check_add_overflow(start, t->size - 1, &end)) {
211 err = -EFAULT;
212 goto unpin_all;
213 }
214
215 /* The uptr's struct cannot span across two pages */
216 if ((start & PAGE_MASK) != (end & PAGE_MASK)) {
217 err = -EOPNOTSUPP;
218 goto unpin_all;
219 }
220
221 err = pin_user_pages_fast(start, 1, FOLL_LONGTERM | FOLL_WRITE, &page);
222 if (err != 1)
223 goto unpin_all;
224
225 if (PageHighMem(page)) {
226 err = -EOPNOTSUPP;
227 unpin_user_page(page);
228 goto unpin_all;
229 }
230
231 *uptr_addr = page_address(page) + offset_in_page(start);
232 }
233
234 return 0;
235
236unpin_all:
237 __bpf_obj_unpin_uptrs(rec, i, obj);
238 return err;
239}
240
158static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,241static int bpf_map_update_value(struct bpf_map *map, struct file *map_file,
159 void *key, void *value, __u64 flags)242 void *key, void *value, __u64 flags)
160{243{
@@ -199,9 +282,14 @@ static int bpf_map_update_value(struct bpf_map *map, struct file *map_file, · 38 unmodified lines
199 map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {282 map->map_type == BPF_MAP_TYPE_BLOOM_FILTER) {
200 err = map->ops->map_push_elem(map, value, flags);283 err = map->ops->map_push_elem(map, value, flags);
201 } else {284 } else {
202 rcu_read_lock();285 err = bpf_obj_pin_uptrs(map->record, value);
203 err = map->ops->map_update_elem(map, key, value, flags);286 if (!err) {
204 rcu_read_unlock();287 rcu_read_lock();
288 err = map->ops->map_update_elem(map, key, value, flags);
289 rcu_read_unlock();
290 if (err)
291 bpf_obj_unpin_uptrs(map->record, value);
292 }
205 }293 }
206 bpf_enable_instrumentation();294 bpf_enable_instrumentation();
207 295
@@ -548,6 +636,7 @@ void btf_record_free(struct btf_record *rec) · 340 unmodified lines
548 case BPF_KPTR_UNREF:636 case BPF_KPTR_UNREF:
549 case BPF_KPTR_REF:637 case BPF_KPTR_REF:
550 case BPF_KPTR_PERCPU:638 case BPF_KPTR_PERCPU:
639 case BPF_UPTR:
551 if (rec->fields[i].kptr.module)640 if (rec->fields[i].kptr.module)
552 module_put(rec->fields[i].kptr.module);641 module_put(rec->fields[i].kptr.module);
553 if (btf_is_kernel(rec->fields[i].kptr.btf))642 if (btf_is_kernel(rec->fields[i].kptr.btf))
@@ -597,6 +686,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec) · 43 unmodified lines
597 case BPF_KPTR_UNREF:686 case BPF_KPTR_UNREF:
598 case BPF_KPTR_REF:687 case BPF_KPTR_REF:
599 case BPF_KPTR_PERCPU:688 case BPF_KPTR_PERCPU:
689 case BPF_UPTR:
600 if (btf_is_kernel(fields[i].kptr.btf))690 if (btf_is_kernel(fields[i].kptr.btf))
601 btf_get(fields[i].kptr.btf);691 btf_get(fields[i].kptr.btf);
602 if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {692 if (fields[i].kptr.module && !try_module_get(fields[i].kptr.module)) {
@@ -714,6 +804,10 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj) · 111 unmodified lines
714 field->kptr.dtor(xchgd_field);804 field->kptr.dtor(xchgd_field);
715 }805 }
716 break;806 break;
807 case BPF_UPTR:
808 /* The caller ensured that no one is using the uptr */
809 unpin_uptr_kaddr(*(void **)field_ptr);
810 break;
717 case BPF_LIST_HEAD:811 case BPF_LIST_HEAD:
718 if (WARN_ON_ONCE(rec->spin_lock_off < 0))812 if (WARN_ON_ONCE(rec->spin_lock_off < 0))
719 continue;813 continue;
@@ -1105,7 +1199,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, · 385 unmodified lines
1105 1199
1106 map->record = btf_parse_fields(btf, value_type,1200 map->record = btf_parse_fields(btf, value_type,
1107 BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |1201 BPF_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
1108 BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE,1202 BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR,
1109 map->value_size);1203 map->value_size);
1110 if (!IS_ERR_OR_NULL(map->record)) {1204 if (!IS_ERR_OR_NULL(map->record)) {
1111 int i;1205 int i;
@@ -1161,6 +1255,12 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token, · 49 unmodified lines
1161 goto free_map_tab;1255 goto free_map_tab;
1162 }1256 }
1163 break;1257 break;
1258 case BPF_UPTR:
1259 if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE) {
1260 ret = -EOPNOTSUPP;
1261 goto free_map_tab;
1262 }
1263 break;
1164 case BPF_LIST_HEAD:1264 case BPF_LIST_HEAD:
1165 case BPF_RB_ROOT:1265 case BPF_RB_ROOT:
1166 if (map->map_type != BPF_MAP_TYPE_HASH &&1266 if (map->map_type != BPF_MAP_TYPE_HASH &&
@@ -3218,7 +3318,8 @@ static void bpf_tracing_link_release(struct bpf_link *link) · 2051 unmodified lines
3218 container_of(link, struct bpf_tracing_link, link.link);3318 container_of(link, struct bpf_tracing_link, link.link);
3219 3319
3220 WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,3320 WARN_ON_ONCE(bpf_trampoline_unlink_prog(&tr_link->link,
3221 tr_link->trampoline));3321 tr_link->trampoline,
3322 tr_link->tgt_prog));
3222 3323
3223 bpf_trampoline_put(tr_link->trampoline);3324 bpf_trampoline_put(tr_link->trampoline);
3224 3325
@@ -3358,7 +3459,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, · 133 unmodified lines
3358 * in prog->aux3459 * in prog->aux
3359 *3460 *
3360 * - if prog->aux->dst_trampoline is NULL, the program has already been3461 * - if prog->aux->dst_trampoline is NULL, the program has already been
3361 * attached to a target and its initial target was cleared (below)3462 * attached to a target and its initial target was cleared (below)
3362 *3463 *
3363 * - if tgt_prog != NULL, the caller specified tgt_prog_fd +3464 * - if tgt_prog != NULL, the caller specified tgt_prog_fd +
3364 * target_btf_id using the link_create API.3465 * target_btf_id using the link_create API.
@@ -3433,7 +3534,7 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog, · 68 unmodified lines
3433 if (err)3534 if (err)
3434 goto out_unlock;3535 goto out_unlock;
3435 3536
3436 err = bpf_trampoline_link_prog(&link->link, tr);3537 err = bpf_trampoline_link_prog(&link->link, tr, tgt_prog);
3437 if (err) {3538 if (err) {
3438 bpf_link_cleanup(&link_primer);3539 bpf_link_cleanup(&link_primer);
3439 link = NULL;3540 link = NULL;
@@ -4002,10 +4103,14 @@ static int bpf_prog_attach_check_attach_type(const struct bpf_prog *prog, · 562 unmodified lines
4002 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&4103 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI &&
4003 attach_type != BPF_TRACE_UPROBE_MULTI)4104 attach_type != BPF_TRACE_UPROBE_MULTI)
4004 return -EINVAL;4105 return -EINVAL;
4106 if (prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION &&
4107 attach_type != BPF_TRACE_UPROBE_SESSION)
4108 return -EINVAL;
4005 if (attach_type != BPF_PERF_EVENT &&4109 if (attach_type != BPF_PERF_EVENT &&
4006 attach_type != BPF_TRACE_KPROBE_MULTI &&4110 attach_type != BPF_TRACE_KPROBE_MULTI &&
4007 attach_type != BPF_TRACE_KPROBE_SESSION &&4111 attach_type != BPF_TRACE_KPROBE_SESSION &&
4008 attach_type != BPF_TRACE_UPROBE_MULTI)4112 attach_type != BPF_TRACE_UPROBE_MULTI &&
4113 attach_type != BPF_TRACE_UPROBE_SESSION)
4009 return -EINVAL;4114 return -EINVAL;
4010 return 0;4115 return 0;
4011 case BPF_PROG_TYPE_SCHED_CLS:4116 case BPF_PROG_TYPE_SCHED_CLS:
@@ -5258,7 +5363,8 @@ static int link_create(union bpf_attr *attr, bpfptr_t uattr) · 1246 unmodified lines
5258 else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||5363 else if (attr->link_create.attach_type == BPF_TRACE_KPROBE_MULTI ||
5259 attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)5364 attr->link_create.attach_type == BPF_TRACE_KPROBE_SESSION)
5260 ret = bpf_kprobe_multi_link_attach(attr, prog);5365 ret = bpf_kprobe_multi_link_attach(attr, prog);
5261 else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI)5366 else if (attr->link_create.attach_type == BPF_TRACE_UPROBE_MULTI ||
5367 attr->link_create.attach_type == BPF_TRACE_UPROBE_SESSION)
5262 ret = bpf_uprobe_multi_link_attach(attr, prog);5368 ret = bpf_uprobe_multi_link_attach(attr, prog);
5263 break;5369 break;
5264 default:5370 default:
kernel/bpf/trampoline.c+50−10
@@ -115,10 +115,14 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog) · 114 unmodified lines
115 (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);115 (ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
116}116}
117 117
118void bpf_image_ksym_add(void *data, unsigned int size, struct bpf_ksym *ksym)118void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym)
119{119{
120 ksym->start = (unsigned long) data;120 ksym->start = (unsigned long) data;
121 ksym->end = ksym->start + size;121 ksym->end = ksym->start + size;
122}
123
124void bpf_image_ksym_add(struct bpf_ksym *ksym)
125{
122 bpf_ksym_add(ksym);126 bpf_ksym_add(ksym);
123 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,127 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, ksym->start,
124 PAGE_SIZE, false, ksym->name);128 PAGE_SIZE, false, ksym->name);
@@ -377,7 +381,8 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, int size) · 252 unmodified lines
377 ksym = &im->ksym;381 ksym = &im->ksym;
378 INIT_LIST_HEAD_RCU(&ksym->lnode);382 INIT_LIST_HEAD_RCU(&ksym->lnode);
379 snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);383 snprintf(ksym->name, KSYM_NAME_LEN, "bpf_trampoline_%llu", key);
380 bpf_image_ksym_add(image, size, ksym);384 bpf_image_ksym_init(image, size, ksym);
385 bpf_image_ksym_add(ksym);
381 return im;386 return im;
382 387
383out_free_image:388out_free_image:
@@ -523,7 +528,27 @@ static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog) · 139 unmodified lines
523 }528 }
524}529}
525 530
526static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)531static int bpf_freplace_check_tgt_prog(struct bpf_prog *tgt_prog)
532{
533 struct bpf_prog_aux *aux = tgt_prog->aux;
534
535 guard(mutex)(&aux->ext_mutex);
536 if (aux->prog_array_member_cnt)
537 /* Program extensions can not extend target prog when the target
538 * prog has been updated to any prog_array map as tail callee.
539 * It's to prevent a potential infinite loop like:
540 * tgt prog entry -> tgt prog subprog -> freplace prog entry
541 * --tailcall-> tgt prog entry.
542 */
543 return -EBUSY;
544
545 aux->is_extended = true;
546 return 0;
547}
548
549static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link,
550 struct bpf_trampoline *tr,
551 struct bpf_prog *tgt_prog)
527{552{
528 enum bpf_tramp_prog_type kind;553 enum bpf_tramp_prog_type kind;
529 struct bpf_tramp_link *link_exiting;554 struct bpf_tramp_link *link_exiting;
@@ -544,6 +569,9 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr · 14 unmodified lines
544 /* Cannot attach extension if fentry/fexit are in use. */569 /* Cannot attach extension if fentry/fexit are in use. */
545 if (cnt)570 if (cnt)
546 return -EBUSY;571 return -EBUSY;
572 err = bpf_freplace_check_tgt_prog(tgt_prog);
573 if (err)
574 return err;
547 tr->extension_prog = link->link.prog;575 tr->extension_prog = link->link.prog;
548 return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,576 return bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP, NULL,
549 link->link.prog->bpf_func);577 link->link.prog->bpf_func);
@@ -570,17 +598,21 @@ static int __bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_tr · 20 unmodified lines
570 return err;598 return err;
571}599}
572 600
573int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)601int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
602 struct bpf_trampoline *tr,
603 struct bpf_prog *tgt_prog)
574{604{
575 int err;605 int err;
576 606
577 mutex_lock(&tr->mutex);607 mutex_lock(&tr->mutex);
578 err = __bpf_trampoline_link_prog(link, tr);608 err = __bpf_trampoline_link_prog(link, tr, tgt_prog);
579 mutex_unlock(&tr->mutex);609 mutex_unlock(&tr->mutex);
580 return err;610 return err;
581}611}
582 612
583static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)613static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
614 struct bpf_trampoline *tr,
615 struct bpf_prog *tgt_prog)
584{616{
585 enum bpf_tramp_prog_type kind;617 enum bpf_tramp_prog_type kind;
586 int err;618 int err;
@@ -591,6 +623,8 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ · 4 unmodified lines
591 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,623 err = bpf_arch_text_poke(tr->func.addr, BPF_MOD_JUMP,
592 tr->extension_prog->bpf_func, NULL);624 tr->extension_prog->bpf_func, NULL);
593 tr->extension_prog = NULL;625 tr->extension_prog = NULL;
626 guard(mutex)(&tgt_prog->aux->ext_mutex);
627 tgt_prog->aux->is_extended = false;
594 return err;628 return err;
595 }629 }
596 hlist_del_init(&link->tramp_hlist);630 hlist_del_init(&link->tramp_hlist);
@@ -599,12 +633,14 @@ static int __bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_ · 2 unmodified lines
599}633}
600 634
601/* bpf_trampoline_unlink_prog() should never fail. */635/* bpf_trampoline_unlink_prog() should never fail. */
602int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr)636int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link,
637 struct bpf_trampoline *tr,
638 struct bpf_prog *tgt_prog)
603{639{
604 int err;640 int err;
605 641
606 mutex_lock(&tr->mutex);642 mutex_lock(&tr->mutex);
607 err = __bpf_trampoline_unlink_prog(link, tr);643 err = __bpf_trampoline_unlink_prog(link, tr, tgt_prog);
608 mutex_unlock(&tr->mutex);644 mutex_unlock(&tr->mutex);
609 return err;645 return err;
610}646}
@@ -619,7 +655,7 @@ static void bpf_shim_tramp_link_release(struct bpf_link *link) · 8 unmodified lines
619 if (!shim_link->trampoline)655 if (!shim_link->trampoline)
620 return;656 return;
621 657
622 WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline));658 WARN_ON_ONCE(bpf_trampoline_unlink_prog(&shim_link->link, shim_link->trampoline, NULL));
623 bpf_trampoline_put(shim_link->trampoline);659 bpf_trampoline_put(shim_link->trampoline);
624}660}
625 661
@@ -733,7 +769,7 @@ int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, · 107 unmodified lines
733 goto err;769 goto err;
734 }770 }
735 771
736 err = __bpf_trampoline_link_prog(&shim_link->link, tr);772 err = __bpf_trampoline_link_prog(&shim_link->link, tr, NULL);
737 if (err)773 if (err)
738 goto err;774 goto err;
739 775
@@ -868,6 +904,8 @@ static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tram · 128 unmodified lines
868 904
869 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {905 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
870 bpf_prog_inc_misses_counter(prog);906 bpf_prog_inc_misses_counter(prog);
907 if (prog->aux->recursion_detected)
908 prog->aux->recursion_detected(prog);
871 return 0;909 return 0;
872 }910 }
873 return bpf_prog_start_time();911 return bpf_prog_start_time();
@@ -944,6 +982,8 @@ u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, · 70 unmodified lines
944 982
945 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {983 if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
946 bpf_prog_inc_misses_counter(prog);984 bpf_prog_inc_misses_counter(prog);
985 if (prog->aux->recursion_detected)
986 prog->aux->recursion_detected(prog);
947 return 0;987 return 0;
948 }988 }
949 return bpf_prog_start_time();989 return bpf_prog_start_time();
kernel/bpf/verifier.c+418−179
@@ -194,6 +194,8 @@ struct bpf_verifier_stack_elem { · 193 unmodified lines
194 194
195#define BPF_GLOBAL_PERCPU_MA_MAX_SIZE 512195#define BPF_GLOBAL_PERCPU_MA_MAX_SIZE 512
196 196
197#define BPF_PRIV_STACK_MIN_SIZE 64
198
197static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);199static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
198static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);200static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
199static void invalidate_non_owning_refs(struct bpf_verifier_env *env);201static void invalidate_non_owning_refs(struct bpf_verifier_env *env);
@@ -418,6 +420,25 @@ static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) · 218 unmodified lines
418 return rec;420 return rec;
419}421}
420 422
423static bool mask_raw_tp_reg_cond(const struct bpf_verifier_env *env, struct bpf_reg_state *reg) {
424 return reg->type == (PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL) &&
425 bpf_prog_is_raw_tp(env->prog) && !reg->ref_obj_id;
426}
427
428static bool mask_raw_tp_reg(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
429{
430 if (!mask_raw_tp_reg_cond(env, reg))
431 return false;
432 reg->type &= ~PTR_MAYBE_NULL;
433 return true;
434}
435
436static void unmask_raw_tp_reg(struct bpf_reg_state *reg, bool result)
437{
438 if (result)
439 reg->type |= PTR_MAYBE_NULL;
440}
441
421static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)442static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog)
422{443{
423 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;444 struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux;
@@ -1265,6 +1286,7 @@ static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_fun · 841 unmodified lines
1265 if (!dst->refs)1286 if (!dst->refs)
1266 return -ENOMEM;1287 return -ENOMEM;
1267 1288
1289 dst->active_locks = src->active_locks;
1268 dst->acquired_refs = src->acquired_refs;1290 dst->acquired_refs = src->acquired_refs;
1269 return 0;1291 return 0;
1270}1292}
@@ -1335,13 +1357,32 @@ static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) · 64 unmodified lines
1335 if (err)1357 if (err)
1336 return err;1358 return err;
1337 id = ++env->id_gen;1359 id = ++env->id_gen;
1360 state->refs[new_ofs].type = REF_TYPE_PTR;
1338 state->refs[new_ofs].id = id;1361 state->refs[new_ofs].id = id;
1339 state->refs[new_ofs].insn_idx = insn_idx;1362 state->refs[new_ofs].insn_idx = insn_idx;
1340 state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0;
1341 1363
1342 return id;1364 return id;
1343}1365}
1344 1366
1367static int acquire_lock_state(struct bpf_verifier_env *env, int insn_idx, enum ref_state_type type,
1368 int id, void *ptr)
1369{
1370 struct bpf_func_state *state = cur_func(env);
1371 int new_ofs = state->acquired_refs;
1372 int err;
1373
1374 err = resize_reference_state(state, state->acquired_refs + 1);
1375 if (err)
1376 return err;
1377 state->refs[new_ofs].type = type;
1378 state->refs[new_ofs].id = id;
1379 state->refs[new_ofs].insn_idx = insn_idx;
1380 state->refs[new_ofs].ptr = ptr;
1381
1382 state->active_locks++;
1383 return 0;
1384}
1385
1345/* release function corresponding to acquire_reference_state(). Idempotent. */1386/* release function corresponding to acquire_reference_state(). Idempotent. */
1346static int release_reference_state(struct bpf_func_state *state, int ptr_id)1387static int release_reference_state(struct bpf_func_state *state, int ptr_id)
1347{1388{
@@ -1349,10 +1390,9 @@ static int release_reference_state(struct bpf_func_state *state, int ptr_id) · 1 unmodified line
1349 1390
1350 last_idx = state->acquired_refs - 1;1391 last_idx = state->acquired_refs - 1;
1351 for (i = 0; i < state->acquired_refs; i++) {1392 for (i = 0; i < state->acquired_refs; i++) {
1393 if (state->refs[i].type != REF_TYPE_PTR)
1394 continue;
1352 if (state->refs[i].id == ptr_id) {1395 if (state->refs[i].id == ptr_id) {
1353 /* Cannot release caller references in callbacks */
1354 if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
1355 return -EINVAL;
1356 if (last_idx && i != last_idx)1396 if (last_idx && i != last_idx)
1357 memcpy(&state->refs[i], &state->refs[last_idx],1397 memcpy(&state->refs[i], &state->refs[last_idx],
1358 sizeof(*state->refs));1398 sizeof(*state->refs));
@@ -1364,6 +1404,45 @@ static int release_reference_state(struct bpf_func_state *state, int ptr_id) · 5 unmodified lines
1364 return -EINVAL;1404 return -EINVAL;
1365}1405}
1366 1406
1407static int release_lock_state(struct bpf_func_state *state, int type, int id, void *ptr)
1408{
1409 int i, last_idx;
1410
1411 last_idx = state->acquired_refs - 1;
1412 for (i = 0; i < state->acquired_refs; i++) {
1413 if (state->refs[i].type != type)
1414 continue;
1415 if (state->refs[i].id == id && state->refs[i].ptr == ptr) {
1416 if (last_idx && i != last_idx)
1417 memcpy(&state->refs[i], &state->refs[last_idx],
1418 sizeof(*state->refs));
1419 memset(&state->refs[last_idx], 0, sizeof(*state->refs));
1420 state->acquired_refs--;
1421 state->active_locks--;
1422 return 0;
1423 }
1424 }
1425 return -EINVAL;
1426}
1427
1428static struct bpf_reference_state *find_lock_state(struct bpf_verifier_env *env, enum ref_state_type type,
1429 int id, void *ptr)
1430{
1431 struct bpf_func_state *state = cur_func(env);
1432 int i;
1433
1434 for (i = 0; i < state->acquired_refs; i++) {
1435 struct bpf_reference_state *s = &state->refs[i];
1436
1437 if (s->type == REF_TYPE_PTR || s->type != type)
1438 continue;
1439
1440 if (s->id == id && s->ptr == ptr)
1441 return s;
1442 }
1443 return NULL;
1444}
1445
1367static void free_func_state(struct bpf_func_state *state)1446static void free_func_state(struct bpf_func_state *state)
1368{1447{
1369 if (!state)1448 if (!state)
@@ -1373,13 +1452,6 @@ static void free_func_state(struct bpf_func_state *state) · 3 unmodified lines
1373 kfree(state);1452 kfree(state);
1374}1453}
1375 1454
1376static void clear_jmp_history(struct bpf_verifier_state *state)
1377{
1378 kfree(state->jmp_history);
1379 state->jmp_history = NULL;
1380 state->jmp_history_cnt = 0;
1381}
1382
1383static void free_verifier_state(struct bpf_verifier_state *state,1455static void free_verifier_state(struct bpf_verifier_state *state,
1384 bool free_self)1456 bool free_self)
1385{1457{
@@ -1389,7 +1461,6 @@ static void free_verifier_state(struct bpf_verifier_state *state, · 3 unmodified lines
1389 free_func_state(state->frame[i]);1461 free_func_state(state->frame[i]);
1390 state->frame[i] = NULL;1462 state->frame[i] = NULL;
1391 }1463 }
1392 clear_jmp_history(state);
1393 if (free_self)1464 if (free_self)
1394 kfree(state);1465 kfree(state);
1395}1466}
@@ -1415,13 +1486,6 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 19 unmodified lines
1415 struct bpf_func_state *dst;1486 struct bpf_func_state *dst;
1416 int i, err;1487 int i, err;
1417 1488
1418 dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
1419 src->jmp_history_cnt, sizeof(*dst_state->jmp_history),
1420 GFP_USER);
1421 if (!dst_state->jmp_history)
1422 return -ENOMEM;
1423 dst_state->jmp_history_cnt = src->jmp_history_cnt;
1424
1425 /* if dst has more stack frames then src frame, free them, this is also1489 /* if dst has more stack frames then src frame, free them, this is also
1426 * necessary in case of exceptional exits using bpf_throw.1490 * necessary in case of exceptional exits using bpf_throw.
1427 */1491 */
@@ -1434,12 +1498,12 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 6 unmodified lines
1434 dst_state->active_preempt_lock = src->active_preempt_lock;1498 dst_state->active_preempt_lock = src->active_preempt_lock;
1435 dst_state->in_sleepable = src->in_sleepable;1499 dst_state->in_sleepable = src->in_sleepable;
1436 dst_state->curframe = src->curframe;1500 dst_state->curframe = src->curframe;
1437 dst_state->active_lock.ptr = src->active_lock.ptr;
1438 dst_state->active_lock.id = src->active_lock.id;
1439 dst_state->branches = src->branches;1501 dst_state->branches = src->branches;
1440 dst_state->parent = src->parent;1502 dst_state->parent = src->parent;
1441 dst_state->first_insn_idx = src->first_insn_idx;1503 dst_state->first_insn_idx = src->first_insn_idx;
1442 dst_state->last_insn_idx = src->last_insn_idx;1504 dst_state->last_insn_idx = src->last_insn_idx;
1505 dst_state->insn_hist_start = src->insn_hist_start;
1506 dst_state->insn_hist_end = src->insn_hist_end;
1443 dst_state->dfs_depth = src->dfs_depth;1507 dst_state->dfs_depth = src->dfs_depth;
1444 dst_state->callback_unroll_depth = src->callback_unroll_depth;1508 dst_state->callback_unroll_depth = src->callback_unroll_depth;
1445 dst_state->used_as_loop_entry = src->used_as_loop_entry;1509 dst_state->used_as_loop_entry = src->used_as_loop_entry;
@@ -2492,9 +2556,14 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, · 1046 unmodified lines
2492 * The caller state doesn't matter.2556 * The caller state doesn't matter.
2493 * This is async callback. It starts in a fresh stack.2557 * This is async callback. It starts in a fresh stack.
2494 * Initialize it similar to do_check_common().2558 * Initialize it similar to do_check_common().
2559 * But we do need to make sure to not clobber insn_hist, so we keep
2560 * chaining insn_hist_start/insn_hist_end indices as for a normal
2561 * child state.
2495 */2562 */
2496 elem->st.branches = 1;2563 elem->st.branches = 1;
2497 elem->st.in_sleepable = is_sleepable;2564 elem->st.in_sleepable = is_sleepable;
2565 elem->st.insn_hist_start = env->cur_state->insn_hist_end;
2566 elem->st.insn_hist_end = elem->st.insn_hist_start;
2498 frame = kzalloc(sizeof(*frame), GFP_KERNEL);2567 frame = kzalloc(sizeof(*frame), GFP_KERNEL);
2499 if (!frame)2568 if (!frame)
2500 goto err;2569 goto err;
@@ -3474,11 +3543,10 @@ static void linked_regs_unpack(u64 val, struct linked_regs *s) · 973 unmodified lines
3474}3543}
3475 3544
3476/* for any branch, call, exit record the history of jmps in the given state */3545/* for any branch, call, exit record the history of jmps in the given state */
3477static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,3546static int push_insn_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
3478 int insn_flags, u64 linked_regs)3547 int insn_flags, u64 linked_regs)
3479{3548{
3480 u32 cnt = cur->jmp_history_cnt;3549 struct bpf_insn_hist_entry *p;
3481 struct bpf_jmp_history_entry *p;
3482 size_t alloc_size;3550 size_t alloc_size;
3483 3551
3484 /* combine instruction flags if we already recorded this instruction */3552 /* combine instruction flags if we already recorded this instruction */
@@ -3498,29 +3566,32 @@ static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_st · 13 unmodified lines
3498 return 0;3566 return 0;
3499 }3567 }
3500 3568
3501 cnt++;3569 if (cur->insn_hist_end + 1 > env->insn_hist_cap) {
3502 alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));3570 alloc_size = size_mul(cur->insn_hist_end + 1, sizeof(*p));
3503 p = krealloc(cur->jmp_history, alloc_size, GFP_USER);3571 p = kvrealloc(env->insn_hist, alloc_size, GFP_USER);
3504 if (!p)3572 if (!p)
3505 return -ENOMEM;3573 return -ENOMEM;
3506 cur->jmp_history = p;3574 env->insn_hist = p;
3575 env->insn_hist_cap = alloc_size / sizeof(*p);
3576 }
3507 3577
3508 p = &cur->jmp_history[cnt - 1];3578 p = &env->insn_hist[cur->insn_hist_end];
3509 p->idx = env->insn_idx;3579 p->idx = env->insn_idx;
3510 p->prev_idx = env->prev_insn_idx;3580 p->prev_idx = env->prev_insn_idx;
3511 p->flags = insn_flags;3581 p->flags = insn_flags;
3512 p->linked_regs = linked_regs;3582 p->linked_regs = linked_regs;
3513 cur->jmp_history_cnt = cnt;3583
3584 cur->insn_hist_end++;
3514 env->cur_hist_ent = p;3585 env->cur_hist_ent = p;
3515 3586
3516 return 0;3587 return 0;
3517}3588}
3518 3589
3519static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_state *st,3590static struct bpf_insn_hist_entry *get_insn_hist_entry(struct bpf_verifier_env *env,
3520 u32 hist_end, int insn_idx)3591 u32 hist_start, u32 hist_end, int insn_idx)
3521{3592{
3522 if (hist_end > 0 && st->jmp_history[hist_end - 1].idx == insn_idx)3593 if (hist_end > hist_start && env->insn_hist[hist_end - 1].idx == insn_idx)
3523 return &st->jmp_history[hist_end - 1];3594 return &env->insn_hist[hist_end - 1];
3524 return NULL;3595 return NULL;
3525}3596}
3526 3597
@@ -3537,25 +3608,26 @@ static struct bpf_jmp_history_entry *get_jmp_hist_entry(struct bpf_verifier_stat · 10 unmodified lines
3537 * history entry recording a jump from last instruction of parent state and3608 * history entry recording a jump from last instruction of parent state and
3538 * first instruction of given state.3609 * first instruction of given state.
3539 */3610 */
3540static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,3611static int get_prev_insn_idx(const struct bpf_verifier_env *env,
3541 u32 *history)3612 struct bpf_verifier_state *st,
3613 int insn_idx, u32 hist_start, u32 *hist_endp)
3542{3614{
3543 u32 cnt = *history;3615 u32 hist_end = *hist_endp;
3616 u32 cnt = hist_end - hist_start;
3544 3617
3545 if (i == st->first_insn_idx) {3618 if (insn_idx == st->first_insn_idx) {
3546 if (cnt == 0)3619 if (cnt == 0)
3547 return -ENOENT;3620 return -ENOENT;
3548 if (cnt == 1 && st->jmp_history[0].idx == i)3621 if (cnt == 1 && env->insn_hist[hist_start].idx == insn_idx)
3549 return -ENOENT;3622 return -ENOENT;
3550 }3623 }
3551 3624
3552 if (cnt && st->jmp_history[cnt - 1].idx == i) {3625 if (cnt && env->insn_hist[hist_end - 1].idx == insn_idx) {
3553 i = st->jmp_history[cnt - 1].prev_idx;3626 (*hist_endp)--;
3554 (*history)--;3627 return env->insn_hist[hist_end - 1].prev_idx;
3555 } else {3628 } else {
3556 i--;3629 return insn_idx - 1;
3557 }3630 }
3558 return i;
3559}3631}
3560 3632
3561static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)3633static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
@@ -3727,7 +3799,7 @@ static void fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask) · 165 unmodified lines
3727/* If any register R in hist->linked_regs is marked as precise in bt,3799/* If any register R in hist->linked_regs is marked as precise in bt,
3728 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.3800 * do bt_set_frame_{reg,slot}(bt, R) for all registers in hist->linked_regs.
3729 */3801 */
3730static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist)3802static void bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_insn_hist_entry *hist)
3731{3803{
3732 struct linked_regs linked_regs;3804 struct linked_regs linked_regs;
3733 bool some_precise = false;3805 bool some_precise = false;
@@ -3772,7 +3844,7 @@ static bool calls_callback(struct bpf_verifier_env *env, int insn_idx); · 38 unmodified lines
3772 * - *was* processed previously during backtracking.3844 * - *was* processed previously during backtracking.
3773 */3845 */
3774static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,3846static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
3775 struct bpf_jmp_history_entry *hist, struct backtrack_state *bt)3847 struct bpf_insn_hist_entry *hist, struct backtrack_state *bt)
3776{3848{
3777 const struct bpf_insn_cbs cbs = {3849 const struct bpf_insn_cbs cbs = {
3778 .cb_call = disasm_kfunc_name,3850 .cb_call = disasm_kfunc_name,
@@ -4191,7 +4263,7 @@ static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_ · 412 unmodified lines
4191 * SCALARS, as well as any other registers and slots that contribute to4263 * SCALARS, as well as any other registers and slots that contribute to
4192 * a tracked state of given registers/stack slots, depending on specific BPF4264 * a tracked state of given registers/stack slots, depending on specific BPF
4193 * assembly instructions (see backtrack_insns() for exact instruction handling4265 * assembly instructions (see backtrack_insns() for exact instruction handling
4194 * logic). This backtracking relies on recorded jmp_history and is able to4266 * logic). This backtracking relies on recorded insn_hist and is able to
4195 * traverse entire chain of parent states. This process ends only when all the4267 * traverse entire chain of parent states. This process ends only when all the
4196 * necessary registers/slots and their transitive dependencies are marked as4268 * necessary registers/slots and their transitive dependencies are marked as
4197 * precise.4269 * precise.
@@ -4308,8 +4380,9 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 110 unmodified lines
4308 4380
4309 for (;;) {4381 for (;;) {
4310 DECLARE_BITMAP(mask, 64);4382 DECLARE_BITMAP(mask, 64);
4311 u32 history = st->jmp_history_cnt;4383 u32 hist_start = st->insn_hist_start;
4312 struct bpf_jmp_history_entry *hist;4384 u32 hist_end = st->insn_hist_end;
4385 struct bpf_insn_hist_entry *hist;
4313 4386
4314 if (env->log.level & BPF_LOG_LEVEL2) {4387 if (env->log.level & BPF_LOG_LEVEL2) {
4315 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",4388 verbose(env, "mark_precise: frame%d: last_idx %d first_idx %d subseq_idx %d \n",
@@ -4348,7 +4421,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 32 unmodified lines
4348 err = 0;4421 err = 0;
4349 skip_first = false;4422 skip_first = false;
4350 } else {4423 } else {
4351 hist = get_jmp_hist_entry(st, history, i);4424 hist = get_insn_hist_entry(env, hist_start, hist_end, i);
4352 err = backtrack_insn(env, i, subseq_idx, hist, bt);4425 err = backtrack_insn(env, i, subseq_idx, hist, bt);
4353 }4426 }
4354 if (err == -ENOTSUPP) {4427 if (err == -ENOTSUPP) {
@@ -4365,7 +4438,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno) · 10 unmodified lines
4365 */4438 */
4366 return 0;4439 return 0;
4367 subseq_idx = i;4440 subseq_idx = i;
4368 i = get_prev_insn_idx(st, i, &history);4441 i = get_prev_insn_idx(env, st, i, hist_start, &hist_end);
4369 if (i == -ENOENT)4442 if (i == -ENOENT)
4370 break;4443 break;
4371 if (i >= env->prog->len) {4444 if (i >= env->prog->len) {
@@ -4731,7 +4804,7 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env, · 359 unmodified lines
4731 }4804 }
4732 4805
4733 if (insn_flags)4806 if (insn_flags)
4734 return push_jmp_history(env, env->cur_state, insn_flags, 0);4807 return push_insn_history(env, env->cur_state, insn_flags, 0);
4735 return 0;4808 return 0;
4736}4809}
4737 4810
@@ -5038,7 +5111,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env, · 300 unmodified lines
5038 insn_flags = 0; /* we are not restoring spilled register */5111 insn_flags = 0; /* we are not restoring spilled register */
5039 }5112 }
5040 if (insn_flags)5113 if (insn_flags)
5041 return push_jmp_history(env, env->cur_state, insn_flags, 0);5114 return push_insn_history(env, env->cur_state, insn_flags, 0);
5042 return 0;5115 return 0;
5043}5116}
5044 5117
@@ -5423,7 +5496,7 @@ static bool in_sleepable(struct bpf_verifier_env *env) · 378 unmodified lines
5423static bool in_rcu_cs(struct bpf_verifier_env *env)5496static bool in_rcu_cs(struct bpf_verifier_env *env)
5424{5497{
5425 return env->cur_state->active_rcu_lock ||5498 return env->cur_state->active_rcu_lock ||
5426 env->cur_state->active_lock.ptr ||5499 cur_func(env)->active_locks ||
5427 !in_sleepable(env);5500 !in_sleepable(env);
5428}5501}
5429 5502
@@ -5491,6 +5564,22 @@ static u32 btf_ld_kptr_type(struct bpf_verifier_env *env, struct btf_field *kptr · 61 unmodified lines
5491 return ret;5564 return ret;
5492}5565}
5493 5566
5567static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,
5568 struct btf_field *field)
5569{
5570 struct bpf_reg_state *reg;
5571 const struct btf_type *t;
5572
5573 t = btf_type_by_id(field->kptr.btf, field->kptr.btf_id);
5574 mark_reg_known_zero(env, cur_regs(env), regno);
5575 reg = reg_state(env, regno);
5576 reg->type = PTR_TO_MEM | PTR_MAYBE_NULL;
5577 reg->mem_size = t->size;
5578 reg->id = ++env->id_gen;
5579
5580 return 0;
5581}
5582
5494static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,5583static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
5495 int value_regno, int insn_idx,5584 int value_regno, int insn_idx,
5496 struct btf_field *kptr_field)5585 struct btf_field *kptr_field)
@@ -5519,9 +5608,15 @@ static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno, · 22 unmodified lines
5519 verbose(env, "store to referenced kptr disallowed\n");5608 verbose(env, "store to referenced kptr disallowed\n");
5520 return -EACCES;5609 return -EACCES;
5521 }5610 }
5611 if (class != BPF_LDX && kptr_field->type == BPF_UPTR) {
5612 verbose(env, "store to uptr disallowed\n");
5613 return -EACCES;
5614 }
5522 5615
5523 if (class == BPF_LDX) {5616 if (class == BPF_LDX) {
5524 val_reg = reg_state(env, value_regno);5617 if (kptr_field->type == BPF_UPTR)
5618 return mark_uptr_ld_reg(env, value_regno, kptr_field);
5619
5525 /* We can simply mark the value_regno receiving the pointer5620 /* We can simply mark the value_regno receiving the pointer
5526 * value from map as PTR_TO_BTF_ID, with the correct type.5621 * value from map as PTR_TO_BTF_ID, with the correct type.
5527 */5622 */
@@ -5579,21 +5674,26 @@ static int check_map_access(struct bpf_verifier_env *env, u32 regno, · 51 unmodified lines
5579 case BPF_KPTR_UNREF:5674 case BPF_KPTR_UNREF:
5580 case BPF_KPTR_REF:5675 case BPF_KPTR_REF:
5581 case BPF_KPTR_PERCPU:5676 case BPF_KPTR_PERCPU:
5677 case BPF_UPTR:
5582 if (src != ACCESS_DIRECT) {5678 if (src != ACCESS_DIRECT) {
5583 verbose(env, "kptr cannot be accessed indirectly by helper\n");5679 verbose(env, "%s cannot be accessed indirectly by helper\n",
5680 btf_field_type_name(field->type));
5584 return -EACCES;5681 return -EACCES;
5585 }5682 }
5586 if (!tnum_is_const(reg->var_off)) {5683 if (!tnum_is_const(reg->var_off)) {
5587 verbose(env, "kptr access cannot have variable offset\n");5684 verbose(env, "%s access cannot have variable offset\n",
5685 btf_field_type_name(field->type));
5588 return -EACCES;5686 return -EACCES;
5589 }5687 }
5590 if (p != off + reg->var_off.value) {5688 if (p != off + reg->var_off.value) {
5591 verbose(env, "kptr access misaligned expected=%u off=%llu\n",5689 verbose(env, "%s access misaligned expected=%u off=%llu\n",
5690 btf_field_type_name(field->type),
5592 p, off + reg->var_off.value);5691 p, off + reg->var_off.value);
5593 return -EACCES;5692 return -EACCES;
5594 }5693 }
5595 if (size != bpf_size_to_bytes(BPF_DW)) {5694 if (size != bpf_size_to_bytes(BPF_DW)) {
5596 verbose(env, "kptr access size must be BPF_DW\n");5695 verbose(env, "%s access size must be BPF_DW\n",
5696 btf_field_type_name(field->type));
5597 return -EACCES;5697 return -EACCES;
5598 }5698 }
5599 break;5699 break;
@@ -5988,6 +6088,34 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, · 388 unmodified lines
5988 strict);6088 strict);
5989}6089}
5990 6090
6091static enum priv_stack_mode bpf_enable_priv_stack(struct bpf_prog *prog)
6092{
6093 if (!bpf_jit_supports_private_stack())
6094 return NO_PRIV_STACK;
6095
6096 /* bpf_prog_check_recur() checks all prog types that use bpf trampoline
6097 * while kprobe/tp/perf_event/raw_tp don't use trampoline hence checked
6098 * explicitly.
6099 */
6100 switch (prog->type) {
6101 case BPF_PROG_TYPE_KPROBE:
6102 case BPF_PROG_TYPE_TRACEPOINT:
6103 case BPF_PROG_TYPE_PERF_EVENT:
6104 case BPF_PROG_TYPE_RAW_TRACEPOINT:
6105 return PRIV_STACK_ADAPTIVE;
6106 case BPF_PROG_TYPE_TRACING:
6107 case BPF_PROG_TYPE_LSM:
6108 case BPF_PROG_TYPE_STRUCT_OPS:
6109 if (prog->aux->priv_stack_requested || bpf_prog_check_recur(prog))
6110 return PRIV_STACK_ADAPTIVE;
6111 fallthrough;
6112 default:
6113 break;
6114 }
6115
6116 return NO_PRIV_STACK;
6117}
6118
5991static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)6119static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth)
5992{6120{
5993 if (env->prog->jit_requested)6121 if (env->prog->jit_requested)
@@ -6005,17 +6133,20 @@ static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth) · 11 unmodified lines
6005 * Since recursion is prevented by check_cfg() this algorithm6133 * Since recursion is prevented by check_cfg() this algorithm
6006 * only needs a local stack of MAX_CALL_FRAMES to remember callsites6134 * only needs a local stack of MAX_CALL_FRAMES to remember callsites
6007 */6135 */
6008static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx)6136static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
6137 bool priv_stack_supported)
6009{6138{
6010 struct bpf_subprog_info *subprog = env->subprog_info;6139 struct bpf_subprog_info *subprog = env->subprog_info;
6011 struct bpf_insn *insn = env->prog->insnsi;6140 struct bpf_insn *insn = env->prog->insnsi;
6012 int depth = 0, frame = 0, i, subprog_end;6141 int depth = 0, frame = 0, i, subprog_end, subprog_depth;
6013 bool tail_call_reachable = false;6142 bool tail_call_reachable = false;
6014 int ret_insn[MAX_CALL_FRAMES];6143 int ret_insn[MAX_CALL_FRAMES];
6015 int ret_prog[MAX_CALL_FRAMES];6144 int ret_prog[MAX_CALL_FRAMES];
6016 int j;6145 int j;
6017 6146
6018 i = subprog[idx].start;6147 i = subprog[idx].start;
6148 if (!priv_stack_supported)
6149 subprog[idx].priv_stack_mode = NO_PRIV_STACK;
6019process_func:6150process_func:
6020 /* protect against potential stack overflow that might happen when6151 /* protect against potential stack overflow that might happen when
6021 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack6152 * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
@@ -6042,11 +6173,31 @@ process_func: · 20 unmodified lines
6042 depth);6173 depth);
6043 return -EACCES;6174 return -EACCES;
6044 }6175 }
6045 depth += round_up_stack_depth(env, subprog[idx].stack_depth);6176
6046 if (depth > MAX_BPF_STACK) {6177 subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);
6047 verbose(env, "combined stack size of %d calls is %d. Too large\n",6178 if (priv_stack_supported) {
6048 frame + 1, depth);6179 /* Request private stack support only if the subprog stack
6049 return -EACCES;6180 * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to
6181 * avoid jit penalty if the stack usage is small.
6182 */
6183 if (subprog[idx].priv_stack_mode == PRIV_STACK_UNKNOWN &&
6184 subprog_depth >= BPF_PRIV_STACK_MIN_SIZE)
6185 subprog[idx].priv_stack_mode = PRIV_STACK_ADAPTIVE;
6186 }
6187
6188 if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
6189 if (subprog_depth > MAX_BPF_STACK) {
6190 verbose(env, "stack size of subprog %d is %d. Too large\n",
6191 idx, subprog_depth);
6192 return -EACCES;
6193 }
6194 } else {
6195 depth += subprog_depth;
6196 if (depth > MAX_BPF_STACK) {
6197 verbose(env, "combined stack size of %d calls is %d. Too large\n",
6198 frame + 1, depth);
6199 return -EACCES;
6200 }
6050 }6201 }
6051continue_func:6202continue_func:
6052 subprog_end = subprog[idx + 1].start;6203 subprog_end = subprog[idx + 1].start;
@@ -6103,6 +6254,8 @@ continue_func: · 50 unmodified lines
6103 }6254 }
6104 i = next_insn;6255 i = next_insn;
6105 idx = sidx;6256 idx = sidx;
6257 if (!priv_stack_supported)
6258 subprog[idx].priv_stack_mode = NO_PRIV_STACK;
6106 6259
6107 if (subprog[idx].has_tail_call)6260 if (subprog[idx].has_tail_call)
6108 tail_call_reachable = true;6261 tail_call_reachable = true;
@@ -6136,7 +6289,8 @@ continue_func: · 27 unmodified lines
6136 */6289 */
6137 if (frame == 0)6290 if (frame == 0)
6138 return 0;6291 return 0;
6139 depth -= round_up_stack_depth(env, subprog[idx].stack_depth);6292 if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE)
6293 depth -= round_up_stack_depth(env, subprog[idx].stack_depth);
6140 frame--;6294 frame--;
6141 i = ret_insn[frame];6295 i = ret_insn[frame];
6142 idx = ret_prog[frame];6296 idx = ret_prog[frame];
@@ -6145,17 +6299,45 @@ continue_func: · 2 unmodified lines
6145 6299
6146static int check_max_stack_depth(struct bpf_verifier_env *env)6300static int check_max_stack_depth(struct bpf_verifier_env *env)
6147{6301{
6302 enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN;
6148 struct bpf_subprog_info *si = env->subprog_info;6303 struct bpf_subprog_info *si = env->subprog_info;
6304 bool priv_stack_supported;
6149 int ret;6305 int ret;
6150 6306
6151 for (int i = 0; i < env->subprog_cnt; i++) {6307 for (int i = 0; i < env->subprog_cnt; i++) {
6308 if (si[i].has_tail_call) {
6309 priv_stack_mode = NO_PRIV_STACK;
6310 break;
6311 }
6312 }
6313
6314 if (priv_stack_mode == PRIV_STACK_UNKNOWN)
6315 priv_stack_mode = bpf_enable_priv_stack(env->prog);
6316
6317 /* All async_cb subprogs use normal kernel stack. If a particular
6318 * subprog appears in both main prog and async_cb subtree, that
6319 * subprog will use normal kernel stack to avoid potential nesting.
6320 * The reverse subprog traversal ensures when main prog subtree is
6321 * checked, the subprogs appearing in async_cb subtrees are already
6322 * marked as using normal kernel stack, so stack size checking can
6323 * be done properly.
6324 */
6325 for (int i = env->subprog_cnt - 1; i >= 0; i--) {
6152 if (!i || si[i].is_async_cb) {6326 if (!i || si[i].is_async_cb) {
6153 ret = check_max_stack_depth_subprog(env, i);6327 priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE;
6328 ret = check_max_stack_depth_subprog(env, i, priv_stack_supported);
6154 if (ret < 0)6329 if (ret < 0)
6155 return ret;6330 return ret;
6156 }6331 }
6157 continue;
6158 }6332 }
6333
6334 for (int i = 0; i < env->subprog_cnt; i++) {
6335 if (si[i].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
6336 env->prog->aux->jits_use_priv_stack = true;
6337 break;
6338 }
6339 }
6340
6159 return 0;6341 return 0;
6160}6342}
6161 6343
@@ -6595,6 +6777,7 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 433 unmodified lines
6595 const char *field_name = NULL;6777 const char *field_name = NULL;
6596 enum bpf_type_flag flag = 0;6778 enum bpf_type_flag flag = 0;
6597 u32 btf_id = 0;6779 u32 btf_id = 0;
6780 bool mask;
6598 int ret;6781 int ret;
6599 6782
6600 if (!env->allow_ptr_leaks) {6783 if (!env->allow_ptr_leaks) {
@@ -6666,7 +6849,21 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 65 unmodified lines
6666 6849
6667 if (ret < 0)6850 if (ret < 0)
6668 return ret;6851 return ret;
6669 6852 /* For raw_tp progs, we allow dereference of PTR_MAYBE_NULL
6853 * trusted PTR_TO_BTF_ID, these are the ones that are possibly
6854 * arguments to the raw_tp. Since internal checks in for trusted
6855 * reg in check_ptr_to_btf_access would consider PTR_MAYBE_NULL
6856 * modifier as problematic, mask it out temporarily for the
6857 * check. Don't apply this to pointers with ref_obj_id > 0, as
6858 * those won't be raw_tp args.
6859 *
6860 * We may end up applying this relaxation to other trusted
6861 * PTR_TO_BTF_ID with maybe null flag, since we cannot
6862 * distinguish PTR_MAYBE_NULL tagged for arguments vs normal
6863 * tagging, but that should expand allowed behavior, and not
6864 * cause regression for existing behavior.
6865 */
6866 mask = mask_raw_tp_reg(env, reg);
6670 if (ret != PTR_TO_BTF_ID) {6867 if (ret != PTR_TO_BTF_ID) {
6671 /* just mark; */6868 /* just mark; */
6672 6869
@@ -6727,8 +6924,13 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 54 unmodified lines
6727 clear_trusted_flags(&flag);6924 clear_trusted_flags(&flag);
6728 }6925 }
6729 6926
6730 if (atype == BPF_READ && value_regno >= 0)6927 if (atype == BPF_READ && value_regno >= 0) {
6731 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);6928 mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
6929 /* We've assigned a new type to regno, so don't undo masking. */
6930 if (regno == value_regno)
6931 mask = false;
6932 }
6933 unmask_raw_tp_reg(reg, mask);
6732 6934
6733 return 0;6935 return 0;
6734}6936}
@@ -6949,7 +7151,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 214 unmodified lines
6949 return err;7151 return err;
6950 if (tnum_is_const(reg->var_off))7152 if (tnum_is_const(reg->var_off))
6951 kptr_field = btf_record_find(reg->map_ptr->record,7153 kptr_field = btf_record_find(reg->map_ptr->record,
6952 off + reg->var_off.value, BPF_KPTR);7154 off + reg->var_off.value, BPF_KPTR | BPF_UPTR);
6953 if (kptr_field) {7155 if (kptr_field) {
6954 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);7156 err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);
6955 } else if (t == BPF_READ && value_regno >= 0) {7157 } else if (t == BPF_READ && value_regno >= 0) {
@@ -7103,7 +7305,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 147 unmodified lines
7103 if (!err && t == BPF_READ && value_regno >= 0)7305 if (!err && t == BPF_READ && value_regno >= 0)
7104 mark_reg_unknown(env, regs, value_regno);7306 mark_reg_unknown(env, regs, value_regno);
7105 } else if (base_type(reg->type) == PTR_TO_BTF_ID &&7307 } else if (base_type(reg->type) == PTR_TO_BTF_ID &&
7106 !type_may_be_null(reg->type)) {7308 (mask_raw_tp_reg_cond(env, reg) || !type_may_be_null(reg->type))) {
7107 err = check_ptr_to_btf_access(env, regs, regno, off, size, t,7309 err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
7108 value_regno);7310 value_regno);
7109 } else if (reg->type == CONST_PTR_TO_MAP) {7311 } else if (reg->type == CONST_PTR_TO_MAP) {
@@ -7648,19 +7850,20 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg · 538 unmodified lines
7648 * Since only one bpf_spin_lock is allowed the checks are simpler than7850 * Since only one bpf_spin_lock is allowed the checks are simpler than
7649 * reg_is_refcounted() logic. The verifier needs to remember only7851 * reg_is_refcounted() logic. The verifier needs to remember only
7650 * one spin_lock instead of array of acquired_refs.7852 * one spin_lock instead of array of acquired_refs.
7651 * cur_state->active_lock remembers which map value element or allocated7853 * cur_func(env)->active_locks remembers which map value element or allocated
7652 * object got locked and clears it after bpf_spin_unlock.7854 * object got locked and clears it after bpf_spin_unlock.
7653 */7855 */
7654static int process_spin_lock(struct bpf_verifier_env *env, int regno,7856static int process_spin_lock(struct bpf_verifier_env *env, int regno,
7655 bool is_lock)7857 bool is_lock)
7656{7858{
7657 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];7859 struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
7658 struct bpf_verifier_state *cur = env->cur_state;
7659 bool is_const = tnum_is_const(reg->var_off);7860 bool is_const = tnum_is_const(reg->var_off);
7861 struct bpf_func_state *cur = cur_func(env);
7660 u64 val = reg->var_off.value;7862 u64 val = reg->var_off.value;
7661 struct bpf_map *map = NULL;7863 struct bpf_map *map = NULL;
7662 struct btf *btf = NULL;7864 struct btf *btf = NULL;
7663 struct btf_record *rec;7865 struct btf_record *rec;
7866 int err;
7664 7867
7665 if (!is_const) {7868 if (!is_const) {
7666 verbose(env,7869 verbose(env,
@@ -7692,16 +7895,23 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 25 unmodified lines
7692 return -EINVAL;7895 return -EINVAL;
7693 }7896 }
7694 if (is_lock) {7897 if (is_lock) {
7695 if (cur->active_lock.ptr) {7898 void *ptr;
7899
7900 if (map)
7901 ptr = map;
7902 else
7903 ptr = btf;
7904
7905 if (cur->active_locks) {
7696 verbose(env,7906 verbose(env,
7697 "Locking two bpf_spin_locks are not allowed\n");7907 "Locking two bpf_spin_locks are not allowed\n");
7698 return -EINVAL;7908 return -EINVAL;
7699 }7909 }
7700 if (map)7910 err = acquire_lock_state(env, env->insn_idx, REF_TYPE_LOCK, reg->id, ptr);
7701 cur->active_lock.ptr = map;7911 if (err < 0) {
7702 else7912 verbose(env, "Failed to acquire lock state\n");
7703 cur->active_lock.ptr = btf;7913 return err;
7704 cur->active_lock.id = reg->id;7914 }
7705 } else {7915 } else {
7706 void *ptr;7916 void *ptr;
7707 7917
@@ -7710,20 +7920,17 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, · 2 unmodified lines
7710 else7920 else
7711 ptr = btf;7921 ptr = btf;
7712 7922
7713 if (!cur->active_lock.ptr) {7923 if (!cur->active_locks) {
7714 verbose(env, "bpf_spin_unlock without taking a lock\n");7924 verbose(env, "bpf_spin_unlock without taking a lock\n");
7715 return -EINVAL;7925 return -EINVAL;
7716 }7926 }
7717 if (cur->active_lock.ptr != ptr ||7927
7718 cur->active_lock.id != reg->id) {7928 if (release_lock_state(cur_func(env), REF_TYPE_LOCK, reg->id, ptr)) {
7719 verbose(env, "bpf_spin_unlock of different lock\n");7929 verbose(env, "bpf_spin_unlock of different lock\n");
7720 return -EINVAL;7930 return -EINVAL;
7721 }7931 }
7722 7932
7723 invalidate_non_owning_refs(env);7933 invalidate_non_owning_refs(env);
7724
7725 cur->active_lock.ptr = NULL;
7726 cur->active_lock.id = 0;
7727 }7934 }
7728 return 0;7935 return 0;
7729}7936}
@@ -8796,6 +9003,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 1066 unmodified lines
8796 enum bpf_reg_type type = reg->type;9003 enum bpf_reg_type type = reg->type;
8797 u32 *arg_btf_id = NULL;9004 u32 *arg_btf_id = NULL;
8798 int err = 0;9005 int err = 0;
9006 bool mask;
8799 9007
8800 if (arg_type == ARG_DONTCARE)9008 if (arg_type == ARG_DONTCARE)
8801 return 0;9009 return 0;
@@ -8836,11 +9044,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 34 unmodified lines
8836 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)9044 base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
8837 arg_btf_id = fn->arg_btf_id[arg];9045 arg_btf_id = fn->arg_btf_id[arg];
8838 9046
9047 mask = mask_raw_tp_reg(env, reg);
8839 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);9048 err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
8840 if (err)
8841 return err;
8842 9049
8843 err = check_func_arg_reg_off(env, reg, regno, arg_type);9050 err = err ?: check_func_arg_reg_off(env, reg, regno, arg_type);
9051 unmask_raw_tp_reg(reg, mask);
8844 if (err)9052 if (err)
8845 return err;9053 return err;
8846 9054
@@ -9635,14 +9843,17 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, · 788 unmodified lines
9635 return ret;9843 return ret;
9636 } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {9844 } else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
9637 struct bpf_call_arg_meta meta;9845 struct bpf_call_arg_meta meta;
9846 bool mask;
9638 int err;9847 int err;
9639 9848
9640 if (register_is_null(reg) && type_may_be_null(arg->arg_type))9849 if (register_is_null(reg) && type_may_be_null(arg->arg_type))
9641 continue;9850 continue;
9642 9851
9643 memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */9852 memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
9853 mask = mask_raw_tp_reg(env, reg);
9644 err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);9854 err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);
9645 err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);9855 err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);
9856 unmask_raw_tp_reg(reg, mask);
9646 if (err)9857 if (err)
9647 return err;9858 return err;
9648 } else {9859 } else {
@@ -9781,7 +9992,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 132 unmodified lines
9781 const char *sub_name = subprog_name(env, subprog);9992 const char *sub_name = subprog_name(env, subprog);
9782 9993
9783 /* Only global subprogs cannot be called with a lock held. */9994 /* Only global subprogs cannot be called with a lock held. */
9784 if (env->cur_state->active_lock.ptr) {9995 if (cur_func(env)->active_locks) {
9785 verbose(env, "global function calls are not allowed while holding a lock,\n"9996 verbose(env, "global function calls are not allowed while holding a lock,\n"
9786 "use static function instead\n");9997 "use static function instead\n");
9787 return -EINVAL;9998 return -EINVAL;
@@ -9910,7 +10121,7 @@ static int set_loop_callback_state(struct bpf_verifier_env *env, · 122 unmodified lines
9910{10121{
9911 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,10122 /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
9912 * u64 flags);10123 * u64 flags);
9913 * callback_fn(u32 index, void *callback_ctx);10124 * callback_fn(u64 index, void *callback_ctx);
9914 */10125 */
9915 callee->regs[BPF_REG_1].type = SCALAR_VALUE;10126 callee->regs[BPF_REG_1].type = SCALAR_VALUE;
9916 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];10127 callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
@@ -10122,17 +10333,10 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) · 205 unmodified lines
10122 caller->regs[BPF_REG_0] = *r0;10333 caller->regs[BPF_REG_0] = *r0;
10123 }10334 }
10124 10335
10125 /* callback_fn frame should have released its own additions to parent's10336 /* Transfer references to the caller */
10126 * reference state at this point, or check_reference_leak would10337 err = copy_reference_state(caller, callee);
10127 * complain, hence it must be the same as the caller. There is no need10338 if (err)
10128 * to copy it back.10339 return err;
10129 */
10130 if (!callee->in_callback_fn) {
10131 /* Transfer references to the caller */
10132 err = copy_reference_state(caller, callee);
10133 if (err)
10134 return err;
10135 }
10136 10340
10137 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,10341 /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite,
10138 * there function call logic would reschedule callback visit. If iteration10342 * there function call logic would reschedule callback visit. If iteration
@@ -10302,11 +10506,11 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi · 163 unmodified lines
10302 bool refs_lingering = false;10506 bool refs_lingering = false;
10303 int i;10507 int i;
10304 10508
10305 if (!exception_exit && state->frameno && !state->in_callback_fn)10509 if (!exception_exit && state->frameno)
10306 return 0;10510 return 0;
10307 10511
10308 for (i = 0; i < state->acquired_refs; i++) {10512 for (i = 0; i < state->acquired_refs; i++) {
10309 if (!exception_exit && state->in_callback_fn && state->refs[i].callback_ref != state->frameno)10513 if (state->refs[i].type != REF_TYPE_PTR)
10310 continue;10514 continue;
10311 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",10515 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
10312 state->refs[i].id, state->refs[i].insn_idx);10516 state->refs[i].id, state->refs[i].insn_idx);
@@ -10315,6 +10519,34 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi · 2 unmodified lines
10315 return refs_lingering ? -EINVAL : 0;10519 return refs_lingering ? -EINVAL : 0;
10316}10520}
10317 10521
10522static int check_resource_leak(struct bpf_verifier_env *env, bool exception_exit, bool check_lock, const char *prefix)
10523{
10524 int err;
10525
10526 if (check_lock && cur_func(env)->active_locks) {
10527 verbose(env, "%s cannot be used inside bpf_spin_lock-ed region\n", prefix);
10528 return -EINVAL;
10529 }
10530
10531 err = check_reference_leak(env, exception_exit);
10532 if (err) {
10533 verbose(env, "%s would lead to reference leak\n", prefix);
10534 return err;
10535 }
10536
10537 if (check_lock && env->cur_state->active_rcu_lock) {
10538 verbose(env, "%s cannot be used inside bpf_rcu_read_lock-ed region\n", prefix);
10539 return -EINVAL;
10540 }
10541
10542 if (check_lock && env->cur_state->active_preempt_lock) {
10543 verbose(env, "%s cannot be used inside bpf_preempt_disable-ed region\n", prefix);
10544 return -EINVAL;
10545 }
10546
10547 return 0;
10548}
10549
10318static int check_bpf_snprintf_call(struct bpf_verifier_env *env,10550static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
10319 struct bpf_reg_state *regs)10551 struct bpf_reg_state *regs)
10320{10552{
@@ -10583,11 +10815,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn · 262 unmodified lines
10583 10815
10584 switch (func_id) {10816 switch (func_id) {
10585 case BPF_FUNC_tail_call:10817 case BPF_FUNC_tail_call:
10586 err = check_reference_leak(env, false);10818 err = check_resource_leak(env, false, true, "tail_call");
10587 if (err) {10819 if (err)
10588 verbose(env, "tail_call would lead to reference leak\n");
10589 return err;10820 return err;
10590 }
10591 break;10821 break;
10592 case BPF_FUNC_get_local_storage:10822 case BPF_FUNC_get_local_storage:
10593 /* check that flags argument in get_local_storage(map, flags) is 0,10823 /* check that flags argument in get_local_storage(map, flags) is 0,
@@ -11252,6 +11482,7 @@ enum special_kfunc_type { · 658 unmodified lines
11252 KF_bpf_preempt_enable,11482 KF_bpf_preempt_enable,
11253 KF_bpf_iter_css_task_new,11483 KF_bpf_iter_css_task_new,
11254 KF_bpf_session_cookie,11484 KF_bpf_session_cookie,
11485 KF_bpf_get_kmem_cache,
11255};11486};
11256 11487
11257BTF_SET_START(special_kfunc_set)11488BTF_SET_START(special_kfunc_set)
@@ -11317,6 +11548,7 @@ BTF_ID(func, bpf_session_cookie) · 59 unmodified lines
11317#else11548#else
11318BTF_ID_UNUSED11549BTF_ID_UNUSED
11319#endif11550#endif
11551BTF_ID(func, bpf_get_kmem_cache)
11320 11552
11321static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)11553static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
11322{11554{
@@ -11512,10 +11744,9 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env, · 189 unmodified lines
11512 11744
11513static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)11745static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
11514{11746{
11515 struct bpf_verifier_state *state = env->cur_state;
11516 struct btf_record *rec = reg_btf_record(reg);11747 struct btf_record *rec = reg_btf_record(reg);
11517 11748
11518 if (!state->active_lock.ptr) {11749 if (!cur_func(env)->active_locks) {
11519 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");11750 verbose(env, "verifier internal error: ref_set_non_owning w/o active lock\n");
11520 return -EFAULT;11751 return -EFAULT;
11521 }11752 }
@@ -11612,6 +11843,7 @@ static int ref_convert_owning_non_owning(struct bpf_verifier_env *env, u32 ref_o · 90 unmodified lines
11612 */11843 */
11613static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)11844static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
11614{11845{
11846 struct bpf_reference_state *s;
11615 void *ptr;11847 void *ptr;
11616 u32 id;11848 u32 id;
11617 11849
@@ -11628,10 +11860,10 @@ static int check_reg_allocation_locked(struct bpf_verifier_env *env, struct bpf_ · 10 unmodified lines
11628 }11860 }
11629 id = reg->id;11861 id = reg->id;
11630 11862
11631 if (!env->cur_state->active_lock.ptr)11863 if (!cur_func(env)->active_locks)
11632 return -EINVAL;11864 return -EINVAL;
11633 if (env->cur_state->active_lock.ptr != ptr ||11865 s = find_lock_state(env, REF_TYPE_LOCK, id, ptr);
11634 env->cur_state->active_lock.id != id) {11866 if (!s) {
11635 verbose(env, "held lock and object are not in the same allocation\n");11867 verbose(env, "held lock and object are not in the same allocation\n");
11636 return -EINVAL;11868 return -EINVAL;
11637 }11869 }
@@ -11942,6 +12174,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 304 unmodified lines
11942 enum bpf_arg_type arg_type = ARG_DONTCARE;12174 enum bpf_arg_type arg_type = ARG_DONTCARE;
11943 u32 regno = i + 1, ref_id, type_size;12175 u32 regno = i + 1, ref_id, type_size;
11944 bool is_ret_buf_sz = false;12176 bool is_ret_buf_sz = false;
12177 bool mask = false;
11945 int kf_arg_type;12178 int kf_arg_type;
11946 12179
11947 t = btf_type_skip_modifiers(btf, args[i].type, NULL);12180 t = btf_type_skip_modifiers(btf, args[i].type, NULL);
@@ -12000,12 +12233,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 52 unmodified lines
12000 return -EINVAL;12233 return -EINVAL;
12001 }12234 }
12002 12235
12236 mask = mask_raw_tp_reg(env, reg);
12003 if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&12237 if ((is_kfunc_trusted_args(meta) || is_kfunc_rcu(meta)) &&
12004 (register_is_null(reg) || type_may_be_null(reg->type)) &&12238 (register_is_null(reg) || type_may_be_null(reg->type)) &&
12005 !is_kfunc_arg_nullable(meta->btf, &args[i])) {12239 !is_kfunc_arg_nullable(meta->btf, &args[i])) {
12006 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);12240 verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
12241 unmask_raw_tp_reg(reg, mask);
12007 return -EACCES;12242 return -EACCES;
12008 }12243 }
12244 unmask_raw_tp_reg(reg, mask);
12009 12245
12010 if (reg->ref_obj_id) {12246 if (reg->ref_obj_id) {
12011 if (is_kfunc_release(meta) && meta->ref_obj_id) {12247 if (is_kfunc_release(meta) && meta->ref_obj_id) {
@@ -12063,16 +12299,24 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 51 unmodified lines
12063 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))12299 if (!is_kfunc_trusted_args(meta) && !is_kfunc_rcu(meta))
12064 break;12300 break;
12065 12301
12302 /* Allow passing maybe NULL raw_tp arguments to
12303 * kfuncs for compatibility. Don't apply this to
12304 * arguments with ref_obj_id > 0.
12305 */
12306 mask = mask_raw_tp_reg(env, reg);
12066 if (!is_trusted_reg(reg)) {12307 if (!is_trusted_reg(reg)) {
12067 if (!is_kfunc_rcu(meta)) {12308 if (!is_kfunc_rcu(meta)) {
12068 verbose(env, "R%d must be referenced or trusted\n", regno);12309 verbose(env, "R%d must be referenced or trusted\n", regno);
12310 unmask_raw_tp_reg(reg, mask);
12069 return -EINVAL;12311 return -EINVAL;
12070 }12312 }
12071 if (!is_rcu_reg(reg)) {12313 if (!is_rcu_reg(reg)) {
12072 verbose(env, "R%d must be a rcu pointer\n", regno);12314 verbose(env, "R%d must be a rcu pointer\n", regno);
12315 unmask_raw_tp_reg(reg, mask);
12073 return -EINVAL;12316 return -EINVAL;
12074 }12317 }
12075 }12318 }
12319 unmask_raw_tp_reg(reg, mask);
12076 fallthrough;12320 fallthrough;
12077 case KF_ARG_PTR_TO_CTX:12321 case KF_ARG_PTR_TO_CTX:
12078 case KF_ARG_PTR_TO_DYNPTR:12322 case KF_ARG_PTR_TO_DYNPTR:
@@ -12095,7 +12339,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 16 unmodified lines
12095 12339
12096 if (is_kfunc_release(meta) && reg->ref_obj_id)12340 if (is_kfunc_release(meta) && reg->ref_obj_id)
12097 arg_type |= OBJ_RELEASE;12341 arg_type |= OBJ_RELEASE;
12342 mask = mask_raw_tp_reg(env, reg);
12098 ret = check_func_arg_reg_off(env, reg, regno, arg_type);12343 ret = check_func_arg_reg_off(env, reg, regno, arg_type);
12344 unmask_raw_tp_reg(reg, mask);
12099 if (ret < 0)12345 if (ret < 0)
12100 return ret;12346 return ret;
12101 12347
@@ -12272,6 +12518,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 170 unmodified lines
12272 ref_tname = btf_name_by_offset(btf, ref_t->name_off);12518 ref_tname = btf_name_by_offset(btf, ref_t->name_off);
12273 fallthrough;12519 fallthrough;
12274 case KF_ARG_PTR_TO_BTF_ID:12520 case KF_ARG_PTR_TO_BTF_ID:
12521 mask = mask_raw_tp_reg(env, reg);
12275 /* Only base_type is checked, further checks are done here */12522 /* Only base_type is checked, further checks are done here */
12276 if ((base_type(reg->type) != PTR_TO_BTF_ID ||12523 if ((base_type(reg->type) != PTR_TO_BTF_ID ||
12277 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&12524 (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
@@ -12280,9 +12527,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 2 unmodified lines
12280 verbose(env, "expected %s or socket\n",12527 verbose(env, "expected %s or socket\n",
12281 reg_type_str(env, base_type(reg->type) |12528 reg_type_str(env, base_type(reg->type) |
12282 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));12529 (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
12530 unmask_raw_tp_reg(reg, mask);
12283 return -EINVAL;12531 return -EINVAL;
12284 }12532 }
12285 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);12533 ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
12534 unmask_raw_tp_reg(reg, mask);
12286 if (ret < 0)12535 if (ret < 0)
12287 return ret;12536 return ret;
12288 break;12537 break;
@@ -12827,6 +13076,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, · 538 unmodified lines
12827 regs[BPF_REG_0].type = PTR_TO_BTF_ID;13076 regs[BPF_REG_0].type = PTR_TO_BTF_ID;
12828 regs[BPF_REG_0].btf_id = ptr_type_id;13077 regs[BPF_REG_0].btf_id = ptr_type_id;
12829 13078
13079 if (meta.func_id == special_kfunc_list[KF_bpf_get_kmem_cache])
13080 regs[BPF_REG_0].type |= PTR_UNTRUSTED;
13081
12830 if (is_iter_next_kfunc(&meta)) {13082 if (is_iter_next_kfunc(&meta)) {
12831 struct bpf_reg_state *cur_iter;13083 struct bpf_reg_state *cur_iter;
12832 13084
@@ -13252,7 +13504,7 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env, · 419 unmodified lines
13252 */13504 */
13253static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,13505static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
13254 struct bpf_insn *insn,13506 struct bpf_insn *insn,
13255 const struct bpf_reg_state *ptr_reg,13507 struct bpf_reg_state *ptr_reg,
13256 const struct bpf_reg_state *off_reg)13508 const struct bpf_reg_state *off_reg)
13257{13509{
13258 struct bpf_verifier_state *vstate = env->cur_state;13510 struct bpf_verifier_state *vstate = env->cur_state;
@@ -13266,6 +13518,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 7 unmodified lines
13266 struct bpf_sanitize_info info = {};13518 struct bpf_sanitize_info info = {};
13267 u8 opcode = BPF_OP(insn->code);13519 u8 opcode = BPF_OP(insn->code);
13268 u32 dst = insn->dst_reg;13520 u32 dst = insn->dst_reg;
13521 bool mask;
13269 int ret;13522 int ret;
13270 13523
13271 dst_reg = &regs[dst];13524 dst_reg = &regs[dst];
@@ -13292,11 +13545,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 20 unmodified lines
13292 return -EACCES;13545 return -EACCES;
13293 }13546 }
13294 13547
13548 mask = mask_raw_tp_reg(env, ptr_reg);
13295 if (ptr_reg->type & PTR_MAYBE_NULL) {13549 if (ptr_reg->type & PTR_MAYBE_NULL) {
13296 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",13550 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
13297 dst, reg_type_str(env, ptr_reg->type));13551 dst, reg_type_str(env, ptr_reg->type));
13552 unmask_raw_tp_reg(ptr_reg, mask);
13298 return -EACCES;13553 return -EACCES;
13299 }13554 }
13555 unmask_raw_tp_reg(ptr_reg, mask);
13300 13556
13301 switch (base_type(ptr_reg->type)) {13557 switch (base_type(ptr_reg->type)) {
13302 case PTR_TO_CTX:13558 case PTR_TO_CTX:
@@ -15480,7 +15736,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 2177 unmodified lines
15480 if (dst_reg->type == SCALAR_VALUE && dst_reg->id)15736 if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
15481 collect_linked_regs(this_branch, dst_reg->id, &linked_regs);15737 collect_linked_regs(this_branch, dst_reg->id, &linked_regs);
15482 if (linked_regs.cnt > 1) {15738 if (linked_regs.cnt > 1) {
15483 err = push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs));15739 err = push_insn_history(env, this_branch, 0, linked_regs_pack(&linked_regs));
15484 if (err)15740 if (err)
15485 return err;15741 return err;
15486 }15742 }
@@ -15744,26 +16000,9 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) · 257 unmodified lines
15744 * gen_ld_abs() may terminate the program at runtime, leading to16000 * gen_ld_abs() may terminate the program at runtime, leading to
15745 * reference leak.16001 * reference leak.
15746 */16002 */
15747 err = check_reference_leak(env, false);16003 err = check_resource_leak(env, false, true, "BPF_LD_[ABS|IND]");
15748 if (err) {16004 if (err)
15749 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
15750 return err;16005 return err;
15751 }
15752
15753 if (env->cur_state->active_lock.ptr) {
15754 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
15755 return -EINVAL;
15756 }
15757
15758 if (env->cur_state->active_rcu_lock) {
15759 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_rcu_read_lock-ed region\n");
15760 return -EINVAL;
15761 }
15762
15763 if (env->cur_state->active_preempt_lock) {
15764 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_preempt_disable-ed region\n");
15765 return -EINVAL;
15766 }
15767 16006
15768 if (regs[ctx_reg].type != PTR_TO_CTX) {16007 if (regs[ctx_reg].type != PTR_TO_CTX) {
15769 verbose(env,16008 verbose(env,
@@ -15909,6 +16148,16 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 139 unmodified lines
15909 return -ENOTSUPP;16148 return -ENOTSUPP;
15910 }16149 }
15911 break;16150 break;
16151 case BPF_PROG_TYPE_KPROBE:
16152 switch (env->prog->expected_attach_type) {
16153 case BPF_TRACE_KPROBE_SESSION:
16154 case BPF_TRACE_UPROBE_SESSION:
16155 range = retval_range(0, 1);
16156 break;
16157 default:
16158 return 0;
16159 }
16160 break;
15912 case BPF_PROG_TYPE_SK_LOOKUP:16161 case BPF_PROG_TYPE_SK_LOOKUP:
15913 range = retval_range(SK_DROP, SK_PASS);16162 range = retval_range(SK_DROP, SK_PASS);
15914 break;16163 break;
@@ -16175,10 +16424,7 @@ static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta) · 260 unmodified lines
16175/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */16424/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */
16176static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)16425static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)
16177{16426{
16178 if (meta->btf == btf_vmlinux)16427 return meta->kfunc_flags & KF_FASTCALL;
16179 return meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
16180 meta->func_id == special_kfunc_list[KF_bpf_rdonly_cast];
16181 return false;
16182}16428}
16183 16429
16184/* LLVM define a bpf_fastcall function attribute.16430/* LLVM define a bpf_fastcall function attribute.
@@ -17513,8 +17759,20 @@ static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur, · 1328 unmodified lines
17513 return false;17759 return false;
17514 17760
17515 for (i = 0; i < old->acquired_refs; i++) {17761 for (i = 0; i < old->acquired_refs; i++) {
17516 if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap))17762 if (!check_ids(old->refs[i].id, cur->refs[i].id, idmap) ||
17763 old->refs[i].type != cur->refs[i].type)
17517 return false;17764 return false;
17765 switch (old->refs[i].type) {
17766 case REF_TYPE_PTR:
17767 break;
17768 case REF_TYPE_LOCK:
17769 if (old->refs[i].ptr != cur->refs[i].ptr)
17770 return false;
17771 break;
17772 default:
17773 WARN_ONCE(1, "Unhandled enum type for reference state: %d\n", old->refs[i].type);
17774 return false;
17775 }
17518 }17776 }
17519 17777
17520 return true;17778 return true;
@@ -17592,19 +17850,6 @@ static bool states_equal(struct bpf_verifier_env *env, · 71 unmodified lines
17592 if (old->speculative && !cur->speculative)17850 if (old->speculative && !cur->speculative)
17593 return false;17851 return false;
17594 17852
17595 if (old->active_lock.ptr != cur->active_lock.ptr)
17596 return false;
17597
17598 /* Old and cur active_lock's have to be either both present
17599 * or both absent.
17600 */
17601 if (!!old->active_lock.id != !!cur->active_lock.id)
17602 return false;
17603
17604 if (old->active_lock.id &&
17605 !check_ids(old->active_lock.id, cur->active_lock.id, &env->idmap_scratch))
17606 return false;
17607
17608 if (old->active_rcu_lock != cur->active_rcu_lock)17853 if (old->active_rcu_lock != cur->active_rcu_lock)
17609 return false;17854 return false;
17610 17855
@@ -17880,7 +18125,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 269 unmodified lines
17880 18125
17881 force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||18126 force_new_state = env->test_state_freq || is_force_checkpoint(env, insn_idx) ||
17882 /* Avoid accumulating infinitely long jmp history */18127 /* Avoid accumulating infinitely long jmp history */
17883 cur->jmp_history_cnt > 40;18128 cur->insn_hist_end - cur->insn_hist_start > 40;
17884 18129
17885 /* bpf progs typically have pruning point every 4 instructions18130 /* bpf progs typically have pruning point every 4 instructions
17886 * http://vger.kernel.org/bpfconf2019.html#session-118131 * http://vger.kernel.org/bpfconf2019.html#session-1
@@ -18078,7 +18323,7 @@ hit: · 191 unmodified lines
18078 * the current state.18323 * the current state.
18079 */18324 */
18080 if (is_jmp_point(env, env->insn_idx))18325 if (is_jmp_point(env, env->insn_idx))
18081 err = err ? : push_jmp_history(env, cur, 0, 0);18326 err = err ? : push_insn_history(env, cur, 0, 0);
18082 err = err ? : propagate_precision(env, &sl->state);18327 err = err ? : propagate_precision(env, &sl->state);
18083 if (err)18328 if (err)
18084 return err;18329 return err;
@@ -18177,8 +18422,8 @@ next: · 92 unmodified lines
18177 18422
18178 cur->parent = new;18423 cur->parent = new;
18179 cur->first_insn_idx = insn_idx;18424 cur->first_insn_idx = insn_idx;
18425 cur->insn_hist_start = cur->insn_hist_end;
18180 cur->dfs_depth = new->dfs_depth + 1;18426 cur->dfs_depth = new->dfs_depth + 1;
18181 clear_jmp_history(cur);
18182 new_sl->next = *explored_state(env, insn_idx);18427 new_sl->next = *explored_state(env, insn_idx);
18183 *explored_state(env, insn_idx) = new_sl;18428 *explored_state(env, insn_idx) = new_sl;
18184 /* connect new state to parentage chain. Current frame needs all18429 /* connect new state to parentage chain. Current frame needs all
@@ -18346,7 +18591,7 @@ static int do_check(struct bpf_verifier_env *env) · 161 unmodified lines
18346 }18591 }
18347 18592
18348 if (is_jmp_point(env, env->insn_idx)) {18593 if (is_jmp_point(env, env->insn_idx)) {
18349 err = push_jmp_history(env, state, 0, 0);18594 err = push_insn_history(env, state, 0, 0);
18350 if (err)18595 if (err)
18351 return err;18596 return err;
18352 }18597 }
@@ -18506,7 +18751,7 @@ static int do_check(struct bpf_verifier_env *env) · 153 unmodified lines
18506 return -EINVAL;18751 return -EINVAL;
18507 }18752 }
18508 18753
18509 if (env->cur_state->active_lock.ptr) {18754 if (cur_func(env)->active_locks) {
18510 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) ||18755 if ((insn->src_reg == BPF_REG_0 && insn->imm != BPF_FUNC_spin_unlock) ||
18511 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&18756 (insn->src_reg == BPF_PSEUDO_KFUNC_CALL &&
18512 (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) {18757 (insn->off != 0 || !is_bpf_graph_api_kfunc(insn->imm)))) {
@@ -18555,30 +18800,14 @@ static int do_check(struct bpf_verifier_env *env) · 42 unmodified lines
18555 return -EINVAL;18800 return -EINVAL;
18556 }18801 }
18557process_bpf_exit_full:18802process_bpf_exit_full:
18558 if (env->cur_state->active_lock.ptr && !env->cur_state->curframe) {
18559 verbose(env, "bpf_spin_unlock is missing\n");
18560 return -EINVAL;
18561 }
18562
18563 if (env->cur_state->active_rcu_lock && !env->cur_state->curframe) {
18564 verbose(env, "bpf_rcu_read_unlock is missing\n");
18565 return -EINVAL;
18566 }
18567
18568 if (env->cur_state->active_preempt_lock && !env->cur_state->curframe) {
18569 verbose(env, "%d bpf_preempt_enable%s missing\n",
18570 env->cur_state->active_preempt_lock,
18571 env->cur_state->active_preempt_lock == 1 ? " is" : "(s) are");
18572 return -EINVAL;
18573 }
18574
18575 /* We must do check_reference_leak here before18803 /* We must do check_reference_leak here before
18576 * prepare_func_exit to handle the case when18804 * prepare_func_exit to handle the case when
18577 * state->curframe > 0, it may be a callback18805 * state->curframe > 0, it may be a callback
18578 * function, for which reference_state must18806 * function, for which reference_state must
18579 * match caller reference state when it exits.18807 * match caller reference state when it exits.
18580 */18808 */
18581 err = check_reference_leak(env, exception_exit);18809 err = check_resource_leak(env, exception_exit, !env->cur_state->curframe,
18810 "BPF_EXIT instruction");
18582 if (err)18811 if (err)
18583 return err;18812 return err;
18584 18813
@@ -19837,6 +20066,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 1252 unmodified lines
19837 * for this case.20066 * for this case.
19838 */20067 */
19839 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:20068 case PTR_TO_BTF_ID | MEM_ALLOC | PTR_UNTRUSTED:
20069 case PTR_TO_BTF_ID | PTR_TRUSTED | PTR_MAYBE_NULL:
19840 if (type == BPF_READ) {20070 if (type == BPF_READ) {
19841 if (BPF_MODE(insn->code) == BPF_MEM)20071 if (BPF_MODE(insn->code) == BPF_MEM)
19842 insn->code = BPF_LDX | BPF_PROBE_MEM |20072 insn->code = BPF_LDX | BPF_PROBE_MEM |
@@ -20041,6 +20271,9 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 198 unmodified lines
20041 20271
20042 func[i]->aux->name[0] = 'F';20272 func[i]->aux->name[0] = 'F';
20043 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;20273 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
20274 if (env->subprog_info[i].priv_stack_mode == PRIV_STACK_ADAPTIVE)
20275 func[i]->aux->jits_use_priv_stack = true;
20276
20044 func[i]->jit_requested = 1;20277 func[i]->jit_requested = 1;
20045 func[i]->blinding_requested = prog->blinding_requested;20278 func[i]->blinding_requested = prog->blinding_requested;
20046 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;20279 func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
@@ -21809,6 +22042,11 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 1762 unmodified lines
21809 }22042 }
21810 }22043 }
21811 22044
22045 if (prog->aux->priv_stack_requested && !bpf_jit_supports_private_stack()) {
22046 verbose(env, "Private stack not supported by jit\n");
22047 return -EACCES;
22048 }
22049
21812 /* btf_ctx_access() used this to provide argument type info */22050 /* btf_ctx_access() used this to provide argument type info */
21813 prog->aux->ctx_arg_info =22051 prog->aux->ctx_arg_info =
21814 st_ops_desc->arg_info[member_idx].info;22052 st_ops_desc->arg_info[member_idx].info;
@@ -22547,6 +22785,7 @@ err_unlock: · 732 unmodified lines
22547 if (!is_priv)22785 if (!is_priv)
22548 mutex_unlock(&bpf_verifier_lock);22786 mutex_unlock(&bpf_verifier_lock);
22549 vfree(env->insn_aux_data);22787 vfree(env->insn_aux_data);
22788 kvfree(env->insn_hist);
22550err_free_env:22789err_free_env:
22551 kvfree(env);22790 kvfree(env);
22552 return ret;22791 return ret;
kernel/trace/bpf_trace.c+86−24
@@ -802,6 +802,8 @@ struct send_signal_irq_work { · 801 unmodified lines
802 struct task_struct *task;802 struct task_struct *task;
803 u32 sig;803 u32 sig;
804 enum pid_type type;804 enum pid_type type;
805 bool has_siginfo;
806 struct kernel_siginfo info;
805};807};
806 808
807static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);809static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work);
@@ -809,27 +811,46 @@ static DEFINE_PER_CPU(struct send_signal_irq_work, send_signal_work); · 1 unmodified line
809static void do_bpf_send_signal(struct irq_work *entry)811static void do_bpf_send_signal(struct irq_work *entry)
810{812{
811 struct send_signal_irq_work *work;813 struct send_signal_irq_work *work;
814 struct kernel_siginfo *siginfo;
812 815
813 work = container_of(entry, struct send_signal_irq_work, irq_work);816 work = container_of(entry, struct send_signal_irq_work, irq_work);
814 group_send_sig_info(work->sig, SEND_SIG_PRIV, work->task, work->type);817 siginfo = work->has_siginfo ? &work->info : SEND_SIG_PRIV;
818
819 group_send_sig_info(work->sig, siginfo, work->task, work->type);
815 put_task_struct(work->task);820 put_task_struct(work->task);
816}821}
817 822
818static int bpf_send_signal_common(u32 sig, enum pid_type type)823static int bpf_send_signal_common(u32 sig, enum pid_type type, struct task_struct *task, u64 value)
819{824{
820 struct send_signal_irq_work *work = NULL;825 struct send_signal_irq_work *work = NULL;
826 struct kernel_siginfo info;
827 struct kernel_siginfo *siginfo;
828
829 if (!task) {
830 task = current;
831 siginfo = SEND_SIG_PRIV;
832 } else {
833 clear_siginfo(&info);
834 info.si_signo = sig;
835 info.si_errno = 0;
836 info.si_code = SI_KERNEL;
837 info.si_pid = 0;
838 info.si_uid = 0;
839 info.si_value.sival_ptr = (void *)(unsigned long)value;
840 siginfo = &info;
841 }
821 842
822 /* Similar to bpf_probe_write_user, task needs to be843 /* Similar to bpf_probe_write_user, task needs to be
823 * in a sound condition and kernel memory access be844 * in a sound condition and kernel memory access be
824 * permitted in order to send signal to the current845 * permitted in order to send signal to the current
825 * task.846 * task.
826 */847 */
827 if (unlikely(current->flags & (PF_KTHREAD | PF_EXITING)))848 if (unlikely(task->flags & (PF_KTHREAD | PF_EXITING)))
828 return -EPERM;849 return -EPERM;
829 if (unlikely(!nmi_uaccess_okay()))850 if (unlikely(!nmi_uaccess_okay()))
830 return -EPERM;851 return -EPERM;
831 /* Task should not be pid=1 to avoid kernel panic. */852 /* Task should not be pid=1 to avoid kernel panic. */
832 if (unlikely(is_global_init(current)))853 if (unlikely(is_global_init(task)))
833 return -EPERM;854 return -EPERM;
834 855
835 if (irqs_disabled()) {856 if (irqs_disabled()) {
@@ -847,19 +868,22 @@ static int bpf_send_signal_common(u32 sig, enum pid_type type) · 11 unmodified lines
847 * to the irq_work. The current task may change when queued868 * to the irq_work. The current task may change when queued
848 * irq works get executed.869 * irq works get executed.
849 */870 */
850 work->task = get_task_struct(current);871 work->task = get_task_struct(task);
872 work->has_siginfo = siginfo == &info;
873 if (work->has_siginfo)
874 copy_siginfo(&work->info, &info);
851 work->sig = sig;875 work->sig = sig;
852 work->type = type;876 work->type = type;
853 irq_work_queue(&work->irq_work);877 irq_work_queue(&work->irq_work);
854 return 0;878 return 0;
855 }879 }
856 880
857 return group_send_sig_info(sig, SEND_SIG_PRIV, current, type);881 return group_send_sig_info(sig, siginfo, task, type);
858}882}
859 883
860BPF_CALL_1(bpf_send_signal, u32, sig)884BPF_CALL_1(bpf_send_signal, u32, sig)
861{885{
862 return bpf_send_signal_common(sig, PIDTYPE_TGID);886 return bpf_send_signal_common(sig, PIDTYPE_TGID, NULL, 0);
863}887}
864 888
865static const struct bpf_func_proto bpf_send_signal_proto = {889static const struct bpf_func_proto bpf_send_signal_proto = {
@@ -871,7 +895,7 @@ static const struct bpf_func_proto bpf_send_signal_proto = { · 5 unmodified lines
871 895
872BPF_CALL_1(bpf_send_signal_thread, u32, sig)896BPF_CALL_1(bpf_send_signal_thread, u32, sig)
873{897{
874 return bpf_send_signal_common(sig, PIDTYPE_PID);898 return bpf_send_signal_common(sig, PIDTYPE_PID, NULL, 0);
875}899}
876 900
877static const struct bpf_func_proto bpf_send_signal_thread_proto = {901static const struct bpf_func_proto bpf_send_signal_thread_proto = {
@@ -1557,6 +1581,17 @@ static inline bool is_kprobe_session(const struct bpf_prog *prog) · 679 unmodified lines
1557 return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;1581 return prog->expected_attach_type == BPF_TRACE_KPROBE_SESSION;
1558}1582}
1559 1583
1584static inline bool is_uprobe_multi(const struct bpf_prog *prog)
1585{
1586 return prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI ||
1587 prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1588}
1589
1590static inline bool is_uprobe_session(const struct bpf_prog *prog)
1591{
1592 return prog->expected_attach_type == BPF_TRACE_UPROBE_SESSION;
1593}
1594
1560static const struct bpf_func_proto *1595static const struct bpf_func_proto *
1561kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)1596kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1562{1597{
@@ -1574,13 +1609,13 @@ kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 11 unmodified lines
1574 case BPF_FUNC_get_func_ip:1609 case BPF_FUNC_get_func_ip:
1575 if (is_kprobe_multi(prog))1610 if (is_kprobe_multi(prog))
1576 return &bpf_get_func_ip_proto_kprobe_multi;1611 return &bpf_get_func_ip_proto_kprobe_multi;
1577 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)1612 if (is_uprobe_multi(prog))
1578 return &bpf_get_func_ip_proto_uprobe_multi;1613 return &bpf_get_func_ip_proto_uprobe_multi;
1579 return &bpf_get_func_ip_proto_kprobe;1614 return &bpf_get_func_ip_proto_kprobe;
1580 case BPF_FUNC_get_attach_cookie:1615 case BPF_FUNC_get_attach_cookie:
1581 if (is_kprobe_multi(prog))1616 if (is_kprobe_multi(prog))
1582 return &bpf_get_attach_cookie_proto_kmulti;1617 return &bpf_get_attach_cookie_proto_kmulti;
1583 if (prog->expected_attach_type == BPF_TRACE_UPROBE_MULTI)1618 if (is_uprobe_multi(prog))
1584 return &bpf_get_attach_cookie_proto_umulti;1619 return &bpf_get_attach_cookie_proto_umulti;
1585 return &bpf_get_attach_cookie_proto_trace;1620 return &bpf_get_attach_cookie_proto_trace;
1586 default:1621 default:
@@ -3072,6 +3107,7 @@ struct bpf_uprobe { · 1485 unmodified lines
3072 u64 cookie;3107 u64 cookie;
3073 struct uprobe *uprobe;3108 struct uprobe *uprobe;
3074 struct uprobe_consumer consumer;3109 struct uprobe_consumer consumer;
3110 bool session;
3075};3111};
3076 3112
3077struct bpf_uprobe_multi_link {3113struct bpf_uprobe_multi_link {
@@ -3084,7 +3120,7 @@ struct bpf_uprobe_multi_link { · 6 unmodified lines
3084};3120};
3085 3121
3086struct bpf_uprobe_multi_run_ctx {3122struct bpf_uprobe_multi_run_ctx {
3087 struct bpf_run_ctx run_ctx;3123 struct bpf_session_run_ctx session_ctx;
3088 unsigned long entry_ip;3124 unsigned long entry_ip;
3089 struct bpf_uprobe *uprobe;3125 struct bpf_uprobe *uprobe;
3090};3126};
@@ -3195,17 +3231,22 @@ static const struct bpf_link_ops bpf_uprobe_multi_link_lops = { · 104 unmodified lines
3195 3231
3196static int uprobe_prog_run(struct bpf_uprobe *uprobe,3232static int uprobe_prog_run(struct bpf_uprobe *uprobe,
3197 unsigned long entry_ip,3233 unsigned long entry_ip,
3198 struct pt_regs *regs)3234 struct pt_regs *regs,
3235 bool is_return, void *data)
3199{3236{
3200 struct bpf_uprobe_multi_link *link = uprobe->link;3237 struct bpf_uprobe_multi_link *link = uprobe->link;
3201 struct bpf_uprobe_multi_run_ctx run_ctx = {3238 struct bpf_uprobe_multi_run_ctx run_ctx = {
3239 .session_ctx = {
3240 .is_return = is_return,
3241 .data = data,
3242 },
3202 .entry_ip = entry_ip,3243 .entry_ip = entry_ip,
3203 .uprobe = uprobe,3244 .uprobe = uprobe,
3204 };3245 };
3205 struct bpf_prog *prog = link->link.prog;3246 struct bpf_prog *prog = link->link.prog;
3206 bool sleepable = prog->sleepable;3247 bool sleepable = prog->sleepable;
3207 struct bpf_run_ctx *old_run_ctx;3248 struct bpf_run_ctx *old_run_ctx;
3208 int err = 0;3249 int err;
3209 3250
3210 if (link->task && !same_thread_group(current, link->task))3251 if (link->task && !same_thread_group(current, link->task))
3211 return 0;3252 return 0;
@@ -3217,7 +3258,7 @@ static int uprobe_prog_run(struct bpf_uprobe *uprobe, · 5 unmodified lines
3217 3258
3218 migrate_disable();3259 migrate_disable();
3219 3260
3220 old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);3261 old_run_ctx = bpf_set_run_ctx(&run_ctx.session_ctx.run_ctx);
3221 err = bpf_prog_run(link->link.prog, regs);3262 err = bpf_prog_run(link->link.prog, regs);
3222 bpf_reset_run_ctx(old_run_ctx);3263 bpf_reset_run_ctx(old_run_ctx);
3223 3264
@@ -3244,9 +3285,13 @@ uprobe_multi_link_handler(struct uprobe_consumer *con, struct pt_regs *regs, · 20 unmodified lines
3244 __u64 *data)3285 __u64 *data)
3245{3286{
3246 struct bpf_uprobe *uprobe;3287 struct bpf_uprobe *uprobe;
3288 int ret;
3247 3289
3248 uprobe = container_of(con, struct bpf_uprobe, consumer);3290 uprobe = container_of(con, struct bpf_uprobe, consumer);
3249 return uprobe_prog_run(uprobe, instruction_pointer(regs), regs);3291 ret = uprobe_prog_run(uprobe, instruction_pointer(regs), regs, false, data);
3292 if (uprobe->session)
3293 return ret ? UPROBE_HANDLER_IGNORE : 0;
3294 return 0;
3250}3295}
3251 3296
3252static int3297static int
@@ -3256,14 +3301,16 @@ uprobe_multi_link_ret_handler(struct uprobe_consumer *con, unsigned long func, s · 3 unmodified lines
3256 struct bpf_uprobe *uprobe;3301 struct bpf_uprobe *uprobe;
3257 3302
3258 uprobe = container_of(con, struct bpf_uprobe, consumer);3303 uprobe = container_of(con, struct bpf_uprobe, consumer);
3259 return uprobe_prog_run(uprobe, func, regs);3304 uprobe_prog_run(uprobe, func, regs, true, data);
3305 return 0;
3260}3306}
3261 3307
3262static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)3308static u64 bpf_uprobe_multi_entry_ip(struct bpf_run_ctx *ctx)
3263{3309{
3264 struct bpf_uprobe_multi_run_ctx *run_ctx;3310 struct bpf_uprobe_multi_run_ctx *run_ctx;
3265 3311
3266 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);3312 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3313 session_ctx.run_ctx);
3267 return run_ctx->entry_ip;3314 return run_ctx->entry_ip;
3268}3315}
3269 3316
@@ -3271,7 +3318,8 @@ static u64 bpf_uprobe_multi_cookie(struct bpf_run_ctx *ctx) · 1 unmodified line
3271{3318{
3272 struct bpf_uprobe_multi_run_ctx *run_ctx;3319 struct bpf_uprobe_multi_run_ctx *run_ctx;
3273 3320
3274 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx, run_ctx);3321 run_ctx = container_of(current->bpf_ctx, struct bpf_uprobe_multi_run_ctx,
3322 session_ctx.run_ctx);
3275 return run_ctx->uprobe->cookie;3323 return run_ctx->uprobe->cookie;
3276}3324}
3277 3325
@@ -3295,7 +3343,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 17 unmodified lines
3295 if (sizeof(u64) != sizeof(void *))3343 if (sizeof(u64) != sizeof(void *))
3296 return -EOPNOTSUPP;3344 return -EOPNOTSUPP;
3297 3345
3298 if (prog->expected_attach_type != BPF_TRACE_UPROBE_MULTI)3346 if (!is_uprobe_multi(prog))
3299 return -EINVAL;3347 return -EINVAL;
3300 3348
3301 flags = attr->link_create.uprobe_multi.flags;3349 flags = attr->link_create.uprobe_multi.flags;
@@ -3371,11 +3419,12 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr · 69 unmodified lines
3371 3419
3372 uprobes[i].link = link;3420 uprobes[i].link = link;
3373 3421
3374 if (flags & BPF_F_UPROBE_MULTI_RETURN)3422 if (!(flags & BPF_F_UPROBE_MULTI_RETURN))
3375 uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3376 else
3377 uprobes[i].consumer.handler = uprobe_multi_link_handler;3423 uprobes[i].consumer.handler = uprobe_multi_link_handler;
3378 3424 if (flags & BPF_F_UPROBE_MULTI_RETURN || is_uprobe_session(prog))
3425 uprobes[i].consumer.ret_handler = uprobe_multi_link_ret_handler;
3426 if (is_uprobe_session(prog))
3427 uprobes[i].session = true;
3379 if (pid)3428 if (pid)
3380 uprobes[i].consumer.filter = uprobe_multi_link_filter;3429 uprobes[i].consumer.filter = uprobe_multi_link_filter;
3381 }3430 }
@@ -3464,7 +3513,7 @@ static int bpf_kprobe_multi_filter(const struct bpf_prog *prog, u32 kfunc_id) · 82 unmodified lines
3464 if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))3513 if (!btf_id_set8_contains(&kprobe_multi_kfunc_set_ids, kfunc_id))
3465 return 0;3514 return 0;
3466 3515
3467 if (!is_kprobe_session(prog))3516 if (!is_kprobe_session(prog) && !is_uprobe_session(prog))
3468 return -EACCES;3517 return -EACCES;
3469 3518
3470 return 0;3519 return 0;
@@ -3482,3 +3531,16 @@ static int __init bpf_kprobe_multi_kfuncs_init(void) · 11 unmodified lines
3482}3531}
3483 3532
3484late_initcall(bpf_kprobe_multi_kfuncs_init);3533late_initcall(bpf_kprobe_multi_kfuncs_init);
3534
3535__bpf_kfunc_start_defs();
3536
3537__bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type,
3538 u64 value)
3539{
3540 if (type != PIDTYPE_PID && type != PIDTYPE_TGID)
3541 return -EINVAL;
3542
3543 return bpf_send_signal_common(sig, type, task, value);
3544}
3545
3546__bpf_kfunc_end_defs();
mm/slab_common.c+19−0
@@ -1332,6 +1332,25 @@ size_t ksize(const void *objp) · 1331 unmodified lines
1332}1332}
1333EXPORT_SYMBOL(ksize);1333EXPORT_SYMBOL(ksize);
1334 1334
1335#ifdef CONFIG_BPF_SYSCALL
1336#include <linux/btf.h>
1337
1338__bpf_kfunc_start_defs();
1339
1340__bpf_kfunc struct kmem_cache *bpf_get_kmem_cache(u64 addr)
1341{
1342 struct slab *slab;
1343
1344 if (!virt_addr_valid((void *)(long)addr))
1345 return NULL;
1346
1347 slab = virt_to_slab((void *)(long)addr);
1348 return slab ? slab->slab_cache : NULL;
1349}
1350
1351__bpf_kfunc_end_defs();
1352#endif /* CONFIG_BPF_SYSCALL */
1353
1335/* Tracepoints definitions. */1354/* Tracepoints definitions. */
1336EXPORT_TRACEPOINT_SYMBOL(kmalloc);1355EXPORT_TRACEPOINT_SYMBOL(kmalloc);
1337EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);1356EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
net/core/bpf_sk_storage.c+3−3
@@ -106,7 +106,7 @@ static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key, · 105 unmodified lines
106 if (sock) {106 if (sock) {
107 sdata = bpf_local_storage_update(107 sdata = bpf_local_storage_update(
108 sock->sk, (struct bpf_local_storage_map *)map, value,108 sock->sk, (struct bpf_local_storage_map *)map, value,
109 map_flags, GFP_ATOMIC);109 map_flags, false, GFP_ATOMIC);
110 sockfd_put(sock);110 sockfd_put(sock);
111 return PTR_ERR_OR_ZERO(sdata);111 return PTR_ERR_OR_ZERO(sdata);
112 }112 }
@@ -137,7 +137,7 @@ bpf_sk_storage_clone_elem(struct sock *newsk, · 24 unmodified lines
137{137{
138 struct bpf_local_storage_elem *copy_selem;138 struct bpf_local_storage_elem *copy_selem;
139 139
140 copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, GFP_ATOMIC);140 copy_selem = bpf_selem_alloc(smap, newsk, NULL, true, false, GFP_ATOMIC);
141 if (!copy_selem)141 if (!copy_selem)
142 return NULL;142 return NULL;
143 143
@@ -243,7 +243,7 @@ BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, · 99 unmodified lines
243 refcount_inc_not_zero(&sk->sk_refcnt)) {243 refcount_inc_not_zero(&sk->sk_refcnt)) {
244 sdata = bpf_local_storage_update(244 sdata = bpf_local_storage_update(
245 sk, (struct bpf_local_storage_map *)map, value,245 sk, (struct bpf_local_storage_map *)map, value,
246 BPF_NOEXIST, gfp_flags);246 BPF_NOEXIST, false, gfp_flags);
247 /* sk must be a fullsock (guaranteed by verifier),247 /* sk must be a fullsock (guaranteed by verifier),
248 * so sock_gen_put() is unnecessary.248 * so sock_gen_put() is unnecessary.
249 */249 */
samples/bpf/Makefile+0−25
@@ -17,20 +17,12 @@ tprogs-y += tracex3 · 16 unmodified lines
17tprogs-y += tracex417tprogs-y += tracex4
18tprogs-y += tracex518tprogs-y += tracex5
19tprogs-y += tracex619tprogs-y += tracex6
20tprogs-y += tracex7
21tprogs-y += test_probe_write_user
22tprogs-y += trace_output20tprogs-y += trace_output
23tprogs-y += lathist21tprogs-y += lathist
24tprogs-y += offwaketime22tprogs-y += offwaketime
25tprogs-y += spintest23tprogs-y += spintest
26tprogs-y += map_perf_test24tprogs-y += map_perf_test
27tprogs-y += test_overhead
28tprogs-y += test_cgrp2_array_pin
29tprogs-y += test_cgrp2_attach
30tprogs-y += test_cgrp2_sock
31tprogs-y += test_cgrp2_sock2
32tprogs-y += xdp_router_ipv425tprogs-y += xdp_router_ipv4
33tprogs-y += test_current_task_under_cgroup
34tprogs-y += trace_event26tprogs-y += trace_event
35tprogs-y += sampleip27tprogs-y += sampleip
36tprogs-y += tc_l2_redirect28tprogs-y += tc_l2_redirect
@@ -66,20 +58,12 @@ tracex3-objs := tracex3_user.o · 29 unmodified lines
66tracex4-objs := tracex4_user.o58tracex4-objs := tracex4_user.o
67tracex5-objs := tracex5_user.o $(TRACE_HELPERS)59tracex5-objs := tracex5_user.o $(TRACE_HELPERS)
68tracex6-objs := tracex6_user.o60tracex6-objs := tracex6_user.o
69tracex7-objs := tracex7_user.o
70test_probe_write_user-objs := test_probe_write_user_user.o
71trace_output-objs := trace_output_user.o61trace_output-objs := trace_output_user.o
72lathist-objs := lathist_user.o62lathist-objs := lathist_user.o
73offwaketime-objs := offwaketime_user.o $(TRACE_HELPERS)63offwaketime-objs := offwaketime_user.o $(TRACE_HELPERS)
74spintest-objs := spintest_user.o $(TRACE_HELPERS)64spintest-objs := spintest_user.o $(TRACE_HELPERS)
75map_perf_test-objs := map_perf_test_user.o65map_perf_test-objs := map_perf_test_user.o
76test_overhead-objs := test_overhead_user.o66test_overhead-objs := test_overhead_user.o
77test_cgrp2_array_pin-objs := test_cgrp2_array_pin.o
78test_cgrp2_attach-objs := test_cgrp2_attach.o
79test_cgrp2_sock-objs := test_cgrp2_sock.o
80test_cgrp2_sock2-objs := test_cgrp2_sock2.o
81test_current_task_under_cgroup-objs := $(CGROUP_HELPERS) \
82 test_current_task_under_cgroup_user.o
83trace_event-objs := trace_event_user.o $(TRACE_HELPERS)67trace_event-objs := trace_event_user.o $(TRACE_HELPERS)
84sampleip-objs := sampleip_user.o $(TRACE_HELPERS)68sampleip-objs := sampleip_user.o $(TRACE_HELPERS)
85tc_l2_redirect-objs := tc_l2_redirect_user.o69tc_l2_redirect-objs := tc_l2_redirect_user.o
@@ -107,9 +91,6 @@ always-y += tracex3.bpf.o · 21 unmodified lines
107always-y += tracex4.bpf.o91always-y += tracex4.bpf.o
108always-y += tracex5.bpf.o92always-y += tracex5.bpf.o
109always-y += tracex6.bpf.o93always-y += tracex6.bpf.o
110always-y += tracex7.bpf.o
111always-y += sock_flags.bpf.o
112always-y += test_probe_write_user.bpf.o
113always-y += trace_output.bpf.o94always-y += trace_output.bpf.o
114always-y += tcbpf1_kern.o95always-y += tcbpf1_kern.o
115always-y += tc_l2_redirect_kern.o96always-y += tc_l2_redirect_kern.o
@@ -117,12 +98,7 @@ always-y += lathist_kern.o · 1 unmodified line
117always-y += offwaketime.bpf.o98always-y += offwaketime.bpf.o
118always-y += spintest.bpf.o99always-y += spintest.bpf.o
119always-y += map_perf_test.bpf.o100always-y += map_perf_test.bpf.o
120always-y += test_overhead_tp.bpf.o
121always-y += test_overhead_raw_tp.bpf.o
122always-y += test_overhead_kprobe.bpf.o
123always-y += parse_varlen.o parse_simple.o parse_ldabs.o101always-y += parse_varlen.o parse_simple.o parse_ldabs.o
124always-y += test_cgrp2_tc.bpf.o
125always-y += test_current_task_under_cgroup.bpf.o
126always-y += trace_event_kern.o102always-y += trace_event_kern.o
127always-y += sampleip_kern.o103always-y += sampleip_kern.o
128always-y += lwt_len_hist.bpf.o104always-y += lwt_len_hist.bpf.o
@@ -195,7 +171,6 @@ TPROGLDLIBS_xdp_router_ipv4 += -lm -pthread · 66 unmodified lines
195TPROGLDLIBS_tracex4 += -lrt171TPROGLDLIBS_tracex4 += -lrt
196TPROGLDLIBS_trace_output += -lrt172TPROGLDLIBS_trace_output += -lrt
197TPROGLDLIBS_map_perf_test += -lrt173TPROGLDLIBS_map_perf_test += -lrt
198TPROGLDLIBS_test_overhead += -lrt
199 174
200# Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:175# Allows pointing LLC/CLANG to a LLVM backend with bpf support, redefine on cmdline:
201# make M=samples/bpf LLC=~/git/llvm-project/llvm/build/bin/llc CLANG=~/git/llvm-project/llvm/build/bin/clang176# make M=samples/bpf LLC=~/git/llvm-project/llvm/build/bin/llc CLANG=~/git/llvm-project/llvm/build/bin/clang
samples/bpf/sock_flags.bpf.c+0−47
@@ -1,47 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2#include "vmlinux.h"
3#include "net_shared.h"
4#include <bpf/bpf_helpers.h>
5
6SEC("cgroup/sock")
7int bpf_prog1(struct bpf_sock *sk)
8{
9 char fmt[] = "socket: family %d type %d protocol %d\n";
10 char fmt2[] = "socket: uid %u gid %u\n";
11 __u64 gid_uid = bpf_get_current_uid_gid();
12 __u32 uid = gid_uid & 0xffffffff;
13 __u32 gid = gid_uid >> 32;
14
15 bpf_trace_printk(fmt, sizeof(fmt), sk->family, sk->type, sk->protocol);
16 bpf_trace_printk(fmt2, sizeof(fmt2), uid, gid);
17
18 /* block AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6 sockets
19 * ie., make ping6 fail
20 */
21 if (sk->family == AF_INET6 &&
22 sk->type == SOCK_DGRAM &&
23 sk->protocol == IPPROTO_ICMPV6)
24 return 0;
25
26 return 1;
27}
28
29SEC("cgroup/sock")
30int bpf_prog2(struct bpf_sock *sk)
31{
32 char fmt[] = "socket: family %d type %d protocol %d\n";
33
34 bpf_trace_printk(fmt, sizeof(fmt), sk->family, sk->type, sk->protocol);
35
36 /* block AF_INET, SOCK_DGRAM, IPPROTO_ICMP sockets
37 * ie., make ping fail
38 */
39 if (sk->family == AF_INET &&
40 sk->type == SOCK_DGRAM &&
41 sk->protocol == IPPROTO_ICMP)
42 return 0;
43
44 return 1;
45}
46
47char _license[] SEC("license") = "GPL";
samples/bpf/syscall_nrs.c+5−0
@@ -2,6 +2,9 @@ · 1 unmodified line
2#include <uapi/linux/unistd.h>2#include <uapi/linux/unistd.h>
3#include <linux/kbuild.h>3#include <linux/kbuild.h>
4 4
5#pragma GCC diagnostic push
6#pragma GCC diagnostic ignored "-Wmissing-prototypes"
7
5#define SYSNR(_NR) DEFINE(SYS ## _NR, _NR)8#define SYSNR(_NR) DEFINE(SYS ## _NR, _NR)
6 9
7void syscall_defines(void)10void syscall_defines(void)
@@ -17,3 +20,5 @@ void syscall_defines(void) · 9 unmodified lines
17#endif20#endif
18 21
19}22}
23
24#pragma GCC diagnostic pop
samples/bpf/tc_l2_redirect_kern.c+0−6
@@ -58,14 +58,11 @@ static __always_inline bool is_vip_addr(__be16 eth_proto, __be32 daddr) · 57 unmodified lines
58SEC("l2_to_iptun_ingress_forward")58SEC("l2_to_iptun_ingress_forward")
59int _l2_to_iptun_ingress_forward(struct __sk_buff *skb)59int _l2_to_iptun_ingress_forward(struct __sk_buff *skb)
60{60{
61 struct bpf_tunnel_key tkey = {};
62 void *data = (void *)(long)skb->data;61 void *data = (void *)(long)skb->data;
63 struct eth_hdr *eth = data;62 struct eth_hdr *eth = data;
64 void *data_end = (void *)(long)skb->data_end;63 void *data_end = (void *)(long)skb->data_end;
65 int key = 0, *ifindex;64 int key = 0, *ifindex;
66 65
67 int ret;
68
69 if (data + sizeof(*eth) > data_end)66 if (data + sizeof(*eth) > data_end)
70 return TC_ACT_OK;67 return TC_ACT_OK;
71 68
@@ -115,8 +112,6 @@ int _l2_to_iptun_ingress_redirect(struct __sk_buff *skb) · 43 unmodified lines
115 void *data_end = (void *)(long)skb->data_end;112 void *data_end = (void *)(long)skb->data_end;
116 int key = 0, *ifindex;113 int key = 0, *ifindex;
117 114
118 int ret;
119
120 if (data + sizeof(*eth) > data_end)115 if (data + sizeof(*eth) > data_end)
121 return TC_ACT_OK;116 return TC_ACT_OK;
122 117
@@ -205,7 +200,6 @@ int _l2_to_ip6tun_ingress_redirect(struct __sk_buff *skb) · 82 unmodified lines
205SEC("drop_non_tun_vip")200SEC("drop_non_tun_vip")
206int _drop_non_tun_vip(struct __sk_buff *skb)201int _drop_non_tun_vip(struct __sk_buff *skb)
207{202{
208 struct bpf_tunnel_key tkey = {};
209 void *data = (void *)(long)skb->data;203 void *data = (void *)(long)skb->data;
210 struct eth_hdr *eth = data;204 struct eth_hdr *eth = data;
211 void *data_end = (void *)(long)skb->data_end;205 void *data_end = (void *)(long)skb->data_end;
samples/bpf/test_cgrp2_array_pin.c+0−106
@@ -1,106 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2016 Facebook
3 */
4#include <linux/unistd.h>
5#include <linux/bpf.h>
6
7#include <stdio.h>
8#include <stdint.h>
9#include <unistd.h>
10#include <string.h>
11#include <errno.h>
12#include <fcntl.h>
13
14#include <bpf/bpf.h>
15
16static void usage(void)
17{
18 printf("Usage: test_cgrp2_array_pin [...]\n");
19 printf(" -F <file> File to pin an BPF cgroup array\n");
20 printf(" -U <file> Update an already pinned BPF cgroup array\n");
21 printf(" -v <value> Full path of the cgroup2\n");
22 printf(" -h Display this help\n");
23}
24
25int main(int argc, char **argv)
26{
27 const char *pinned_file = NULL, *cg2 = NULL;
28 int create_array = 1;
29 int array_key = 0;
30 int array_fd = -1;
31 int cg2_fd = -1;
32 int ret = -1;
33 int opt;
34
35 while ((opt = getopt(argc, argv, "F:U:v:")) != -1) {
36 switch (opt) {
37 /* General args */
38 case 'F':
39 pinned_file = optarg;
40 break;
41 case 'U':
42 pinned_file = optarg;
43 create_array = 0;
44 break;
45 case 'v':
46 cg2 = optarg;
47 break;
48 default:
49 usage();
50 goto out;
51 }
52 }
53
54 if (!cg2 || !pinned_file) {
55 usage();
56 goto out;
57 }
58
59 cg2_fd = open(cg2, O_RDONLY);
60 if (cg2_fd < 0) {
61 fprintf(stderr, "open(%s,...): %s(%d)\n",
62 cg2, strerror(errno), errno);
63 goto out;
64 }
65
66 if (create_array) {
67 array_fd = bpf_map_create(BPF_MAP_TYPE_CGROUP_ARRAY, NULL,
68 sizeof(uint32_t), sizeof(uint32_t),
69 1, NULL);
70 if (array_fd < 0) {
71 fprintf(stderr,
72 "bpf_create_map(BPF_MAP_TYPE_CGROUP_ARRAY,...): %s(%d)\n",
73 strerror(errno), errno);
74 goto out;
75 }
76 } else {
77 array_fd = bpf_obj_get(pinned_file);
78 if (array_fd < 0) {
79 fprintf(stderr, "bpf_obj_get(%s): %s(%d)\n",
80 pinned_file, strerror(errno), errno);
81 goto out;
82 }
83 }
84
85 ret = bpf_map_update_elem(array_fd, &array_key, &cg2_fd, 0);
86 if (ret) {
87 perror("bpf_map_update_elem");
88 goto out;
89 }
90
91 if (create_array) {
92 ret = bpf_obj_pin(array_fd, pinned_file);
93 if (ret) {
94 fprintf(stderr, "bpf_obj_pin(..., %s): %s(%d)\n",
95 pinned_file, strerror(errno), errno);
96 goto out;
97 }
98 }
99
100out:
101 if (array_fd != -1)
102 close(array_fd);
103 if (cg2_fd != -1)
104 close(cg2_fd);
105 return ret;
106}
samples/bpf/test_cgrp2_attach.c+0−177
@@ -1,177 +0,0 @@
1/* eBPF example program:
2 *
3 * - Creates arraymap in kernel with 4 bytes keys and 8 byte values
4 *
5 * - Loads eBPF program
6 *
7 * The eBPF program accesses the map passed in to store two pieces of
8 * information. The number of invocations of the program, which maps
9 * to the number of packets received, is stored to key 0. Key 1 is
10 * incremented on each iteration by the number of bytes stored in
11 * the skb.
12 *
13 * - Attaches the new program to a cgroup using BPF_PROG_ATTACH
14 *
15 * - Every second, reads map[0] and map[1] to see how many bytes and
16 * packets were seen on any socket of tasks in the given cgroup.
17 */
18
19#define _GNU_SOURCE
20
21#include <stdio.h>
22#include <stdlib.h>
23#include <stddef.h>
24#include <string.h>
25#include <unistd.h>
26#include <assert.h>
27#include <errno.h>
28#include <fcntl.h>
29
30#include <linux/bpf.h>
31#include <bpf/bpf.h>
32
33#include "bpf_insn.h"
34#include "bpf_util.h"
35
36enum {
37 MAP_KEY_PACKETS,
38 MAP_KEY_BYTES,
39};
40
41char bpf_log_buf[BPF_LOG_BUF_SIZE];
42
43static int prog_load(int map_fd, int verdict)
44{
45 struct bpf_insn prog[] = {
46 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1), /* save r6 so it's not clobbered by BPF_CALL */
47
48 /* Count packets */
49 BPF_MOV64_IMM(BPF_REG_0, MAP_KEY_PACKETS), /* r0 = 0 */
50 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4), /* *(u32 *)(fp - 4) = r0 */
51 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
52 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), /* r2 = fp - 4 */
53 BPF_LD_MAP_FD(BPF_REG_1, map_fd), /* load map fd to r1 */
54 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
55 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
56 BPF_MOV64_IMM(BPF_REG_1, 1), /* r1 = 1 */
57 BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0),
58
59 /* Count bytes */
60 BPF_MOV64_IMM(BPF_REG_0, MAP_KEY_BYTES), /* r0 = 1 */
61 BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4), /* *(u32 *)(fp - 4) = r0 */
62 BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
63 BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), /* r2 = fp - 4 */
64 BPF_LD_MAP_FD(BPF_REG_1, map_fd),
65 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
66 BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
67 BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_6, offsetof(struct __sk_buff, len)), /* r1 = skb->len */
68
69 BPF_ATOMIC_OP(BPF_DW, BPF_ADD, BPF_REG_0, BPF_REG_1, 0),
70
71 BPF_MOV64_IMM(BPF_REG_0, verdict), /* r0 = verdict */
72 BPF_EXIT_INSN(),
73 };
74 size_t insns_cnt = ARRAY_SIZE(prog);
75 LIBBPF_OPTS(bpf_prog_load_opts, opts,
76 .log_buf = bpf_log_buf,
77 .log_size = BPF_LOG_BUF_SIZE,
78 );
79
80 return bpf_prog_load(BPF_PROG_TYPE_CGROUP_SKB, NULL, "GPL",
81 prog, insns_cnt, &opts);
82}
83
84static int usage(const char *argv0)
85{
86 printf("Usage: %s [-d] [-D] <cg-path> <egress|ingress>\n", argv0);
87 printf(" -d Drop Traffic\n");
88 printf(" -D Detach filter, and exit\n");
89 return EXIT_FAILURE;
90}
91
92static int attach_filter(int cg_fd, int type, int verdict)
93{
94 int prog_fd, map_fd, ret, key;
95 long long pkt_cnt, byte_cnt;
96
97 map_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL,
98 sizeof(key), sizeof(byte_cnt),
99 256, NULL);
100 if (map_fd < 0) {
101 printf("Failed to create map: '%s'\n", strerror(errno));
102 return EXIT_FAILURE;
103 }
104
105 prog_fd = prog_load(map_fd, verdict);
106 printf("Output from kernel verifier:\n%s\n-------\n", bpf_log_buf);
107
108 if (prog_fd < 0) {
109 printf("Failed to load prog: '%s'\n", strerror(errno));
110 return EXIT_FAILURE;
111 }
112
113 ret = bpf_prog_attach(prog_fd, cg_fd, type, 0);
114 if (ret < 0) {
115 printf("Failed to attach prog to cgroup: '%s'\n",
116 strerror(errno));
117 return EXIT_FAILURE;
118 }
119 while (1) {
120 key = MAP_KEY_PACKETS;
121 assert(bpf_map_lookup_elem(map_fd, &key, &pkt_cnt) == 0);
122
123 key = MAP_KEY_BYTES;
124 assert(bpf_map_lookup_elem(map_fd, &key, &byte_cnt) == 0);
125
126 printf("cgroup received %lld packets, %lld bytes\n",
127 pkt_cnt, byte_cnt);
128 sleep(1);
129 }
130
131 return EXIT_SUCCESS;
132}
133
134int main(int argc, char **argv)
135{
136 int detach_only = 0, verdict = 1;
137 enum bpf_attach_type type;
138 int opt, cg_fd, ret;
139
140 while ((opt = getopt(argc, argv, "Dd")) != -1) {
141 switch (opt) {
142 case 'd':
143 verdict = 0;
144 break;
145 case 'D':
146 detach_only = 1;
147 break;
148 default:
149 return usage(argv[0]);
150 }
151 }
152
153 if (argc - optind < 2)
154 return usage(argv[0]);
155
156 if (strcmp(argv[optind + 1], "ingress") == 0)
157 type = BPF_CGROUP_INET_INGRESS;
158 else if (strcmp(argv[optind + 1], "egress") == 0)
159 type = BPF_CGROUP_INET_EGRESS;
160 else
161 return usage(argv[0]);
162
163 cg_fd = open(argv[optind], O_DIRECTORY | O_RDONLY);
164 if (cg_fd < 0) {
165 printf("Failed to open cgroup path: '%s'\n", strerror(errno));
166 return EXIT_FAILURE;
167 }
168
169 if (detach_only) {
170 ret = bpf_prog_detach(cg_fd, type);
171 printf("bpf_prog_detach() returned '%s' (%d)\n",
172 strerror(errno), errno);
173 } else
174 ret = attach_filter(cg_fd, type, verdict);
175
176 return ret;
177}
samples/bpf/test_cgrp2_sock.c+0−294
@@ -1,294 +0,0 @@
1/* eBPF example program:
2 *
3 * - Loads eBPF program
4 *
5 * The eBPF program sets the sk_bound_dev_if index in new AF_INET{6}
6 * sockets opened by processes in the cgroup.
7 *
8 * - Attaches the new program to a cgroup using BPF_PROG_ATTACH
9 */
10
11#define _GNU_SOURCE
12
13#include <stdio.h>
14#include <stdlib.h>
15#include <stddef.h>
16#include <string.h>
17#include <unistd.h>
18#include <assert.h>
19#include <errno.h>
20#include <fcntl.h>
21#include <net/if.h>
22#include <inttypes.h>
23#include <linux/bpf.h>
24#include <bpf/bpf.h>
25
26#include "bpf_insn.h"
27
28char bpf_log_buf[BPF_LOG_BUF_SIZE];
29
30static int prog_load(__u32 idx, __u32 mark, __u32 prio)
31{
32 /* save pointer to context */
33 struct bpf_insn prog_start[] = {
34 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
35 };
36 struct bpf_insn prog_end[] = {
37 BPF_MOV64_IMM(BPF_REG_0, 1), /* r0 = verdict */
38 BPF_EXIT_INSN(),
39 };
40
41 /* set sk_bound_dev_if on socket */
42 struct bpf_insn prog_dev[] = {
43 BPF_MOV64_IMM(BPF_REG_3, idx),
44 BPF_MOV64_IMM(BPF_REG_2, offsetof(struct bpf_sock, bound_dev_if)),
45 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, offsetof(struct bpf_sock, bound_dev_if)),
46 };
47
48 /* set mark on socket */
49 struct bpf_insn prog_mark[] = {
50 /* get uid of process */
51 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0,
52 BPF_FUNC_get_current_uid_gid),
53 BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffffffff),
54
55 /* if uid is 0, use given mark, else use the uid as the mark */
56 BPF_MOV64_REG(BPF_REG_3, BPF_REG_0),
57 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
58 BPF_MOV64_IMM(BPF_REG_3, mark),
59
60 /* set the mark on the new socket */
61 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
62 BPF_MOV64_IMM(BPF_REG_2, offsetof(struct bpf_sock, mark)),
63 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, offsetof(struct bpf_sock, mark)),
64 };
65
66 /* set priority on socket */
67 struct bpf_insn prog_prio[] = {
68 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
69 BPF_MOV64_IMM(BPF_REG_3, prio),
70 BPF_MOV64_IMM(BPF_REG_2, offsetof(struct bpf_sock, priority)),
71 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3, offsetof(struct bpf_sock, priority)),
72 };
73 LIBBPF_OPTS(bpf_prog_load_opts, opts,
74 .log_buf = bpf_log_buf,
75 .log_size = BPF_LOG_BUF_SIZE,
76 );
77
78 struct bpf_insn *prog;
79 size_t insns_cnt;
80 void *p;
81 int ret;
82
83 insns_cnt = sizeof(prog_start) + sizeof(prog_end);
84 if (idx)
85 insns_cnt += sizeof(prog_dev);
86
87 if (mark)
88 insns_cnt += sizeof(prog_mark);
89
90 if (prio)
91 insns_cnt += sizeof(prog_prio);
92
93 p = prog = malloc(insns_cnt);
94 if (!prog) {
95 fprintf(stderr, "Failed to allocate memory for instructions\n");
96 return EXIT_FAILURE;
97 }
98
99 memcpy(p, prog_start, sizeof(prog_start));
100 p += sizeof(prog_start);
101
102 if (idx) {
103 memcpy(p, prog_dev, sizeof(prog_dev));
104 p += sizeof(prog_dev);
105 }
106
107 if (mark) {
108 memcpy(p, prog_mark, sizeof(prog_mark));
109 p += sizeof(prog_mark);
110 }
111
112 if (prio) {
113 memcpy(p, prog_prio, sizeof(prog_prio));
114 p += sizeof(prog_prio);
115 }
116
117 memcpy(p, prog_end, sizeof(prog_end));
118 p += sizeof(prog_end);
119
120 insns_cnt /= sizeof(struct bpf_insn);
121
122 ret = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL",
123 prog, insns_cnt, &opts);
124
125 free(prog);
126
127 return ret;
128}
129
130static int get_bind_to_device(int sd, char *name, size_t len)
131{
132 socklen_t optlen = len;
133 int rc;
134
135 name[0] = '\0';
136 rc = getsockopt(sd, SOL_SOCKET, SO_BINDTODEVICE, name, &optlen);
137 if (rc < 0)
138 perror("setsockopt(SO_BINDTODEVICE)");
139
140 return rc;
141}
142
143static unsigned int get_somark(int sd)
144{
145 unsigned int mark = 0;
146 socklen_t optlen = sizeof(mark);
147 int rc;
148
149 rc = getsockopt(sd, SOL_SOCKET, SO_MARK, &mark, &optlen);
150 if (rc < 0)
151 perror("getsockopt(SO_MARK)");
152
153 return mark;
154}
155
156static unsigned int get_priority(int sd)
157{
158 unsigned int prio = 0;
159 socklen_t optlen = sizeof(prio);
160 int rc;
161
162 rc = getsockopt(sd, SOL_SOCKET, SO_PRIORITY, &prio, &optlen);
163 if (rc < 0)
164 perror("getsockopt(SO_PRIORITY)");
165
166 return prio;
167}
168
169static int show_sockopts(int family)
170{
171 unsigned int mark, prio;
172 char name[16];
173 int sd;
174
175 sd = socket(family, SOCK_DGRAM, 17);
176 if (sd < 0) {
177 perror("socket");
178 return 1;
179 }
180
181 if (get_bind_to_device(sd, name, sizeof(name)) < 0)
182 return 1;
183
184 mark = get_somark(sd);
185 prio = get_priority(sd);
186
187 close(sd);
188
189 printf("sd %d: dev %s, mark %u, priority %u\n", sd, name, mark, prio);
190
191 return 0;
192}
193
194static int usage(const char *argv0)
195{
196 printf("Usage:\n");
197 printf(" Attach a program\n");
198 printf(" %s -b bind-to-dev -m mark -p prio cg-path\n", argv0);
199 printf("\n");
200 printf(" Detach a program\n");
201 printf(" %s -d cg-path\n", argv0);
202 printf("\n");
203 printf(" Show inherited socket settings (mark, priority, and device)\n");
204 printf(" %s [-6]\n", argv0);
205 return EXIT_FAILURE;
206}
207
208int main(int argc, char **argv)
209{
210 __u32 idx = 0, mark = 0, prio = 0;
211 const char *cgrp_path = NULL;
212 int cg_fd, prog_fd, ret;
213 int family = PF_INET;
214 int do_attach = 1;
215 int rc;
216
217 while ((rc = getopt(argc, argv, "db:m:p:6")) != -1) {
218 switch (rc) {
219 case 'd':
220 do_attach = 0;
221 break;
222 case 'b':
223 idx = if_nametoindex(optarg);
224 if (!idx) {
225 idx = strtoumax(optarg, NULL, 0);
226 if (!idx) {
227 printf("Invalid device name\n");
228 return EXIT_FAILURE;
229 }
230 }
231 break;
232 case 'm':
233 mark = strtoumax(optarg, NULL, 0);
234 break;
235 case 'p':
236 prio = strtoumax(optarg, NULL, 0);
237 break;
238 case '6':
239 family = PF_INET6;
240 break;
241 default:
242 return usage(argv[0]);
243 }
244 }
245
246 if (optind == argc)
247 return show_sockopts(family);
248
249 cgrp_path = argv[optind];
250 if (!cgrp_path) {
251 fprintf(stderr, "cgroup path not given\n");
252 return EXIT_FAILURE;
253 }
254
255 if (do_attach && !idx && !mark && !prio) {
256 fprintf(stderr,
257 "One of device, mark or priority must be given\n");
258 return EXIT_FAILURE;
259 }
260
261 cg_fd = open(cgrp_path, O_DIRECTORY | O_RDONLY);
262 if (cg_fd < 0) {
263 printf("Failed to open cgroup path: '%s'\n", strerror(errno));
264 return EXIT_FAILURE;
265 }
266
267 if (do_attach) {
268 prog_fd = prog_load(idx, mark, prio);
269 if (prog_fd < 0) {
270 printf("Failed to load prog: '%s'\n", strerror(errno));
271 printf("Output from kernel verifier:\n%s\n-------\n",
272 bpf_log_buf);
273 return EXIT_FAILURE;
274 }
275
276 ret = bpf_prog_attach(prog_fd, cg_fd,
277 BPF_CGROUP_INET_SOCK_CREATE, 0);
278 if (ret < 0) {
279 printf("Failed to attach prog to cgroup: '%s'\n",
280 strerror(errno));
281 return EXIT_FAILURE;
282 }
283 } else {
284 ret = bpf_prog_detach(cg_fd, BPF_CGROUP_INET_SOCK_CREATE);
285 if (ret < 0) {
286 printf("Failed to detach prog from cgroup: '%s'\n",
287 strerror(errno));
288 return EXIT_FAILURE;
289 }
290 }
291
292 close(cg_fd);
293 return EXIT_SUCCESS;
294}
samples/bpf/test_cgrp2_sock.sh+0−137
@@ -1,137 +0,0 @@
1#!/bin/sh
2# SPDX-License-Identifier: GPL-2.0
3
4# Test various socket options that can be set by attaching programs to cgroups.
5
6MY_DIR=$(dirname $0)
7TEST=$MY_DIR/test_cgrp2_sock
8CGRP_MNT="/tmp/cgroupv2-test_cgrp2_sock"
9
10################################################################################
11#
12print_result()
13{
14 local rc=$1
15 local status=" OK "
16
17 [ $rc -ne 0 ] && status="FAIL"
18
19 printf "%-50s [%4s]\n" "$2" "$status"
20}
21
22check_sock()
23{
24 out=$($TEST)
25 echo $out | grep -q "$1"
26 if [ $? -ne 0 ]; then
27 print_result 1 "IPv4: $2"
28 echo " expected: $1"
29 echo " have: $out"
30 rc=1
31 else
32 print_result 0 "IPv4: $2"
33 fi
34}
35
36check_sock6()
37{
38 out=$($TEST -6)
39 echo $out | grep -q "$1"
40 if [ $? -ne 0 ]; then
41 print_result 1 "IPv6: $2"
42 echo " expected: $1"
43 echo " have: $out"
44 rc=1
45 else
46 print_result 0 "IPv6: $2"
47 fi
48}
49
50################################################################################
51#
52
53cleanup()
54{
55 echo $$ >> ${CGRP_MNT}/cgroup.procs
56 rmdir ${CGRP_MNT}/sockopts
57}
58
59cleanup_and_exit()
60{
61 local rc=$1
62 local msg="$2"
63
64 [ -n "$msg" ] && echo "ERROR: $msg"
65
66 $TEST -d ${CGRP_MNT}/sockopts
67 ip li del cgrp2_sock
68 umount ${CGRP_MNT}
69
70 exit $rc
71}
72
73
74################################################################################
75# main
76
77rc=0
78
79ip li add cgrp2_sock type dummy 2>/dev/null
80
81set -e
82mkdir -p ${CGRP_MNT}
83mount -t cgroup2 none ${CGRP_MNT}
84set +e
85
86
87# make sure we have a known start point
88cleanup 2>/dev/null
89
90mkdir -p ${CGRP_MNT}/sockopts
91[ $? -ne 0 ] && cleanup_and_exit 1 "Failed to create cgroup hierarchy"
92
93
94# set pid into cgroup
95echo $$ > ${CGRP_MNT}/sockopts/cgroup.procs
96
97# no bpf program attached, so socket should show no settings
98check_sock "dev , mark 0, priority 0" "No programs attached"
99check_sock6 "dev , mark 0, priority 0" "No programs attached"
100
101# verify device is set
102#
103$TEST -b cgrp2_sock ${CGRP_MNT}/sockopts
104if [ $? -ne 0 ]; then
105 cleanup_and_exit 1 "Failed to install program to set device"
106fi
107check_sock "dev cgrp2_sock, mark 0, priority 0" "Device set"
108check_sock6 "dev cgrp2_sock, mark 0, priority 0" "Device set"
109
110# verify mark is set
111#
112$TEST -m 666 ${CGRP_MNT}/sockopts
113if [ $? -ne 0 ]; then
114 cleanup_and_exit 1 "Failed to install program to set mark"
115fi
116check_sock "dev , mark 666, priority 0" "Mark set"
117check_sock6 "dev , mark 666, priority 0" "Mark set"
118
119# verify priority is set
120#
121$TEST -p 123 ${CGRP_MNT}/sockopts
122if [ $? -ne 0 ]; then
123 cleanup_and_exit 1 "Failed to install program to set priority"
124fi
125check_sock "dev , mark 0, priority 123" "Priority set"
126check_sock6 "dev , mark 0, priority 123" "Priority set"
127
128# all 3 at once
129#
130$TEST -b cgrp2_sock -m 666 -p 123 ${CGRP_MNT}/sockopts
131if [ $? -ne 0 ]; then
132 cleanup_and_exit 1 "Failed to install program to set device, mark and priority"
133fi
134check_sock "dev cgrp2_sock, mark 666, priority 123" "Priority set"
135check_sock6 "dev cgrp2_sock, mark 666, priority 123" "Priority set"
136
137cleanup_and_exit $rc
samples/bpf/test_cgrp2_sock2.c+0−95
@@ -1,95 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/* eBPF example program:
3 *
4 * - Loads eBPF program
5 *
6 * The eBPF program loads a filter from file and attaches the
7 * program to a cgroup using BPF_PROG_ATTACH
8 */
9
10#define _GNU_SOURCE
11
12#include <stdio.h>
13#include <stdlib.h>
14#include <stddef.h>
15#include <string.h>
16#include <unistd.h>
17#include <assert.h>
18#include <errno.h>
19#include <fcntl.h>
20#include <net/if.h>
21#include <linux/bpf.h>
22#include <bpf/bpf.h>
23#include <bpf/libbpf.h>
24
25#include "bpf_insn.h"
26
27static int usage(const char *argv0)
28{
29 printf("Usage: %s cg-path filter-path [filter-id]\n", argv0);
30 return EXIT_FAILURE;
31}
32
33int main(int argc, char **argv)
34{
35 int cg_fd, err, ret = EXIT_FAILURE, filter_id = 0, prog_cnt = 0;
36 const char *link_pin_path = "/sys/fs/bpf/test_cgrp2_sock2";
37 struct bpf_link *link = NULL;
38 struct bpf_program *progs[2];
39 struct bpf_program *prog;
40 struct bpf_object *obj;
41
42 if (argc < 3)
43 return usage(argv[0]);
44
45 if (argc > 3)
46 filter_id = atoi(argv[3]);
47
48 cg_fd = open(argv[1], O_DIRECTORY | O_RDONLY);
49 if (cg_fd < 0) {
50 printf("Failed to open cgroup path: '%s'\n", strerror(errno));
51 return ret;
52 }
53
54 obj = bpf_object__open_file(argv[2], NULL);
55 if (libbpf_get_error(obj)) {
56 printf("ERROR: opening BPF object file failed\n");
57 return ret;
58 }
59
60 bpf_object__for_each_program(prog, obj) {
61 progs[prog_cnt] = prog;
62 prog_cnt++;
63 }
64
65 if (filter_id >= prog_cnt) {
66 printf("Invalid program id; program not found in file\n");
67 goto cleanup;
68 }
69
70 /* load BPF program */
71 if (bpf_object__load(obj)) {
72 printf("ERROR: loading BPF object file failed\n");
73 goto cleanup;
74 }
75
76 link = bpf_program__attach_cgroup(progs[filter_id], cg_fd);
77 if (libbpf_get_error(link)) {
78 printf("ERROR: bpf_program__attach failed\n");
79 link = NULL;
80 goto cleanup;
81 }
82
83 err = bpf_link__pin(link, link_pin_path);
84 if (err < 0) {
85 printf("ERROR: bpf_link__pin failed: %d\n", err);
86 goto cleanup;
87 }
88
89 ret = EXIT_SUCCESS;
90
91cleanup:
92 bpf_link__destroy(link);
93 bpf_object__close(obj);
94 return ret;
95}
samples/bpf/test_cgrp2_sock2.sh+0−103
@@ -1,103 +0,0 @@
1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4BPFFS=/sys/fs/bpf
5MY_DIR=$(dirname $0)
6TEST=$MY_DIR/test_cgrp2_sock2
7LINK_PIN=$BPFFS/test_cgrp2_sock2
8BPF_PROG=$MY_DIR/sock_flags.bpf.o
9
10function config_device {
11 ip netns add at_ns0
12 ip link add veth0 type veth peer name veth0b
13 ip link set veth0 netns at_ns0
14 ip netns exec at_ns0 sysctl -q net.ipv6.conf.veth0.disable_ipv6=0
15 ip netns exec at_ns0 ip addr add 172.16.1.100/24 dev veth0
16 ip netns exec at_ns0 ip addr add 2401:db00::1/64 dev veth0 nodad
17 ip netns exec at_ns0 ip link set dev veth0 up
18 sysctl -q net.ipv6.conf.veth0b.disable_ipv6=0
19 ip addr add 172.16.1.101/24 dev veth0b
20 ip addr add 2401:db00::2/64 dev veth0b nodad
21 ip link set veth0b up
22}
23
24function config_cgroup {
25 rm -rf /tmp/cgroupv2
26 mkdir -p /tmp/cgroupv2
27 mount -t cgroup2 none /tmp/cgroupv2
28 mkdir -p /tmp/cgroupv2/foo
29 echo $$ >> /tmp/cgroupv2/foo/cgroup.procs
30}
31
32function config_bpffs {
33 if mount | grep $BPFFS > /dev/null; then
34 echo "bpffs already mounted"
35 else
36 echo "bpffs not mounted. Mounting..."
37 mount -t bpf none $BPFFS
38 fi
39}
40
41function attach_bpf {
42 $TEST /tmp/cgroupv2/foo $BPF_PROG $1
43 [ $? -ne 0 ] && exit 1
44}
45
46function cleanup {
47 rm -rf $LINK_PIN
48 ip link del veth0b
49 ip netns delete at_ns0
50 umount /tmp/cgroupv2
51 rm -rf /tmp/cgroupv2
52}
53
54cleanup 2>/dev/null
55
56set -e
57config_device
58config_cgroup
59config_bpffs
60set +e
61
62#
63# Test 1 - fail ping6
64#
65attach_bpf 0
66ping -c1 -w1 172.16.1.100
67if [ $? -ne 0 ]; then
68 echo "ping failed when it should succeed"
69 cleanup
70 exit 1
71fi
72
73ping6 -c1 -w1 2401:db00::1
74if [ $? -eq 0 ]; then
75 echo "ping6 succeeded when it should not"
76 cleanup
77 exit 1
78fi
79
80rm -rf $LINK_PIN
81sleep 1 # Wait for link detach
82
83#
84# Test 2 - fail ping
85#
86attach_bpf 1
87ping6 -c1 -w1 2401:db00::1
88if [ $? -ne 0 ]; then
89 echo "ping6 failed when it should succeed"
90 cleanup
91 exit 1
92fi
93
94ping -c1 -w1 172.16.1.100
95if [ $? -eq 0 ]; then
96 echo "ping succeeded when it should not"
97 cleanup
98 exit 1
99fi
100
101cleanup
102echo
103echo "*** PASS ***"
samples/bpf/test_cgrp2_tc.bpf.c+0−56
@@ -1,56 +0,0 @@
1/* Copyright (c) 2016 Facebook
2 *
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 */
7#define KBUILD_MODNAME "foo"
8#include "vmlinux.h"
9#include "net_shared.h"
10#include <bpf/bpf_helpers.h>
11
12/* copy of 'struct ethhdr' without __packed */
13struct eth_hdr {
14 unsigned char h_dest[ETH_ALEN];
15 unsigned char h_source[ETH_ALEN];
16 unsigned short h_proto;
17};
18
19struct {
20 __uint(type, BPF_MAP_TYPE_CGROUP_ARRAY);
21 __type(key, u32);
22 __type(value, u32);
23 __uint(pinning, LIBBPF_PIN_BY_NAME);
24 __uint(max_entries, 1);
25} test_cgrp2_array_pin SEC(".maps");
26
27SEC("filter")
28int handle_egress(struct __sk_buff *skb)
29{
30 void *data = (void *)(long)skb->data;
31 struct eth_hdr *eth = data;
32 struct ipv6hdr *ip6h = data + sizeof(*eth);
33 void *data_end = (void *)(long)skb->data_end;
34 char dont_care_msg[] = "dont care %04x %d\n";
35 char pass_msg[] = "pass\n";
36 char reject_msg[] = "reject\n";
37
38 /* single length check */
39 if (data + sizeof(*eth) + sizeof(*ip6h) > data_end)
40 return TC_ACT_OK;
41
42 if (eth->h_proto != bpf_htons(ETH_P_IPV6) ||
43 ip6h->nexthdr != IPPROTO_ICMPV6) {
44 bpf_trace_printk(dont_care_msg, sizeof(dont_care_msg),
45 eth->h_proto, ip6h->nexthdr);
46 return TC_ACT_OK;
47 } else if (bpf_skb_under_cgroup(skb, &test_cgrp2_array_pin, 0) != 1) {
48 bpf_trace_printk(pass_msg, sizeof(pass_msg));
49 return TC_ACT_OK;
50 } else {
51 bpf_trace_printk(reject_msg, sizeof(reject_msg));
52 return TC_ACT_SHOT;
53 }
54}
55
56char _license[] SEC("license") = "GPL";
samples/bpf/test_cgrp2_tc.sh+0−187
@@ -1,187 +0,0 @@
1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3
4MY_DIR=$(dirname $0)
5# Details on the bpf prog
6BPF_CGRP2_ARRAY_NAME='test_cgrp2_array_pin'
7BPF_PROG="$MY_DIR/test_cgrp2_tc.bpf.o"
8BPF_SECTION='filter'
9
10[ -z "$TC" ] && TC='tc'
11[ -z "$IP" ] && IP='ip'
12
13# Names of the veth interface, net namespace...etc.
14HOST_IFC='ve'
15NS_IFC='vens'
16NS='ns'
17
18find_mnt() {
19 cat /proc/mounts | \
20 awk '{ if ($3 == "'$1'" && mnt == "") { mnt = $2 }} END { print mnt }'
21}
22
23# Init cgroup2 vars
24init_cgrp2_vars() {
25 CGRP2_ROOT=$(find_mnt cgroup2)
26 if [ -z "$CGRP2_ROOT" ]
27 then
28 CGRP2_ROOT='/mnt/cgroup2'
29 MOUNT_CGRP2="yes"
30 fi
31 CGRP2_TC="$CGRP2_ROOT/tc"
32 CGRP2_TC_LEAF="$CGRP2_TC/leaf"
33}
34
35# Init bpf fs vars
36init_bpf_fs_vars() {
37 local bpf_fs_root=$(find_mnt bpf)
38 [ -n "$bpf_fs_root" ] || return -1
39 BPF_FS_TC_SHARE="$bpf_fs_root/tc/globals"
40}
41
42setup_cgrp2() {
43 case $1 in
44 start)
45 if [ "$MOUNT_CGRP2" == 'yes' ]
46 then
47 [ -d $CGRP2_ROOT ] || mkdir -p $CGRP2_ROOT
48 mount -t cgroup2 none $CGRP2_ROOT || return $?
49 fi
50 mkdir -p $CGRP2_TC_LEAF
51 ;;
52 *)
53 rmdir $CGRP2_TC_LEAF && rmdir $CGRP2_TC
54 [ "$MOUNT_CGRP2" == 'yes' ] && umount $CGRP2_ROOT
55 ;;
56 esac
57}
58
59setup_bpf_cgrp2_array() {
60 local bpf_cgrp2_array="$BPF_FS_TC_SHARE/$BPF_CGRP2_ARRAY_NAME"
61 case $1 in
62 start)
63 $MY_DIR/test_cgrp2_array_pin -U $bpf_cgrp2_array -v $CGRP2_TC
64 ;;
65 *)
66 [ -d "$BPF_FS_TC_SHARE" ] && rm -f $bpf_cgrp2_array
67 ;;
68 esac
69}
70
71setup_net() {
72 case $1 in
73 start)
74 $IP link add $HOST_IFC type veth peer name $NS_IFC || return $?
75 $IP link set dev $HOST_IFC up || return $?
76 sysctl -q net.ipv6.conf.$HOST_IFC.disable_ipv6=0
77 sysctl -q net.ipv6.conf.$HOST_IFC.accept_dad=0
78
79 $IP netns add $NS || return $?
80 $IP link set dev $NS_IFC netns $NS || return $?
81 $IP -n $NS link set dev $NS_IFC up || return $?
82 $IP netns exec $NS sysctl -q net.ipv6.conf.$NS_IFC.disable_ipv6=0
83 $IP netns exec $NS sysctl -q net.ipv6.conf.$NS_IFC.accept_dad=0
84 $TC qdisc add dev $HOST_IFC clsact || return $?
85 $TC filter add dev $HOST_IFC egress bpf da obj $BPF_PROG sec $BPF_SECTION || return $?
86 ;;
87 *)
88 $IP netns del $NS
89 $IP link del $HOST_IFC
90 ;;
91 esac
92}
93
94run_in_cgrp() {
95 # Fork another bash and move it under the specified cgroup.
96 # It makes the cgroup cleanup easier at the end of the test.
97 cmd='echo $$ > '
98 cmd="$cmd $1/cgroup.procs; exec $2"
99 bash -c "$cmd"
100}
101
102do_test() {
103 run_in_cgrp $CGRP2_TC_LEAF "ping -6 -c3 ff02::1%$HOST_IFC >& /dev/null"
104 local dropped=$($TC -s qdisc show dev $HOST_IFC | tail -3 | \
105 awk '/drop/{print substr($7, 0, index($7, ",")-1)}')
106 if [[ $dropped -eq 0 ]]
107 then
108 echo "FAIL"
109 return 1
110 else
111 echo "Successfully filtered $dropped packets"
112 return 0
113 fi
114}
115
116do_exit() {
117 if [ "$DEBUG" == "yes" ] && [ "$MODE" != 'cleanuponly' ]
118 then
119 echo "------ DEBUG ------"
120 echo "mount: "; mount | grep -E '(cgroup2|bpf)'; echo
121 echo "$CGRP2_TC_LEAF: "; ls -l $CGRP2_TC_LEAF; echo
122 if [ -d "$BPF_FS_TC_SHARE" ]
123 then
124 echo "$BPF_FS_TC_SHARE: "; ls -l $BPF_FS_TC_SHARE; echo
125 fi
126 echo "Host net:"
127 $IP netns
128 $IP link show dev $HOST_IFC
129 $IP -6 a show dev $HOST_IFC
130 $TC -s qdisc show dev $HOST_IFC
131 echo
132 echo "$NS net:"
133 $IP -n $NS link show dev $NS_IFC
134 $IP -n $NS -6 link show dev $NS_IFC
135 echo "------ DEBUG ------"
136 echo
137 fi
138
139 if [ "$MODE" != 'nocleanup' ]
140 then
141 setup_net stop
142 setup_bpf_cgrp2_array stop
143 setup_cgrp2 stop
144 fi
145}
146
147init_cgrp2_vars
148init_bpf_fs_vars
149
150while [[ $# -ge 1 ]]
151do
152 a="$1"
153 case $a in
154 debug)
155 DEBUG='yes'
156 shift 1
157 ;;
158 cleanup-only)
159 MODE='cleanuponly'
160 shift 1
161 ;;
162 no-cleanup)
163 MODE='nocleanup'
164 shift 1
165 ;;
166 *)
167 echo "test_cgrp2_tc [debug] [cleanup-only | no-cleanup]"
168 echo " debug: Print cgrp and network setup details at the end of the test"
169 echo " cleanup-only: Try to cleanup things from last test. No test will be run"
170 echo " no-cleanup: Run the test but don't do cleanup at the end"
171 echo "[Note: If no arg is given, it will run the test and do cleanup at the end]"
172 echo
173 exit -1
174 ;;
175 esac
176done
177
178trap do_exit 0
179
180[ "$MODE" == 'cleanuponly' ] && exit
181
182setup_cgrp2 start || exit $?
183setup_net start || exit $?
184init_bpf_fs_vars || exit $?
185setup_bpf_cgrp2_array start || exit $?
186do_test
187echo
samples/bpf/test_current_task_under_cgroup.bpf.c+0−43
@@ -1,43 +0,0 @@
1/* Copyright (c) 2016 Sargun Dhillon <sargun@sargun.me>
2 *
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 */
7
8#include "vmlinux.h"
9#include <linux/version.h>
10#include <bpf/bpf_helpers.h>
11#include <bpf/bpf_tracing.h>
12#include <bpf/bpf_core_read.h>
13
14struct {
15 __uint(type, BPF_MAP_TYPE_CGROUP_ARRAY);
16 __uint(key_size, sizeof(u32));
17 __uint(value_size, sizeof(u32));
18 __uint(max_entries, 1);
19} cgroup_map SEC(".maps");
20
21struct {
22 __uint(type, BPF_MAP_TYPE_ARRAY);
23 __type(key, u32);
24 __type(value, u64);
25 __uint(max_entries, 1);
26} perf_map SEC(".maps");
27
28/* Writes the last PID that called sync to a map at index 0 */
29SEC("ksyscall/sync")
30int BPF_KSYSCALL(bpf_prog1)
31{
32 u64 pid = bpf_get_current_pid_tgid();
33 int idx = 0;
34
35 if (!bpf_current_task_under_cgroup(&cgroup_map, 0))
36 return 0;
37
38 bpf_map_update_elem(&perf_map, &idx, &pid, BPF_ANY);
39 return 0;
40}
41
42char _license[] SEC("license") = "GPL";
43u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/test_current_task_under_cgroup_user.c+0−115
@@ -1,115 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2016 Sargun Dhillon <sargun@sargun.me>
3 */
4
5#define _GNU_SOURCE
6#include <stdio.h>
7#include <unistd.h>
8#include <bpf/bpf.h>
9#include <bpf/libbpf.h>
10#include "cgroup_helpers.h"
11
12#define CGROUP_PATH "/my-cgroup"
13
14int main(int argc, char **argv)
15{
16 pid_t remote_pid, local_pid = getpid();
17 int cg2 = -1, idx = 0, rc = 1;
18 struct bpf_link *link = NULL;
19 struct bpf_program *prog;
20 struct bpf_object *obj;
21 char filename[256];
22 int map_fd[2];
23
24 snprintf(filename, sizeof(filename), "%s.bpf.o", argv[0]);
25 obj = bpf_object__open_file(filename, NULL);
26 if (libbpf_get_error(obj)) {
27 fprintf(stderr, "ERROR: opening BPF object file failed\n");
28 return 0;
29 }
30
31 prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
32 if (!prog) {
33 printf("finding a prog in obj file failed\n");
34 goto cleanup;
35 }
36
37 /* load BPF program */
38 if (bpf_object__load(obj)) {
39 fprintf(stderr, "ERROR: loading BPF object file failed\n");
40 goto cleanup;
41 }
42
43 map_fd[0] = bpf_object__find_map_fd_by_name(obj, "cgroup_map");
44 map_fd[1] = bpf_object__find_map_fd_by_name(obj, "perf_map");
45 if (map_fd[0] < 0 || map_fd[1] < 0) {
46 fprintf(stderr, "ERROR: finding a map in obj file failed\n");
47 goto cleanup;
48 }
49
50 link = bpf_program__attach(prog);
51 if (libbpf_get_error(link)) {
52 fprintf(stderr, "ERROR: bpf_program__attach failed\n");
53 link = NULL;
54 goto cleanup;
55 }
56
57 if (setup_cgroup_environment())
58 goto err;
59
60 cg2 = create_and_get_cgroup(CGROUP_PATH);
61
62 if (cg2 < 0)
63 goto err;
64
65 if (bpf_map_update_elem(map_fd[0], &idx, &cg2, BPF_ANY)) {
66 log_err("Adding target cgroup to map");
67 goto err;
68 }
69
70 if (join_cgroup(CGROUP_PATH))
71 goto err;
72
73 /*
74 * The installed helper program catched the sync call, and should
75 * write it to the map.
76 */
77
78 sync();
79 bpf_map_lookup_elem(map_fd[1], &idx, &remote_pid);
80
81 if (local_pid != remote_pid) {
82 fprintf(stderr,
83 "BPF Helper didn't write correct PID to map, but: %d\n",
84 remote_pid);
85 goto err;
86 }
87
88 /* Verify the negative scenario; leave the cgroup */
89 if (join_cgroup("/"))
90 goto err;
91
92 remote_pid = 0;
93 bpf_map_update_elem(map_fd[1], &idx, &remote_pid, BPF_ANY);
94
95 sync();
96 bpf_map_lookup_elem(map_fd[1], &idx, &remote_pid);
97
98 if (local_pid == remote_pid) {
99 fprintf(stderr, "BPF cgroup negative test did not work\n");
100 goto err;
101 }
102
103 rc = 0;
104
105err:
106 if (cg2 != -1)
107 close(cg2);
108
109 cleanup_cgroup_environment();
110
111cleanup:
112 bpf_link__destroy(link);
113 bpf_object__close(obj);
114 return rc;
115}
samples/bpf/test_overhead_kprobe.bpf.c+0−41
@@ -1,41 +0,0 @@
1/* Copyright (c) 2016 Facebook
2 *
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 */
7#include "vmlinux.h"
8#include <linux/version.h>
9#include <bpf/bpf_helpers.h>
10#include <bpf/bpf_tracing.h>
11#include <bpf/bpf_core_read.h>
12
13SEC("kprobe/__set_task_comm")
14int prog(struct pt_regs *ctx)
15{
16 struct signal_struct *signal;
17 struct task_struct *tsk;
18 char oldcomm[TASK_COMM_LEN] = {};
19 char newcomm[TASK_COMM_LEN] = {};
20 u16 oom_score_adj;
21 u32 pid;
22
23 tsk = (void *)PT_REGS_PARM1_CORE(ctx);
24
25 pid = BPF_CORE_READ(tsk, pid);
26 bpf_core_read_str(oldcomm, sizeof(oldcomm), &tsk->comm);
27 bpf_core_read_str(newcomm, sizeof(newcomm),
28 (void *)PT_REGS_PARM2(ctx));
29 signal = BPF_CORE_READ(tsk, signal);
30 oom_score_adj = BPF_CORE_READ(signal, oom_score_adj);
31 return 0;
32}
33
34SEC("kprobe/fib_table_lookup")
35int prog2(struct pt_regs *ctx)
36{
37 return 0;
38}
39
40char _license[] SEC("license") = "GPL";
41u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/test_overhead_raw_tp.bpf.c+0−17
@@ -1,17 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2018 Facebook */
3#include "vmlinux.h"
4#include <bpf/bpf_helpers.h>
5
6SEC("raw_tracepoint/task_rename")
7int prog(struct bpf_raw_tracepoint_args *ctx)
8{
9 return 0;
10}
11
12SEC("raw_tracepoint/fib_table_lookup")
13int prog2(struct bpf_raw_tracepoint_args *ctx)
14{
15 return 0;
16}
17char _license[] SEC("license") = "GPL";
samples/bpf/test_overhead_tp.bpf.c+0−23
@@ -1,23 +0,0 @@
1/* Copyright (c) 2016 Facebook
2 *
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 */
7#include "vmlinux.h"
8#include <bpf/bpf_helpers.h>
9
10/* from /sys/kernel/tracing/events/task/task_rename/format */
11SEC("tracepoint/task/task_rename")
12int prog(struct trace_event_raw_task_rename *ctx)
13{
14 return 0;
15}
16
17/* from /sys/kernel/tracing/events/fib/fib_table_lookup/format */
18SEC("tracepoint/fib/fib_table_lookup")
19int prog2(struct trace_event_raw_fib_table_lookup *ctx)
20{
21 return 0;
22}
23char _license[] SEC("license") = "GPL";
samples/bpf/test_overhead_user.c+0−225
@@ -1,225 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2016 Facebook
3 */
4#define _GNU_SOURCE
5#include <sched.h>
6#include <errno.h>
7#include <stdio.h>
8#include <sys/types.h>
9#include <asm/unistd.h>
10#include <fcntl.h>
11#include <unistd.h>
12#include <assert.h>
13#include <sys/wait.h>
14#include <sys/socket.h>
15#include <arpa/inet.h>
16#include <stdlib.h>
17#include <signal.h>
18#include <linux/bpf.h>
19#include <string.h>
20#include <time.h>
21#include <bpf/bpf.h>
22#include <bpf/libbpf.h>
23
24#define MAX_CNT 1000000
25#define DUMMY_IP "127.0.0.1"
26#define DUMMY_PORT 80
27
28static struct bpf_link *links[2];
29static struct bpf_object *obj;
30static int cnt;
31
32static __u64 time_get_ns(void)
33{
34 struct timespec ts;
35
36 clock_gettime(CLOCK_MONOTONIC, &ts);
37 return ts.tv_sec * 1000000000ull + ts.tv_nsec;
38}
39
40static void test_task_rename(int cpu)
41{
42 char buf[] = "test\n";
43 __u64 start_time;
44 int i, fd;
45
46 fd = open("/proc/self/comm", O_WRONLY|O_TRUNC);
47 if (fd < 0) {
48 printf("couldn't open /proc\n");
49 exit(1);
50 }
51 start_time = time_get_ns();
52 for (i = 0; i < MAX_CNT; i++) {
53 if (write(fd, buf, sizeof(buf)) < 0) {
54 printf("task rename failed: %s\n", strerror(errno));
55 close(fd);
56 return;
57 }
58 }
59 printf("task_rename:%d: %lld events per sec\n",
60 cpu, MAX_CNT * 1000000000ll / (time_get_ns() - start_time));
61 close(fd);
62}
63
64static void test_fib_table_lookup(int cpu)
65{
66 struct sockaddr_in addr;
67 char buf[] = "test\n";
68 __u64 start_time;
69 int i, fd;
70
71 fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
72 if (fd < 0) {
73 printf("couldn't open socket\n");
74 exit(1);
75 }
76 memset((char *)&addr, 0, sizeof(addr));
77 addr.sin_addr.s_addr = inet_addr(DUMMY_IP);
78 addr.sin_port = htons(DUMMY_PORT);
79 addr.sin_family = AF_INET;
80 start_time = time_get_ns();
81 for (i = 0; i < MAX_CNT; i++) {
82 if (sendto(fd, buf, strlen(buf), 0,
83 (struct sockaddr *)&addr, sizeof(addr)) < 0) {
84 printf("failed to start ping: %s\n", strerror(errno));
85 close(fd);
86 return;
87 }
88 }
89 printf("fib_table_lookup:%d: %lld events per sec\n",
90 cpu, MAX_CNT * 1000000000ll / (time_get_ns() - start_time));
91 close(fd);
92}
93
94static void loop(int cpu, int flags)
95{
96 cpu_set_t cpuset;
97
98 CPU_ZERO(&cpuset);
99 CPU_SET(cpu, &cpuset);
100 sched_setaffinity(0, sizeof(cpuset), &cpuset);
101
102 if (flags & 1)
103 test_task_rename(cpu);
104 if (flags & 2)
105 test_fib_table_lookup(cpu);
106}
107
108static void run_perf_test(int tasks, int flags)
109{
110 pid_t pid[tasks];
111 int i;
112
113 for (i = 0; i < tasks; i++) {
114 pid[i] = fork();
115 if (pid[i] == 0) {
116 loop(i, flags);
117 exit(0);
118 } else if (pid[i] == -1) {
119 printf("couldn't spawn #%d process\n", i);
120 exit(1);
121 }
122 }
123 for (i = 0; i < tasks; i++) {
124 int status;
125
126 assert(waitpid(pid[i], &status, 0) == pid[i]);
127 assert(status == 0);
128 }
129}
130
131static int load_progs(char *filename)
132{
133 struct bpf_program *prog;
134 int err = 0;
135
136 obj = bpf_object__open_file(filename, NULL);
137 err = libbpf_get_error(obj);
138 if (err < 0) {
139 fprintf(stderr, "ERROR: opening BPF object file failed\n");
140 return err;
141 }
142
143 /* load BPF program */
144 err = bpf_object__load(obj);
145 if (err < 0) {
146 fprintf(stderr, "ERROR: loading BPF object file failed\n");
147 return err;
148 }
149
150 bpf_object__for_each_program(prog, obj) {
151 links[cnt] = bpf_program__attach(prog);
152 err = libbpf_get_error(links[cnt]);
153 if (err < 0) {
154 fprintf(stderr, "ERROR: bpf_program__attach failed\n");
155 links[cnt] = NULL;
156 return err;
157 }
158 cnt++;
159 }
160
161 return err;
162}
163
164static void unload_progs(void)
165{
166 while (cnt)
167 bpf_link__destroy(links[--cnt]);
168
169 bpf_object__close(obj);
170}
171
172int main(int argc, char **argv)
173{
174 int num_cpu = sysconf(_SC_NPROCESSORS_ONLN);
175 int test_flags = ~0;
176 char filename[256];
177 int err = 0;
178
179
180 if (argc > 1)
181 test_flags = atoi(argv[1]) ? : test_flags;
182 if (argc > 2)
183 num_cpu = atoi(argv[2]) ? : num_cpu;
184
185 if (test_flags & 0x3) {
186 printf("BASE\n");
187 run_perf_test(num_cpu, test_flags);
188 }
189
190 if (test_flags & 0xC) {
191 snprintf(filename, sizeof(filename),
192 "%s_kprobe.bpf.o", argv[0]);
193
194 printf("w/KPROBE\n");
195 err = load_progs(filename);
196 if (!err)
197 run_perf_test(num_cpu, test_flags >> 2);
198
199 unload_progs();
200 }
201
202 if (test_flags & 0x30) {
203 snprintf(filename, sizeof(filename),
204 "%s_tp.bpf.o", argv[0]);
205 printf("w/TRACEPOINT\n");
206 err = load_progs(filename);
207 if (!err)
208 run_perf_test(num_cpu, test_flags >> 4);
209
210 unload_progs();
211 }
212
213 if (test_flags & 0xC0) {
214 snprintf(filename, sizeof(filename),
215 "%s_raw_tp.bpf.o", argv[0]);
216 printf("w/RAW_TRACEPOINT\n");
217 err = load_progs(filename);
218 if (!err)
219 run_perf_test(num_cpu, test_flags >> 6);
220
221 unload_progs();
222 }
223
224 return err;
225}
samples/bpf/test_override_return.sh+0−16
@@ -1,16 +0,0 @@
1#!/bin/bash
2
3rm -r tmpmnt
4rm -f testfile.img
5dd if=/dev/zero of=testfile.img bs=1M seek=1000 count=1
6DEVICE=$(losetup --show -f testfile.img)
7mkfs.btrfs -f $DEVICE
8mkdir tmpmnt
9./tracex7 $DEVICE
10if [ $? -eq 0 ]
11then
12 echo "SUCCESS!"
13else
14 echo "FAILED!"
15fi
16losetup -d $DEVICE
samples/bpf/test_probe_write_user.bpf.c+0−52
@@ -1,52 +0,0 @@
1/* Copyright (c) 2016 Sargun Dhillon <sargun@sargun.me>
2 *
3 * This program is free software; you can redistribute it and/or
4 * modify it under the terms of version 2 of the GNU General Public
5 * License as published by the Free Software Foundation.
6 */
7#include "vmlinux.h"
8#include <string.h>
9#include <linux/version.h>
10#include <bpf/bpf_helpers.h>
11#include <bpf/bpf_tracing.h>
12#include <bpf/bpf_core_read.h>
13
14struct {
15 __uint(type, BPF_MAP_TYPE_HASH);
16 __type(key, struct sockaddr_in);
17 __type(value, struct sockaddr_in);
18 __uint(max_entries, 256);
19} dnat_map SEC(".maps");
20
21/* kprobe is NOT a stable ABI
22 * kernel functions can be removed, renamed or completely change semantics.
23 * Number of arguments and their positions can change, etc.
24 * In such case this bpf+kprobe example will no longer be meaningful
25 *
26 * This example sits on a syscall, and the syscall ABI is relatively stable
27 * of course, across platforms, and over time, the ABI may change.
28 */
29SEC("ksyscall/connect")
30int BPF_KSYSCALL(bpf_prog1, int fd, struct sockaddr_in *uservaddr,
31 int addrlen)
32{
33 struct sockaddr_in new_addr, orig_addr = {};
34 struct sockaddr_in *mapped_addr;
35
36 if (addrlen > sizeof(orig_addr))
37 return 0;
38
39 if (bpf_probe_read_user(&orig_addr, sizeof(orig_addr), uservaddr) != 0)
40 return 0;
41
42 mapped_addr = bpf_map_lookup_elem(&dnat_map, &orig_addr);
43 if (mapped_addr != NULL) {
44 memcpy(&new_addr, mapped_addr, sizeof(new_addr));
45 bpf_probe_write_user(uservaddr, &new_addr,
46 sizeof(new_addr));
47 }
48 return 0;
49}
50
51char _license[] SEC("license") = "GPL";
52u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/test_probe_write_user_user.c+0−108
@@ -1,108 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <stdio.h>
3#include <assert.h>
4#include <unistd.h>
5#include <bpf/bpf.h>
6#include <bpf/libbpf.h>
7#include <sys/socket.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10
11int main(int ac, char **argv)
12{
13 struct sockaddr_in *serv_addr_in, *mapped_addr_in, *tmp_addr_in;
14 struct sockaddr serv_addr, mapped_addr, tmp_addr;
15 int serverfd, serverconnfd, clientfd, map_fd;
16 struct bpf_link *link = NULL;
17 struct bpf_program *prog;
18 struct bpf_object *obj;
19 socklen_t sockaddr_len;
20 char filename[256];
21 char *ip;
22
23 serv_addr_in = (struct sockaddr_in *)&serv_addr;
24 mapped_addr_in = (struct sockaddr_in *)&mapped_addr;
25 tmp_addr_in = (struct sockaddr_in *)&tmp_addr;
26
27 snprintf(filename, sizeof(filename), "%s.bpf.o", argv[0]);
28 obj = bpf_object__open_file(filename, NULL);
29 if (libbpf_get_error(obj)) {
30 fprintf(stderr, "ERROR: opening BPF object file failed\n");
31 return 0;
32 }
33
34 prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
35 if (libbpf_get_error(prog)) {
36 fprintf(stderr, "ERROR: finding a prog in obj file failed\n");
37 goto cleanup;
38 }
39
40 /* load BPF program */
41 if (bpf_object__load(obj)) {
42 fprintf(stderr, "ERROR: loading BPF object file failed\n");
43 goto cleanup;
44 }
45
46 map_fd = bpf_object__find_map_fd_by_name(obj, "dnat_map");
47 if (map_fd < 0) {
48 fprintf(stderr, "ERROR: finding a map in obj file failed\n");
49 goto cleanup;
50 }
51
52 link = bpf_program__attach(prog);
53 if (libbpf_get_error(link)) {
54 fprintf(stderr, "ERROR: bpf_program__attach failed\n");
55 link = NULL;
56 goto cleanup;
57 }
58
59 assert((serverfd = socket(AF_INET, SOCK_STREAM, 0)) > 0);
60 assert((clientfd = socket(AF_INET, SOCK_STREAM, 0)) > 0);
61
62 /* Bind server to ephemeral port on lo */
63 memset(&serv_addr, 0, sizeof(serv_addr));
64 serv_addr_in->sin_family = AF_INET;
65 serv_addr_in->sin_port = 0;
66 serv_addr_in->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
67
68 assert(bind(serverfd, &serv_addr, sizeof(serv_addr)) == 0);
69
70 sockaddr_len = sizeof(serv_addr);
71 assert(getsockname(serverfd, &serv_addr, &sockaddr_len) == 0);
72 ip = inet_ntoa(serv_addr_in->sin_addr);
73 printf("Server bound to: %s:%d\n", ip, ntohs(serv_addr_in->sin_port));
74
75 memset(&mapped_addr, 0, sizeof(mapped_addr));
76 mapped_addr_in->sin_family = AF_INET;
77 mapped_addr_in->sin_port = htons(5555);
78 mapped_addr_in->sin_addr.s_addr = inet_addr("255.255.255.255");
79
80 assert(!bpf_map_update_elem(map_fd, &mapped_addr, &serv_addr, BPF_ANY));
81
82 assert(listen(serverfd, 5) == 0);
83
84 ip = inet_ntoa(mapped_addr_in->sin_addr);
85 printf("Client connecting to: %s:%d\n",
86 ip, ntohs(mapped_addr_in->sin_port));
87 assert(connect(clientfd, &mapped_addr, sizeof(mapped_addr)) == 0);
88
89 sockaddr_len = sizeof(tmp_addr);
90 ip = inet_ntoa(tmp_addr_in->sin_addr);
91 assert((serverconnfd = accept(serverfd, &tmp_addr, &sockaddr_len)) > 0);
92 printf("Server received connection from: %s:%d\n",
93 ip, ntohs(tmp_addr_in->sin_port));
94
95 sockaddr_len = sizeof(tmp_addr);
96 assert(getpeername(clientfd, &tmp_addr, &sockaddr_len) == 0);
97 ip = inet_ntoa(tmp_addr_in->sin_addr);
98 printf("Client's peer address: %s:%d\n",
99 ip, ntohs(tmp_addr_in->sin_port));
100
101 /* Is the server's getsockname = the socket getpeername */
102 assert(memcmp(&serv_addr, &tmp_addr, sizeof(struct sockaddr_in)) == 0);
103
104cleanup:
105 bpf_link__destroy(link);
106 bpf_object__close(obj);
107 return 0;
108}
samples/bpf/tracex7.bpf.c+0−15
@@ -1,15 +0,0 @@
1#include "vmlinux.h"
2#include <linux/version.h>
3#include <bpf/bpf_helpers.h>
4
5SEC("kprobe/open_ctree")
6int bpf_prog1(struct pt_regs *ctx)
7{
8 unsigned long rc = -12;
9
10 bpf_override_return(ctx, rc);
11 return 0;
12}
13
14char _license[] SEC("license") = "GPL";
15u32 _version SEC("version") = LINUX_VERSION_CODE;
samples/bpf/tracex7_user.c+0−56
@@ -1,56 +0,0 @@
1#define _GNU_SOURCE
2
3#include <stdio.h>
4#include <unistd.h>
5#include <bpf/libbpf.h>
6
7int main(int argc, char **argv)
8{
9 struct bpf_link *link = NULL;
10 struct bpf_program *prog;
11 struct bpf_object *obj;
12 char filename[256];
13 char command[256];
14 int ret = 0;
15 FILE *f;
16
17 if (!argv[1]) {
18 fprintf(stderr, "ERROR: Run with the btrfs device argument!\n");
19 return 0;
20 }
21
22 snprintf(filename, sizeof(filename), "%s.bpf.o", argv[0]);
23 obj = bpf_object__open_file(filename, NULL);
24 if (libbpf_get_error(obj)) {
25 fprintf(stderr, "ERROR: opening BPF object file failed\n");
26 return 0;
27 }
28
29 prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
30 if (!prog) {
31 fprintf(stderr, "ERROR: finding a prog in obj file failed\n");
32 goto cleanup;
33 }
34
35 /* load BPF program */
36 if (bpf_object__load(obj)) {
37 fprintf(stderr, "ERROR: loading BPF object file failed\n");
38 goto cleanup;
39 }
40
41 link = bpf_program__attach(prog);
42 if (libbpf_get_error(link)) {
43 fprintf(stderr, "ERROR: bpf_program__attach failed\n");
44 link = NULL;
45 goto cleanup;
46 }
47
48 snprintf(command, 256, "mount %s tmpmnt/", argv[1]);
49 f = popen(command, "r");
50 ret = pclose(f);
51
52cleanup:
53 bpf_link__destroy(link);
54 bpf_object__close(obj);
55 return ret ? 0 : 1;
56}
samples/bpf/xdp2skb_meta_kern.c+1−1
@@ -32,7 +32,7 @@ SEC("xdp_mark") · 31 unmodified lines
32int _xdp_mark(struct xdp_md *ctx)32int _xdp_mark(struct xdp_md *ctx)
33{33{
34 struct meta_info *meta;34 struct meta_info *meta;
35 void *data, *data_end;35 void *data;
36 int ret;36 int ret;
37 37
38 /* Reserve space in-front of data pointer for our meta info.38 /* Reserve space in-front of data pointer for our meta info.
samples/bpf/xdp_adjust_tail_kern.c+1−0
@@ -57,6 +57,7 @@ static __always_inline void swap_mac(void *data, struct ethhdr *orig_eth) · 56 unmodified lines
57 57
58static __always_inline __u16 csum_fold_helper(__u32 csum)58static __always_inline __u16 csum_fold_helper(__u32 csum)
59{59{
60 csum = (csum & 0xffff) + (csum >> 16);
60 return ~((csum & 0xffff) + (csum >> 16));61 return ~((csum & 0xffff) + (csum >> 16));
61}62}
62 63
scripts/Makefile.btf+4−2
@@ -3,6 +3,8 @@ · 2 unmodified lines
3pahole-ver := $(CONFIG_PAHOLE_VERSION)3pahole-ver := $(CONFIG_PAHOLE_VERSION)
4pahole-flags-y :=4pahole-flags-y :=
5 5
6JOBS := $(patsubst -j%,%,$(filter -j%,$(MAKEFLAGS)))
7
6ifeq ($(call test-le, $(pahole-ver), 125),y)8ifeq ($(call test-le, $(pahole-ver), 125),y)
7 9
8# pahole 1.18 through 1.21 can't handle zero-sized per-CPU vars10# pahole 1.18 through 1.21 can't handle zero-sized per-CPU vars
@@ -12,14 +14,14 @@ endif · 3 unmodified lines
12 14
13pahole-flags-$(call test-ge, $(pahole-ver), 121) += --btf_gen_floats15pahole-flags-$(call test-ge, $(pahole-ver), 121) += --btf_gen_floats
14 16
15pahole-flags-$(call test-ge, $(pahole-ver), 122) += -j17pahole-flags-$(call test-ge, $(pahole-ver), 122) += -j$(JOBS)
16 18
17pahole-flags-$(call test-ge, $(pahole-ver), 125) += --skip_encoding_btf_inconsistent_proto --btf_gen_optimized19pahole-flags-$(call test-ge, $(pahole-ver), 125) += --skip_encoding_btf_inconsistent_proto --btf_gen_optimized
18 20
19else21else
20 22
21# Switch to using --btf_features for v1.26 and later.23# Switch to using --btf_features for v1.26 and later.
22pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs24pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs
23 25
24ifneq ($(KBUILD_EXTMOD),)26ifneq ($(KBUILD_EXTMOD),)
25module-pahole-flags-$(call test-ge, $(pahole-ver), 126) += --btf_features=distilled_base27module-pahole-flags-$(call test-ge, $(pahole-ver), 126) += --btf_features=distilled_base
scripts/bpf_doc.py+50−3
@@ -37,10 +37,11 @@ class APIElement(object): · 36 unmodified lines
37 @desc: textual description of the symbol37 @desc: textual description of the symbol
38 @ret: (optional) description of any associated return value38 @ret: (optional) description of any associated return value
39 """39 """
40 def __init__(self, proto='', desc='', ret=''):40 def __init__(self, proto='', desc='', ret='', attrs=[]):
41 self.proto = proto41 self.proto = proto
42 self.desc = desc42 self.desc = desc
43 self.ret = ret43 self.ret = ret
44 self.attrs = attrs
44 45
45 46
46class Helper(APIElement):47class Helper(APIElement):
@@ -81,6 +82,11 @@ class Helper(APIElement): · 34 unmodified lines
81 return res82 return res
82 83
83 84
85ATTRS = {
86 '__bpf_fastcall': 'bpf_fastcall'
87}
88
89
84class HeaderParser(object):90class HeaderParser(object):
85 """91 """
86 An object used to parse a file in order to extract the documentation of a92 An object used to parse a file in order to extract the documentation of a
@@ -111,7 +117,8 @@ class HeaderParser(object): · 24 unmodified lines
111 proto = self.parse_proto()117 proto = self.parse_proto()
112 desc = self.parse_desc(proto)118 desc = self.parse_desc(proto)
113 ret = self.parse_ret(proto)119 ret = self.parse_ret(proto)
114 return Helper(proto=proto, desc=desc, ret=ret)120 attrs = self.parse_attrs(proto)
121 return Helper(proto=proto, desc=desc, ret=ret, attrs=attrs)
115 122
116 def parse_symbol(self):123 def parse_symbol(self):
117 p = re.compile(r' \* ?(BPF\w+)$')124 p = re.compile(r' \* ?(BPF\w+)$')
@@ -192,6 +199,28 @@ class HeaderParser(object): · 74 unmodified lines
192 raise Exception("No return found for " + proto)199 raise Exception("No return found for " + proto)
193 return ret200 return ret
194 201
202 def parse_attrs(self, proto):
203 p = re.compile(r' \* ?(?:\t| {5,8})Attributes$')
204 capture = p.match(self.line)
205 if not capture:
206 return []
207 # Expect a single line with mnemonics for attributes separated by spaces
208 self.line = self.reader.readline()
209 p = re.compile(r' \* ?(?:\t| {5,8})(?:\t| {8})(.*)')
210 capture = p.match(self.line)
211 if not capture:
212 raise Exception("Incomplete 'Attributes' section for " + proto)
213 attrs = capture.group(1).split(' ')
214 for attr in attrs:
215 if attr not in ATTRS:
216 raise Exception("Unexpected attribute '" + attr + "' specified for " + proto)
217 self.line = self.reader.readline()
218 if self.line != ' *\n':
219 raise Exception("Expecting empty line after 'Attributes' section for " + proto)
220 # Prepare a line for next self.parse_* to consume
221 self.line = self.reader.readline()
222 return attrs
223
195 def seek_to(self, target, help_message, discard_lines = 1):224 def seek_to(self, target, help_message, discard_lines = 1):
196 self.reader.seek(0)225 self.reader.seek(0)
197 offset = self.reader.read().find(target)226 offset = self.reader.read().find(target)
@@ -789,6 +818,21 @@ class PrinterHelpers(Printer): · 591 unmodified lines
789 print('%s;' % fwd)818 print('%s;' % fwd)
790 print('')819 print('')
791 820
821 used_attrs = set()
822 for helper in self.elements:
823 for attr in helper.attrs:
824 used_attrs.add(attr)
825 for attr in sorted(used_attrs):
826 print('#ifndef %s' % attr)
827 print('#if __has_attribute(%s)' % ATTRS[attr])
828 print('#define %s __attribute__((%s))' % (attr, ATTRS[attr]))
829 print('#else')
830 print('#define %s' % attr)
831 print('#endif')
832 print('#endif')
833 if used_attrs:
834 print('')
835
792 def print_footer(self):836 def print_footer(self):
793 footer = ''837 footer = ''
794 print(footer)838 print(footer)
@@ -827,7 +871,10 @@ class PrinterHelpers(Printer): · 32 unmodified lines
827 print(' *{}{}'.format(' \t' if line else '', line))871 print(' *{}{}'.format(' \t' if line else '', line))
828 872
829 print(' */')873 print(' */')
830 print('static %s %s(* const %s)(' % (self.map_type(proto['ret_type']),874 print('static ', end='')
875 if helper.attrs:
876 print('%s ' % (" ".join(helper.attrs)), end='')
877 print('%s %s(* const %s)(' % (self.map_type(proto['ret_type']),
831 proto['ret_star'], proto['name']), end='')878 proto['ret_star'], proto['name']), end='')
832 comma = ''879 comma = ''
833 for i, a in enumerate(proto['args']):880 for i, a in enumerate(proto['args']):
tools/bpf/bpf_jit_disasm.c+1−1
@@ -210,7 +210,7 @@ static uint8_t *get_last_jit_image(char *haystack, size_t hlen, · 209 unmodified lines
210 return NULL;210 return NULL;
211 }211 }
212 if (proglen > 1000000) {212 if (proglen > 1000000) {
213 printf("proglen of %d too big, stopping\n", proglen);213 printf("proglen of %u too big, stopping\n", proglen);
214 return NULL;214 return NULL;
215 }215 }
216 216
tools/bpf/bpftool/Makefile+5−1
@@ -147,7 +147,11 @@ ifeq ($(feature-llvm),1) · 146 unmodified lines
147 # If LLVM is available, use it for JIT disassembly147 # If LLVM is available, use it for JIT disassembly
148 CFLAGS += -DHAVE_LLVM_SUPPORT148 CFLAGS += -DHAVE_LLVM_SUPPORT
149 LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets149 LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
150 CFLAGS += $(shell $(LLVM_CONFIG) --cflags)150 # llvm-config always adds -D_GNU_SOURCE, however, it may already be in CFLAGS
151 # (e.g. when bpftool build is called from selftests build as selftests
152 # Makefile includes lib.mk which sets -D_GNU_SOURCE) which would cause
153 # compilation error due to redefinition. Let's filter it out here.
154 CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
151 LIBS += $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))155 LIBS += $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))
152 ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static)156 ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static)
153 LIBS += $(shell $(LLVM_CONFIG) --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))157 LIBS += $(shell $(LLVM_CONFIG) --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
tools/bpf/bpftool/btf.c+89−11
@@ -1,11 +1,15 @@
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2/* Copyright (C) 2019 Facebook */2/* Copyright (C) 2019 Facebook */
3 3
4#ifndef _GNU_SOURCE
5#define _GNU_SOURCE
6#endif
4#include <errno.h>7#include <errno.h>
5#include <fcntl.h>8#include <fcntl.h>
6#include <linux/err.h>9#include <linux/err.h>
7#include <stdbool.h>10#include <stdbool.h>
8#include <stdio.h>11#include <stdio.h>
12#include <stdlib.h>
9#include <string.h>13#include <string.h>
10#include <unistd.h>14#include <unistd.h>
11#include <linux/btf.h>15#include <linux/btf.h>
@@ -21,6 +25,7 @@ · 9 unmodified lines
21#include "main.h"25#include "main.h"
22 26
23#define KFUNC_DECL_TAG "bpf_kfunc"27#define KFUNC_DECL_TAG "bpf_kfunc"
28#define FASTCALL_DECL_TAG "bpf_fastcall"
24 29
25static const char * const btf_kind_str[NR_BTF_KINDS] = {30static const char * const btf_kind_str[NR_BTF_KINDS] = {
26 [BTF_KIND_UNKN] = "UNKNOWN",31 [BTF_KIND_UNKN] = "UNKNOWN",
@@ -284,7 +289,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, · 257 unmodified lines
284 } else {289 } else {
285 if (btf_kflag(t))290 if (btf_kflag(t))
286 printf("\n\t'%s' val=%lldLL", name,291 printf("\n\t'%s' val=%lldLL", name,
287 (unsigned long long)val);292 (long long)val);
288 else293 else
289 printf("\n\t'%s' val=%lluULL", name,294 printf("\n\t'%s' val=%lluULL", name,
290 (unsigned long long)val);295 (unsigned long long)val);
@@ -464,19 +469,59 @@ static int dump_btf_raw(const struct btf *btf, · 173 unmodified lines
464 return 0;469 return 0;
465}470}
466 471
472struct ptr_array {
473 __u32 cnt;
474 __u32 cap;
475 const void **elems;
476};
477
478static int ptr_array_push(const void *ptr, struct ptr_array *arr)
479{
480 __u32 new_cap;
481 void *tmp;
482
483 if (arr->cnt == arr->cap) {
484 new_cap = (arr->cap ?: 16) * 2;
485 tmp = realloc(arr->elems, sizeof(*arr->elems) * new_cap);
486 if (!tmp)
487 return -ENOMEM;
488 arr->elems = tmp;
489 arr->cap = new_cap;
490 }
491 arr->elems[arr->cnt++] = ptr;
492 return 0;
493}
494
495static void ptr_array_free(struct ptr_array *arr)
496{
497 free(arr->elems);
498}
499
500static int cmp_kfuncs(const void *pa, const void *pb, void *ctx)
501{
502 struct btf *btf = ctx;
503 const struct btf_type *a = *(void **)pa;
504 const struct btf_type *b = *(void **)pb;
505
506 return strcmp(btf__str_by_offset(btf, a->name_off),
507 btf__str_by_offset(btf, b->name_off));
508}
509
467static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf)510static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf)
468{511{
469 LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts);512 LIBBPF_OPTS(btf_dump_emit_type_decl_opts, opts);
470 int cnt = btf__type_cnt(btf);513 __u32 cnt = btf__type_cnt(btf), i, j;
471 int i;514 struct ptr_array fastcalls = {};
515 struct ptr_array kfuncs = {};
516 int err = 0;
472 517
473 printf("\n/* BPF kfuncs */\n");518 printf("\n/* BPF kfuncs */\n");
474 printf("#ifndef BPF_NO_KFUNC_PROTOTYPES\n");519 printf("#ifndef BPF_NO_KFUNC_PROTOTYPES\n");
475 520
476 for (i = 1; i < cnt; i++) {521 for (i = 1; i < cnt; i++) {
477 const struct btf_type *t = btf__type_by_id(btf, i);522 const struct btf_type *t = btf__type_by_id(btf, i);
523 const struct btf_type *ft;
478 const char *name;524 const char *name;
479 int err;
480 525
481 if (!btf_is_decl_tag(t))526 if (!btf_is_decl_tag(t))
482 continue;527 continue;
@@ -484,27 +529,53 @@ static int dump_btf_kfuncs(struct btf_dump *d, const struct btf *btf) · 1 unmodified line
484 if (btf_decl_tag(t)->component_idx != -1)529 if (btf_decl_tag(t)->component_idx != -1)
485 continue;530 continue;
486 531
487 name = btf__name_by_offset(btf, t->name_off);532 ft = btf__type_by_id(btf, t->type);
488 if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)))533 if (!btf_is_func(ft))
489 continue;534 continue;
490 535
491 t = btf__type_by_id(btf, t->type);536 name = btf__name_by_offset(btf, t->name_off);
492 if (!btf_is_func(t))537 if (strncmp(name, KFUNC_DECL_TAG, sizeof(KFUNC_DECL_TAG)) == 0) {
493 continue;538 err = ptr_array_push(ft, &kfuncs);
539 if (err)
540 goto out;
541 }
542
543 if (strncmp(name, FASTCALL_DECL_TAG, sizeof(FASTCALL_DECL_TAG)) == 0) {
544 err = ptr_array_push(ft, &fastcalls);
545 if (err)
546 goto out;
547 }
548 }
549
550 /* Sort kfuncs by name for improved vmlinux.h stability */
551 qsort_r(kfuncs.elems, kfuncs.cnt, sizeof(*kfuncs.elems), cmp_kfuncs, (void *)btf);
552 for (i = 0; i < kfuncs.cnt; i++) {
553 const struct btf_type *t = kfuncs.elems[i];
494 554
495 printf("extern ");555 printf("extern ");
496 556
557 /* Assume small amount of fastcall kfuncs */
558 for (j = 0; j < fastcalls.cnt; j++) {
559 if (fastcalls.elems[j] == t) {
560 printf("__bpf_fastcall ");
561 break;
562 }
563 }
564
497 opts.field_name = btf__name_by_offset(btf, t->name_off);565 opts.field_name = btf__name_by_offset(btf, t->name_off);
498 err = btf_dump__emit_type_decl(d, t->type, &opts);566 err = btf_dump__emit_type_decl(d, t->type, &opts);
499 if (err)567 if (err)
500 return err;568 goto out;
501 569
502 printf(" __weak __ksym;\n");570 printf(" __weak __ksym;\n");
503 }571 }
504 572
505 printf("#endif\n\n");573 printf("#endif\n\n");
506 574
507 return 0;575out:
576 ptr_array_free(&fastcalls);
577 ptr_array_free(&kfuncs);
578 return err;
508}579}
509 580
510static void __printf(2, 0) btf_dump_printf(void *ctx,581static void __printf(2, 0) btf_dump_printf(void *ctx,
@@ -718,6 +789,13 @@ static int dump_btf_c(const struct btf *btf, · 207 unmodified lines
718 printf("#ifndef __weak\n");789 printf("#ifndef __weak\n");
719 printf("#define __weak __attribute__((weak))\n");790 printf("#define __weak __attribute__((weak))\n");
720 printf("#endif\n\n");791 printf("#endif\n\n");
792 printf("#ifndef __bpf_fastcall\n");
793 printf("#if __has_attribute(bpf_fastcall)\n");
794 printf("#define __bpf_fastcall __attribute__((bpf_fastcall))\n");
795 printf("#else\n");
796 printf("#define __bpf_fastcall\n");
797 printf("#endif\n");
798 printf("#endif\n\n");
721 799
722 if (root_type_cnt) {800 if (root_type_cnt) {
723 for (i = 0; i < root_type_cnt; i++) {801 for (i = 0; i < root_type_cnt; i++) {
tools/bpf/bpftool/jit_disasm.c+29−11
@@ -80,7 +80,8 @@ symbol_lookup_callback(__maybe_unused void *disasm_info, · 79 unmodified lines
80static int80static int
81init_context(disasm_ctx_t *ctx, const char *arch,81init_context(disasm_ctx_t *ctx, const char *arch,
82 __maybe_unused const char *disassembler_options,82 __maybe_unused const char *disassembler_options,
83 __maybe_unused unsigned char *image, __maybe_unused ssize_t len)83 __maybe_unused unsigned char *image, __maybe_unused ssize_t len,
84 __maybe_unused __u64 func_ksym)
84{85{
85 char *triple;86 char *triple;
86 87
@@ -109,12 +110,13 @@ static void destroy_context(disasm_ctx_t *ctx) · 22 unmodified lines
109}110}
110 111
111static int112static int
112disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc)113disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc,
114 __u64 func_ksym)
113{115{
114 char buf[256];116 char buf[256];
115 int count;117 int count;
116 118
117 count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, pc,119 count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, func_ksym + pc,
118 buf, sizeof(buf));120 buf, sizeof(buf));
119 if (json_output)121 if (json_output)
120 printf_json(buf);122 printf_json(buf);
@@ -136,8 +138,21 @@ int disasm_init(void) · 15 unmodified lines
136#ifdef HAVE_LIBBFD_SUPPORT138#ifdef HAVE_LIBBFD_SUPPORT
137#define DISASM_SPACER "\t"139#define DISASM_SPACER "\t"
138 140
141struct disasm_info {
142 struct disassemble_info info;
143 __u64 func_ksym;
144};
145
146static void disasm_print_addr(bfd_vma addr, struct disassemble_info *info)
147{
148 struct disasm_info *dinfo = container_of(info, struct disasm_info, info);
149
150 addr += dinfo->func_ksym;
151 generic_print_address(addr, info);
152}
153
139typedef struct {154typedef struct {
140 struct disassemble_info *info;155 struct disasm_info *info;
141 disassembler_ftype disassemble;156 disassembler_ftype disassemble;
142 bfd *bfdf;157 bfd *bfdf;
143} disasm_ctx_t;158} disasm_ctx_t;
@@ -215,7 +230,7 @@ static int fprintf_json_styled(void *out, · 71 unmodified lines
215 230
216static int init_context(disasm_ctx_t *ctx, const char *arch,231static int init_context(disasm_ctx_t *ctx, const char *arch,
217 const char *disassembler_options,232 const char *disassembler_options,
218 unsigned char *image, ssize_t len)233 unsigned char *image, ssize_t len, __u64 func_ksym)
219{234{
220 struct disassemble_info *info;235 struct disassemble_info *info;
221 char tpath[PATH_MAX];236 char tpath[PATH_MAX];
@@ -238,12 +253,13 @@ static int init_context(disasm_ctx_t *ctx, const char *arch, · 16 unmodified lines
238 }253 }
239 bfdf = ctx->bfdf;254 bfdf = ctx->bfdf;
240 255
241 ctx->info = malloc(sizeof(struct disassemble_info));256 ctx->info = malloc(sizeof(struct disasm_info));
242 if (!ctx->info) {257 if (!ctx->info) {
243 p_err("mem alloc failed");258 p_err("mem alloc failed");
244 goto err_close;259 goto err_close;
245 }260 }
246 info = ctx->info;261 ctx->info->func_ksym = func_ksym;
262 info = &ctx->info->info;
247 263
248 if (json_output)264 if (json_output)
249 init_disassemble_info_compat(info, stdout,265 init_disassemble_info_compat(info, stdout,
@@ -272,6 +288,7 @@ static int init_context(disasm_ctx_t *ctx, const char *arch, · 22 unmodified lines
272 info->disassembler_options = disassembler_options;288 info->disassembler_options = disassembler_options;
273 info->buffer = image;289 info->buffer = image;
274 info->buffer_length = len;290 info->buffer_length = len;
291 info->print_address_func = disasm_print_addr;
275 292
276 disassemble_init_for_target(info);293 disassemble_init_for_target(info);
277 294
@@ -304,9 +321,10 @@ static void destroy_context(disasm_ctx_t *ctx) · 26 unmodified lines
304 321
305static int322static int
306disassemble_insn(disasm_ctx_t *ctx, __maybe_unused unsigned char *image,323disassemble_insn(disasm_ctx_t *ctx, __maybe_unused unsigned char *image,
307 __maybe_unused ssize_t len, int pc)324 __maybe_unused ssize_t len, int pc,
325 __maybe_unused __u64 func_ksym)
308{326{
309 return ctx->disassemble(pc, ctx->info);327 return ctx->disassemble(pc, &ctx->info->info);
310}328}
311 329
312int disasm_init(void)330int disasm_init(void)
@@ -331,7 +349,7 @@ int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, · 18 unmodified lines
331 if (!len)349 if (!len)
332 return -1;350 return -1;
333 351
334 if (init_context(&ctx, arch, disassembler_options, image, len))352 if (init_context(&ctx, arch, disassembler_options, image, len, func_ksym))
335 return -1;353 return -1;
336 354
337 if (json_output)355 if (json_output)
@@ -360,7 +378,7 @@ int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, · 22 unmodified lines
360 printf("%4x:" DISASM_SPACER, pc);378 printf("%4x:" DISASM_SPACER, pc);
361 }379 }
362 380
363 count = disassemble_insn(&ctx, image, len, pc);381 count = disassemble_insn(&ctx, image, len, pc, func_ksym);
364 382
365 if (json_output) {383 if (json_output) {
366 /* Operand array, was started in fprintf_json. Before384 /* Operand array, was started in fprintf_json. Before
tools/bpf/resolve_btfids/main.c+2−2
@@ -679,8 +679,8 @@ static int sets_patch(struct object *obj) · 678 unmodified lines
679 679
680 next = rb_first(&obj->sets);680 next = rb_first(&obj->sets);
681 while (next) {681 while (next) {
682 struct btf_id_set8 *set8;682 struct btf_id_set8 *set8 = NULL;
683 struct btf_id_set *set;683 struct btf_id_set *set = NULL;
684 unsigned long addr, off;684 unsigned long addr, off;
685 struct btf_id *id;685 struct btf_id *id;
686 686
tools/bpf/runqslower/runqslower.bpf.c+0−1
@@ -70,7 +70,6 @@ int handle__sched_switch(u64 *ctx) · 69 unmodified lines
70 struct task_struct *next = (struct task_struct *)ctx[2];70 struct task_struct *next = (struct task_struct *)ctx[2];
71 struct runq_event event = {};71 struct runq_event event = {};
72 u64 *tsp, delta_us;72 u64 *tsp, delta_us;
73 long state;
74 u32 pid;73 u32 pid;
75 74
76 /* ivcsw: treat like an enqueue event and store timestamp */75 /* ivcsw: treat like an enqueue event and store timestamp */
tools/include/uapi/linux/bpf.h+4−5
@@ -1116,6 +1116,7 @@ enum bpf_attach_type { · 1115 unmodified lines
1116 BPF_NETKIT_PRIMARY,1116 BPF_NETKIT_PRIMARY,
1117 BPF_NETKIT_PEER,1117 BPF_NETKIT_PEER,
1118 BPF_TRACE_KPROBE_SESSION,1118 BPF_TRACE_KPROBE_SESSION,
1119 BPF_TRACE_UPROBE_SESSION,
1119 __MAX_BPF_ATTACH_TYPE1120 __MAX_BPF_ATTACH_TYPE
1120};1121};
1121 1122
@@ -1973,6 +1974,8 @@ union bpf_attr { · 851 unmodified lines
1973 * program.1974 * program.
1974 * Return1975 * Return
1975 * The SMP id of the processor running the program.1976 * The SMP id of the processor running the program.
1977 * Attributes
1978 * __bpf_fastcall
1976 *1979 *
1977 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)1980 * long bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, u32 len, u64 flags)
1978 * Description1981 * Description
@@ -3104,10 +3107,6 @@ union bpf_attr { · 1125 unmodified lines
3104 * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration3107 * with the **CONFIG_BPF_KPROBE_OVERRIDE** configuration
3105 * option, and in this case it only works on functions tagged with3108 * option, and in this case it only works on functions tagged with
3106 * **ALLOW_ERROR_INJECTION** in the kernel code.3109 * **ALLOW_ERROR_INJECTION** in the kernel code.
3107 *
3108 * Also, the helper is only available for the architectures having
3109 * the CONFIG_FUNCTION_ERROR_INJECTION option. As of this writing,
3110 * x86 architecture is the only one to support this feature.
3111 * Return3110 * Return
3112 * 03111 * 0
3113 *3112 *
@@ -5372,7 +5371,7 @@ union bpf_attr { · 2258 unmodified lines
5372 * Currently, the **flags** must be 0. Currently, nr_loops is5371 * Currently, the **flags** must be 0. Currently, nr_loops is
5373 * limited to 1 << 23 (~8 million) loops.5372 * limited to 1 << 23 (~8 million) loops.
5374 *5373 *
5375 * long (\*callback_fn)(u32 index, void \*ctx);5374 * long (\*callback_fn)(u64 index, void \*ctx);
5376 *5375 *
5377 * where **index** is the current index in the loop. The index5376 * where **index** is the current index in the loop. The index
5378 * is zero-indexed.5377 * is zero-indexed.
tools/lib/bpf/Makefile+2−1
@@ -61,7 +61,8 @@ ifndef VERBOSE · 60 unmodified lines
61endif61endif
62 62
63INCLUDES = -I$(or $(OUTPUT),.) \63INCLUDES = -I$(or $(OUTPUT),.) \
64 -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi64 -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi \
65 -I$(srctree)/tools/arch/$(SRCARCH)/include
65 66
66export prefix libdir src obj67export prefix libdir src obj
67 68
tools/lib/bpf/bpf.c+1−0
@@ -776,6 +776,7 @@ int bpf_link_create(int prog_fd, int target_fd, · 775 unmodified lines
776 return libbpf_err(-EINVAL);776 return libbpf_err(-EINVAL);
777 break;777 break;
778 case BPF_TRACE_UPROBE_MULTI:778 case BPF_TRACE_UPROBE_MULTI:
779 case BPF_TRACE_UPROBE_SESSION:
779 attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);780 attr.link_create.uprobe_multi.flags = OPTS_GET(opts, uprobe_multi.flags, 0);
780 attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);781 attr.link_create.uprobe_multi.cnt = OPTS_GET(opts, uprobe_multi.cnt, 0);
781 attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));782 attr.link_create.uprobe_multi.path = ptr_to_u64(OPTS_GET(opts, uprobe_multi.path, 0));
tools/lib/bpf/bpf_gen_internal.h+1−0
@@ -34,6 +34,7 @@ struct bpf_gen { · 33 unmodified lines
34 void *data_cur;34 void *data_cur;
35 void *insn_start;35 void *insn_start;
36 void *insn_cur;36 void *insn_cur;
37 bool swapped_endian;
37 ssize_t cleanup_label;38 ssize_t cleanup_label;
38 __u32 nr_progs;39 __u32 nr_progs;
39 __u32 nr_maps;40 __u32 nr_maps;
tools/lib/bpf/bpf_helpers.h+1−0
@@ -185,6 +185,7 @@ enum libbpf_tristate { · 184 unmodified lines
185#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted")))185#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted")))
186#define __kptr __attribute__((btf_type_tag("kptr")))186#define __kptr __attribute__((btf_type_tag("kptr")))
187#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))187#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr")))
188#define __uptr __attribute__((btf_type_tag("uptr")))
188 189
189#if defined (__clang__)190#if defined (__clang__)
190#define bpf_ksym_exists(sym) ({ \191#define bpf_ksym_exists(sym) ({ \
tools/lib/bpf/btf.c+236−72
@@ -22,6 +22,7 @@ · 21 unmodified lines
22#include "libbpf_internal.h"22#include "libbpf_internal.h"
23#include "hashmap.h"23#include "hashmap.h"
24#include "strset.h"24#include "strset.h"
25#include "str_error.h"
25 26
26#define BTF_MAX_NR_TYPES 0x7fffffffU27#define BTF_MAX_NR_TYPES 0x7fffffffU
27#define BTF_MAX_STR_OFFSET 0x7fffffffU28#define BTF_MAX_STR_OFFSET 0x7fffffffU
@@ -1179,7 +1180,7 @@ static struct btf *btf_parse_elf(const char *path, struct btf *base_btf, · 1151 unmodified lines
1179 fd = open(path, O_RDONLY | O_CLOEXEC);1180 fd = open(path, O_RDONLY | O_CLOEXEC);
1180 if (fd < 0) {1181 if (fd < 0) {
1181 err = -errno;1182 err = -errno;
1182 pr_warn("failed to open %s: %s\n", path, strerror(errno));1183 pr_warn("failed to open %s: %s\n", path, errstr(err));
1183 return ERR_PTR(err);1184 return ERR_PTR(err);
1184 }1185 }
1185 1186
@@ -1445,7 +1446,7 @@ retry_load: · 259 unmodified lines
1445 goto retry_load;1446 goto retry_load;
1446 1447
1447 err = -errno;1448 err = -errno;
1448 pr_warn("BTF loading error: %d\n", err);1449 pr_warn("BTF loading error: %s\n", errstr(err));
1449 /* don't print out contents of custom log_buf */1450 /* don't print out contents of custom log_buf */
1450 if (!log_buf && buf[0])1451 if (!log_buf && buf[0])
1451 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf);1452 pr_warn("-- BEGIN BTF LOAD LOG ---\n%s\n-- END BTF LOAD LOG --\n", buf);
@@ -2885,7 +2886,7 @@ int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id, · 1433 unmodified lines
2885 return btf_commit_type(btf, sz);2886 return btf_commit_type(btf, sz);
2886}2887}
2887 2888
2888struct btf_ext_sec_setup_param {2889struct btf_ext_sec_info_param {
2889 __u32 off;2890 __u32 off;
2890 __u32 len;2891 __u32 len;
2891 __u32 min_rec_size;2892 __u32 min_rec_size;
@@ -2893,14 +2894,20 @@ struct btf_ext_sec_setup_param { · 1 unmodified line
2893 const char *desc;2894 const char *desc;
2894};2895};
2895 2896
2896static int btf_ext_setup_info(struct btf_ext *btf_ext,2897/*
2897 struct btf_ext_sec_setup_param *ext_sec)2898 * Parse a single info subsection of the BTF.ext info data:
2899 * - validate subsection structure and elements
2900 * - save info subsection start and sizing details in struct btf_ext
2901 * - endian-independent operation, for calling before byte-swapping
2902 */
2903static int btf_ext_parse_sec_info(struct btf_ext *btf_ext,
2904 struct btf_ext_sec_info_param *ext_sec,
2905 bool is_native)
2898{2906{
2899 const struct btf_ext_info_sec *sinfo;2907 const struct btf_ext_info_sec *sinfo;
2900 struct btf_ext_info *ext_info;2908 struct btf_ext_info *ext_info;
2901 __u32 info_left, record_size;2909 __u32 info_left, record_size;
2902 size_t sec_cnt = 0;2910 size_t sec_cnt = 0;
2903 /* The start of the info sec (including the __u32 record_size). */
2904 void *info;2911 void *info;
2905 2912
2906 if (ext_sec->len == 0)2913 if (ext_sec->len == 0)
@@ -2912,6 +2919,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 5 unmodified lines
2912 return -EINVAL;2919 return -EINVAL;
2913 }2920 }
2914 2921
2922 /* The start of the info sec (including the __u32 record_size). */
2915 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;2923 info = btf_ext->data + btf_ext->hdr->hdr_len + ext_sec->off;
2916 info_left = ext_sec->len;2924 info_left = ext_sec->len;
2917 2925
@@ -2927,9 +2935,13 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 9 unmodified lines
2927 return -EINVAL;2935 return -EINVAL;
2928 }2936 }
2929 2937
2930 /* The record size needs to meet the minimum standard */2938 /* The record size needs to meet either the minimum standard or, when
2931 record_size = *(__u32 *)info;2939 * handling non-native endianness data, the exact standard so as
2940 * to allow safe byte-swapping.
2941 */
2942 record_size = is_native ? *(__u32 *)info : bswap_32(*(__u32 *)info);
2932 if (record_size < ext_sec->min_rec_size ||2943 if (record_size < ext_sec->min_rec_size ||
2944 (!is_native && record_size != ext_sec->min_rec_size) ||
2933 record_size & 0x03) {2945 record_size & 0x03) {
2934 pr_debug("%s section in .BTF.ext has invalid record size %u\n",2946 pr_debug("%s section in .BTF.ext has invalid record size %u\n",
2935 ext_sec->desc, record_size);2947 ext_sec->desc, record_size);
@@ -2941,7 +2953,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 5 unmodified lines
2941 2953
2942 /* If no records, return failure now so .BTF.ext won't be used. */2954 /* If no records, return failure now so .BTF.ext won't be used. */
2943 if (!info_left) {2955 if (!info_left) {
2944 pr_debug("%s section in .BTF.ext has no records", ext_sec->desc);2956 pr_debug("%s section in .BTF.ext has no records\n", ext_sec->desc);
2945 return -EINVAL;2957 return -EINVAL;
2946 }2958 }
2947 2959
@@ -2956,7 +2968,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 8 unmodified lines
2956 return -EINVAL;2968 return -EINVAL;
2957 }2969 }
2958 2970
2959 num_records = sinfo->num_info;2971 num_records = is_native ? sinfo->num_info : bswap_32(sinfo->num_info);
2960 if (num_records == 0) {2972 if (num_records == 0) {
2961 pr_debug("%s section has incorrect num_records in .BTF.ext\n",2973 pr_debug("%s section has incorrect num_records in .BTF.ext\n",
2962 ext_sec->desc);2974 ext_sec->desc);
@@ -2984,64 +2996,157 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, · 21 unmodified lines
2984 return 0;2996 return 0;
2985}2997}
2986 2998
2987static int btf_ext_setup_func_info(struct btf_ext *btf_ext)2999/* Parse all info secs in the BTF.ext info data */
3000static int btf_ext_parse_info(struct btf_ext *btf_ext, bool is_native)
2988{3001{
2989 struct btf_ext_sec_setup_param param = {3002 struct btf_ext_sec_info_param func_info = {
2990 .off = btf_ext->hdr->func_info_off,3003 .off = btf_ext->hdr->func_info_off,
2991 .len = btf_ext->hdr->func_info_len,3004 .len = btf_ext->hdr->func_info_len,
2992 .min_rec_size = sizeof(struct bpf_func_info_min),3005 .min_rec_size = sizeof(struct bpf_func_info_min),
2993 .ext_info = &btf_ext->func_info,3006 .ext_info = &btf_ext->func_info,
2994 .desc = "func_info"3007 .desc = "func_info"
2995 };3008 };
2996 3009 struct btf_ext_sec_info_param line_info = {
2997 return btf_ext_setup_info(btf_ext, &param);
2998}
2999
3000static int btf_ext_setup_line_info(struct btf_ext *btf_ext)
3001{
3002 struct btf_ext_sec_setup_param param = {
3003 .off = btf_ext->hdr->line_info_off,3010 .off = btf_ext->hdr->line_info_off,
3004 .len = btf_ext->hdr->line_info_len,3011 .len = btf_ext->hdr->line_info_len,
3005 .min_rec_size = sizeof(struct bpf_line_info_min),3012 .min_rec_size = sizeof(struct bpf_line_info_min),
3006 .ext_info = &btf_ext->line_info,3013 .ext_info = &btf_ext->line_info,
3007 .desc = "line_info",3014 .desc = "line_info",
3008 };3015 };
3009 3016 struct btf_ext_sec_info_param core_relo = {
3010 return btf_ext_setup_info(btf_ext, &param);
3011}
3012
3013static int btf_ext_setup_core_relos(struct btf_ext *btf_ext)
3014{
3015 struct btf_ext_sec_setup_param param = {
3016 .off = btf_ext->hdr->core_relo_off,3017 .off = btf_ext->hdr->core_relo_off,
3017 .len = btf_ext->hdr->core_relo_len,3018 .len = btf_ext->hdr->core_relo_len,
3018 .min_rec_size = sizeof(struct bpf_core_relo),3019 .min_rec_size = sizeof(struct bpf_core_relo),
3019 .ext_info = &btf_ext->core_relo_info,3020 .ext_info = &btf_ext->core_relo_info,
3020 .desc = "core_relo",3021 .desc = "core_relo",
3021 };3022 };
3023 int err;
3024
3025 err = btf_ext_parse_sec_info(btf_ext, &func_info, is_native);
3026 if (err)
3027 return err;
3028
3029 err = btf_ext_parse_sec_info(btf_ext, &line_info, is_native);
3030 if (err)
3031 return err;
3032
3033 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3034 return 0; /* skip core relos parsing */
3035
3036 err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native);
3037 if (err)
3038 return err;
3039
3040 return 0;
3041}
3042
3043/* Swap byte-order of BTF.ext header with any endianness */
3044static void btf_ext_bswap_hdr(struct btf_ext_header *h)
3045{
3046 bool is_native = h->magic == BTF_MAGIC;
3047 __u32 hdr_len;
3048
3049 hdr_len = is_native ? h->hdr_len : bswap_32(h->hdr_len);
3050
3051 h->magic = bswap_16(h->magic);
3052 h->hdr_len = bswap_32(h->hdr_len);
3053 h->func_info_off = bswap_32(h->func_info_off);
3054 h->func_info_len = bswap_32(h->func_info_len);
3055 h->line_info_off = bswap_32(h->line_info_off);
3056 h->line_info_len = bswap_32(h->line_info_len);
3057
3058 if (hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3059 return;
3060
3061 h->core_relo_off = bswap_32(h->core_relo_off);
3062 h->core_relo_len = bswap_32(h->core_relo_len);
3063}
3064
3065/* Swap byte-order of generic info subsection */
3066static void btf_ext_bswap_info_sec(void *info, __u32 len, bool is_native,
3067 info_rec_bswap_fn bswap_fn)
3068{
3069 struct btf_ext_info_sec *sec;
3070 __u32 info_left, rec_size, *rs;
3071
3072 if (len == 0)
3073 return;
3074
3075 rs = info; /* info record size */
3076 rec_size = is_native ? *rs : bswap_32(*rs);
3077 *rs = bswap_32(*rs);
3078
3079 sec = info + sizeof(__u32); /* info sec #1 */
3080 info_left = len - sizeof(__u32);
3081 while (info_left) {
3082 unsigned int sec_hdrlen = sizeof(struct btf_ext_info_sec);
3083 __u32 i, num_recs;
3084 void *p;
3085
3086 num_recs = is_native ? sec->num_info : bswap_32(sec->num_info);
3087 sec->sec_name_off = bswap_32(sec->sec_name_off);
3088 sec->num_info = bswap_32(sec->num_info);
3089 p = sec->data; /* info rec #1 */
3090 for (i = 0; i < num_recs; i++, p += rec_size)
3091 bswap_fn(p);
3092 sec = p;
3093 info_left -= sec_hdrlen + (__u64)rec_size * num_recs;
3094 }
3095}
3096
3097/*
3098 * Swap byte-order of all info data in a BTF.ext section
3099 * - requires BTF.ext hdr in native endianness
3100 */
3101static void btf_ext_bswap_info(struct btf_ext *btf_ext, void *data)
3102{
3103 const bool is_native = btf_ext->swapped_endian;
3104 const struct btf_ext_header *h = data;
3105 void *info;
3106
3107 /* Swap func_info subsection byte-order */
3108 info = data + h->hdr_len + h->func_info_off;
3109 btf_ext_bswap_info_sec(info, h->func_info_len, is_native,
3110 (info_rec_bswap_fn)bpf_func_info_bswap);
3111
3112 /* Swap line_info subsection byte-order */
3113 info = data + h->hdr_len + h->line_info_off;
3114 btf_ext_bswap_info_sec(info, h->line_info_len, is_native,
3115 (info_rec_bswap_fn)bpf_line_info_bswap);
3116
3117 /* Swap core_relo subsection byte-order (if present) */
3118 if (h->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3119 return;
3022 3120
3023 return btf_ext_setup_info(btf_ext, &param);3121 info = data + h->hdr_len + h->core_relo_off;
3122 btf_ext_bswap_info_sec(info, h->core_relo_len, is_native,
3123 (info_rec_bswap_fn)bpf_core_relo_bswap);
3024}3124}
3025 3125
3026static int btf_ext_parse_hdr(__u8 *data, __u32 data_size)3126/* Parse hdr data and info sections: check and convert to native endianness */
3127static int btf_ext_parse(struct btf_ext *btf_ext)
3027{3128{
3028 const struct btf_ext_header *hdr = (struct btf_ext_header *)data;3129 __u32 hdr_len, data_size = btf_ext->data_size;
3130 struct btf_ext_header *hdr = btf_ext->hdr;
3131 bool swapped_endian = false;
3132 int err;
3029 3133
3030 if (data_size < offsetofend(struct btf_ext_header, hdr_len) ||3134 if (data_size < offsetofend(struct btf_ext_header, hdr_len)) {
3031 data_size < hdr->hdr_len) {3135 pr_debug("BTF.ext header too short\n");
3032 pr_debug("BTF.ext header not found");
3033 return -EINVAL;3136 return -EINVAL;
3034 }3137 }
3035 3138
3139 hdr_len = hdr->hdr_len;
3036 if (hdr->magic == bswap_16(BTF_MAGIC)) {3140 if (hdr->magic == bswap_16(BTF_MAGIC)) {
3037 pr_warn("BTF.ext in non-native endianness is not supported\n");3141 swapped_endian = true;
3038 return -ENOTSUP;3142 hdr_len = bswap_32(hdr_len);
3039 } else if (hdr->magic != BTF_MAGIC) {3143 } else if (hdr->magic != BTF_MAGIC) {
3040 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);3144 pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
3041 return -EINVAL;3145 return -EINVAL;
3042 }3146 }
3043 3147
3044 if (hdr->version != BTF_VERSION) {3148 /* Ensure known version of structs, current BTF_VERSION == 1 */
3149 if (hdr->version != 1) {
3045 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);3150 pr_debug("Unsupported BTF.ext version:%u\n", hdr->version);
3046 return -ENOTSUP;3151 return -ENOTSUP;
3047 }3152 }
@@ -3051,11 +3156,39 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size) · 3 unmodified lines
3051 return -ENOTSUP;3156 return -ENOTSUP;
3052 }3157 }
3053 3158
3054 if (data_size == hdr->hdr_len) {3159 if (data_size < hdr_len) {
3160 pr_debug("BTF.ext header not found\n");
3161 return -EINVAL;
3162 } else if (data_size == hdr_len) {
3055 pr_debug("BTF.ext has no data\n");3163 pr_debug("BTF.ext has no data\n");
3056 return -EINVAL;3164 return -EINVAL;
3057 }3165 }
3058 3166
3167 /* Verify mandatory hdr info details present */
3168 if (hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
3169 pr_warn("BTF.ext header missing func_info, line_info\n");
3170 return -EINVAL;
3171 }
3172
3173 /* Keep hdr native byte-order in memory for introspection */
3174 if (swapped_endian)
3175 btf_ext_bswap_hdr(btf_ext->hdr);
3176
3177 /* Validate info subsections and cache key metadata */
3178 err = btf_ext_parse_info(btf_ext, !swapped_endian);
3179 if (err)
3180 return err;
3181
3182 /* Keep infos native byte-order in memory for introspection */
3183 if (swapped_endian)
3184 btf_ext_bswap_info(btf_ext, btf_ext->data);
3185
3186 /*
3187 * Set btf_ext->swapped_endian only after all header and info data has
3188 * been swapped, helping bswap functions determine if their data are
3189 * in native byte-order when called.
3190 */
3191 btf_ext->swapped_endian = swapped_endian;
3059 return 0;3192 return 0;
3060}3193}
3061 3194
@@ -3067,6 +3200,7 @@ void btf_ext__free(struct btf_ext *btf_ext) · 5 unmodified lines
3067 free(btf_ext->line_info.sec_idxs);3200 free(btf_ext->line_info.sec_idxs);
3068 free(btf_ext->core_relo_info.sec_idxs);3201 free(btf_ext->core_relo_info.sec_idxs);
3069 free(btf_ext->data);3202 free(btf_ext->data);
3203 free(btf_ext->data_swapped);
3070 free(btf_ext);3204 free(btf_ext);
3071}3205}
3072 3206
@@ -3087,29 +3221,7 @@ struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) · 14 unmodified lines
3087 }3221 }
3088 memcpy(btf_ext->data, data, size);3222 memcpy(btf_ext->data, data, size);
3089 3223
3090 err = btf_ext_parse_hdr(btf_ext->data, size);3224 err = btf_ext_parse(btf_ext);
3091 if (err)
3092 goto done;
3093
3094 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, line_info_len)) {
3095 err = -EINVAL;
3096 goto done;
3097 }
3098
3099 err = btf_ext_setup_func_info(btf_ext);
3100 if (err)
3101 goto done;
3102
3103 err = btf_ext_setup_line_info(btf_ext);
3104 if (err)
3105 goto done;
3106
3107 if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
3108 goto done; /* skip core relos parsing */
3109
3110 err = btf_ext_setup_core_relos(btf_ext);
3111 if (err)
3112 goto done;
3113 3225
3114done:3226done:
3115 if (err) {3227 if (err) {
@@ -3120,15 +3232,66 @@ done: · 4 unmodified lines
3120 return btf_ext;3232 return btf_ext;
3121}3233}
3122 3234
3235static void *btf_ext_raw_data(const struct btf_ext *btf_ext_ro, bool swap_endian)
3236{
3237 struct btf_ext *btf_ext = (struct btf_ext *)btf_ext_ro;
3238 const __u32 data_sz = btf_ext->data_size;
3239 void *data;
3240
3241 /* Return native data (always present) or swapped data if present */
3242 if (!swap_endian)
3243 return btf_ext->data;
3244 else if (btf_ext->data_swapped)
3245 return btf_ext->data_swapped;
3246
3247 /* Recreate missing swapped data, then cache and return */
3248 data = calloc(1, data_sz);
3249 if (!data)
3250 return NULL;
3251 memcpy(data, btf_ext->data, data_sz);
3252
3253 btf_ext_bswap_info(btf_ext, data);
3254 btf_ext_bswap_hdr(data);
3255 btf_ext->data_swapped = data;
3256 return data;
3257}
3258
3123const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size)3259const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size)
3124{3260{
3261 void *data;
3262
3263 data = btf_ext_raw_data(btf_ext, btf_ext->swapped_endian);
3264 if (!data)
3265 return errno = ENOMEM, NULL;
3266
3125 *size = btf_ext->data_size;3267 *size = btf_ext->data_size;
3126 return btf_ext->data;3268 return data;
3127}3269}
3128 3270
3129__attribute__((alias("btf_ext__raw_data")))3271__attribute__((alias("btf_ext__raw_data")))
3130const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size);3272const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size);
3131 3273
3274enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext)
3275{
3276 if (is_host_big_endian())
3277 return btf_ext->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
3278 else
3279 return btf_ext->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
3280}
3281
3282int btf_ext__set_endianness(struct btf_ext *btf_ext, enum btf_endianness endian)
3283{
3284 if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
3285 return libbpf_err(-EINVAL);
3286
3287 btf_ext->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
3288
3289 if (!btf_ext->swapped_endian) {
3290 free(btf_ext->data_swapped);
3291 btf_ext->data_swapped = NULL;
3292 }
3293 return 0;
3294}
3132 3295
3133struct btf_dedup;3296struct btf_dedup;
3134 3297
@@ -3291,7 +3454,7 @@ int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) · 156 unmodified lines
3291 3454
3292 d = btf_dedup_new(btf, opts);3455 d = btf_dedup_new(btf, opts);
3293 if (IS_ERR(d)) {3456 if (IS_ERR(d)) {
3294 pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d));3457 pr_debug("btf_dedup_new failed: %ld\n", PTR_ERR(d));
3295 return libbpf_err(-EINVAL);3458 return libbpf_err(-EINVAL);
3296 }3459 }
3297 3460
@@ -3302,42 +3465,42 @@ int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) · 4 unmodified lines
3302 3465
3303 err = btf_dedup_prep(d);3466 err = btf_dedup_prep(d);
3304 if (err) {3467 if (err) {
3305 pr_debug("btf_dedup_prep failed:%d\n", err);3468 pr_debug("btf_dedup_prep failed: %s\n", errstr(err));
3306 goto done;3469 goto done;
3307 }3470 }
3308 err = btf_dedup_strings(d);3471 err = btf_dedup_strings(d);
3309 if (err < 0) {3472 if (err < 0) {
3310 pr_debug("btf_dedup_strings failed:%d\n", err);3473 pr_debug("btf_dedup_strings failed: %s\n", errstr(err));
3311 goto done;3474 goto done;
3312 }3475 }
3313 err = btf_dedup_prim_types(d);3476 err = btf_dedup_prim_types(d);
3314 if (err < 0) {3477 if (err < 0) {
3315 pr_debug("btf_dedup_prim_types failed:%d\n", err);3478 pr_debug("btf_dedup_prim_types failed: %s\n", errstr(err));
3316 goto done;3479 goto done;
3317 }3480 }
3318 err = btf_dedup_struct_types(d);3481 err = btf_dedup_struct_types(d);
3319 if (err < 0) {3482 if (err < 0) {
3320 pr_debug("btf_dedup_struct_types failed:%d\n", err);3483 pr_debug("btf_dedup_struct_types failed: %s\n", errstr(err));
3321 goto done;3484 goto done;
3322 }3485 }
3323 err = btf_dedup_resolve_fwds(d);3486 err = btf_dedup_resolve_fwds(d);
3324 if (err < 0) {3487 if (err < 0) {
3325 pr_debug("btf_dedup_resolve_fwds failed:%d\n", err);3488 pr_debug("btf_dedup_resolve_fwds failed: %s\n", errstr(err));
3326 goto done;3489 goto done;
3327 }3490 }
3328 err = btf_dedup_ref_types(d);3491 err = btf_dedup_ref_types(d);
3329 if (err < 0) {3492 if (err < 0) {
3330 pr_debug("btf_dedup_ref_types failed:%d\n", err);3493 pr_debug("btf_dedup_ref_types failed: %s\n", errstr(err));
3331 goto done;3494 goto done;
3332 }3495 }
3333 err = btf_dedup_compact_types(d);3496 err = btf_dedup_compact_types(d);
3334 if (err < 0) {3497 if (err < 0) {
3335 pr_debug("btf_dedup_compact_types failed:%d\n", err);3498 pr_debug("btf_dedup_compact_types failed: %s\n", errstr(err));
3336 goto done;3499 goto done;
3337 }3500 }
3338 err = btf_dedup_remap_types(d);3501 err = btf_dedup_remap_types(d);
3339 if (err < 0) {3502 if (err < 0) {
3340 pr_debug("btf_dedup_remap_types failed:%d\n", err);3503 pr_debug("btf_dedup_remap_types failed: %s\n", errstr(err));
3341 goto done;3504 goto done;
3342 }3505 }
3343 3506
@@ -3385,7 +3548,7 @@ struct btf_dedup { · 41 unmodified lines
3385 struct strset *strs_set;3548 struct strset *strs_set;
3386};3549};
3387 3550
3388static long hash_combine(long h, long value)3551static unsigned long hash_combine(unsigned long h, unsigned long value)
3389{3552{
3390 return h * 31 + value;3553 return h * 31 + value;
3391}3554}
@@ -5056,7 +5219,8 @@ struct btf *btf__load_vmlinux_btf(void) · 1664 unmodified lines
5056 btf = btf__parse(sysfs_btf_path, NULL);5219 btf = btf__parse(sysfs_btf_path, NULL);
5057 if (!btf) {5220 if (!btf) {
5058 err = -errno;5221 err = -errno;
5059 pr_warn("failed to read kernel BTF from '%s': %d\n", sysfs_btf_path, err);5222 pr_warn("failed to read kernel BTF from '%s': %s\n",
5223 sysfs_btf_path, errstr(err));
5060 return libbpf_err_ptr(err);5224 return libbpf_err_ptr(err);
5061 }5225 }
5062 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path);5226 pr_debug("loaded kernel BTF from '%s'\n", sysfs_btf_path);
@@ -5073,7 +5237,7 @@ struct btf *btf__load_vmlinux_btf(void) · 10 unmodified lines
5073 5237
5074 btf = btf__parse(path, NULL);5238 btf = btf__parse(path, NULL);
5075 err = libbpf_get_error(btf);5239 err = libbpf_get_error(btf);
5076 pr_debug("loading kernel BTF '%s': %d\n", path, err);5240 pr_debug("loading kernel BTF '%s': %s\n", path, errstr(err));
5077 if (err)5241 if (err)
5078 continue;5242 continue;
5079 5243
tools/lib/bpf/btf.h+3−0
@@ -167,6 +167,9 @@ LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); · 166 unmodified lines
167LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size);167LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size);
168LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext);168LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext);
169LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size);169LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size);
170LIBBPF_API enum btf_endianness btf_ext__endianness(const struct btf_ext *btf_ext);
171LIBBPF_API int btf_ext__set_endianness(struct btf_ext *btf_ext,
172 enum btf_endianness endian);
170 173
171LIBBPF_API int btf__find_str(struct btf *btf, const char *s);174LIBBPF_API int btf__find_str(struct btf *btf, const char *s);
172LIBBPF_API int btf__add_str(struct btf *btf, const char *s);175LIBBPF_API int btf__add_str(struct btf *btf, const char *s);
tools/lib/bpf/btf_dump.c+4−3
@@ -21,6 +21,7 @@ · 20 unmodified lines
21#include "hashmap.h"21#include "hashmap.h"
22#include "libbpf.h"22#include "libbpf.h"
23#include "libbpf_internal.h"23#include "libbpf_internal.h"
24#include "str_error.h"
24 25
25static const char PREFIXES[] = "\t\t\t\t\t\t\t\t\t\t\t\t\t";26static const char PREFIXES[] = "\t\t\t\t\t\t\t\t\t\t\t\t\t";
26static const size_t PREFIX_CNT = sizeof(PREFIXES) - 1;27static const size_t PREFIX_CNT = sizeof(PREFIXES) - 1;
@@ -867,8 +868,8 @@ static void btf_dump_emit_bit_padding(const struct btf_dump *d, · 840 unmodified lines
867 } pads[] = {868 } pads[] = {
868 {"long", d->ptr_sz * 8}, {"int", 32}, {"short", 16}, {"char", 8}869 {"long", d->ptr_sz * 8}, {"int", 32}, {"short", 16}, {"char", 8}
869 };870 };
870 int new_off, pad_bits, bits, i;871 int new_off = 0, pad_bits = 0, bits, i;
871 const char *pad_type;872 const char *pad_type = NULL;
872 873
873 if (cur_off >= next_off)874 if (cur_off >= next_off)
874 return; /* no gap */875 return; /* no gap */
@@ -1304,7 +1305,7 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, · 429 unmodified lines
1304 * chain, restore stack, emit warning, and try to1305 * chain, restore stack, emit warning, and try to
1305 * proceed nevertheless1306 * proceed nevertheless
1306 */1307 */
1307 pr_warn("not enough memory for decl stack:%d", err);1308 pr_warn("not enough memory for decl stack: %s\n", errstr(err));
1308 d->decl_stack_cnt = stack_start;1309 d->decl_stack_cnt = stack_start;
1309 return;1310 return;
1310 }1311 }
tools/lib/bpf/btf_relocate.c+1−1
@@ -428,7 +428,7 @@ static int btf_relocate_rewrite_strs(struct btf_relocate *r, __u32 i) · 427 unmodified lines
428 } else {428 } else {
429 off = r->str_map[*str_off];429 off = r->str_map[*str_off];
430 if (!off) {430 if (!off) {
431 pr_warn("string '%s' [offset %u] is not mapped to base BTF",431 pr_warn("string '%s' [offset %u] is not mapped to base BTF\n",
432 btf__str_by_offset(r->btf, off), *str_off);432 btf__str_by_offset(r->btf, off), *str_off);
433 return -ENOENT;433 return -ENOENT;
434 }434 }
tools/lib/bpf/elf.c+1−3
@@ -24,7 +24,6 @@ · 23 unmodified lines
24 24
25int elf_open(const char *binary_path, struct elf_fd *elf_fd)25int elf_open(const char *binary_path, struct elf_fd *elf_fd)
26{26{
27 char errmsg[STRERR_BUFSIZE];
28 int fd, ret;27 int fd, ret;
29 Elf *elf;28 Elf *elf;
30 29
@@ -38,8 +37,7 @@ int elf_open(const char *binary_path, struct elf_fd *elf_fd) · 7 unmodified lines
38 fd = open(binary_path, O_RDONLY | O_CLOEXEC);37 fd = open(binary_path, O_RDONLY | O_CLOEXEC);
39 if (fd < 0) {38 if (fd < 0) {
40 ret = -errno;39 ret = -errno;
41 pr_warn("elf: failed to open %s: %s\n", binary_path,40 pr_warn("elf: failed to open %s: %s\n", binary_path, errstr(ret));
42 libbpf_strerror_r(ret, errmsg, sizeof(errmsg)));
43 return ret;41 return ret;
44 }42 }
45 elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);43 elf = elf_begin(fd, ELF_C_READ_MMAP, NULL);
tools/lib/bpf/features.c+6−9
@@ -47,7 +47,6 @@ static int probe_kern_prog_name(int token_fd) · 46 unmodified lines
47 47
48static int probe_kern_global_data(int token_fd)48static int probe_kern_global_data(int token_fd)
49{49{
50 char *cp, errmsg[STRERR_BUFSIZE];
51 struct bpf_insn insns[] = {50 struct bpf_insn insns[] = {
52 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),51 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
53 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),52 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
@@ -67,9 +66,8 @@ static int probe_kern_global_data(int token_fd) · 13 unmodified lines
67 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, &map_opts);66 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, &map_opts);
68 if (map < 0) {67 if (map < 0) {
69 ret = -errno;68 ret = -errno;
70 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));69 pr_warn("Error in %s(): %s. Couldn't create simple array map.\n",
71 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",70 __func__, errstr(ret));
72 __func__, cp, -ret);
73 return ret;71 return ret;
74 }72 }
75 73
@@ -267,7 +265,6 @@ static int probe_kern_probe_read_kernel(int token_fd) · 191 unmodified lines
267 265
268static int probe_prog_bind_map(int token_fd)266static int probe_prog_bind_map(int token_fd)
269{267{
270 char *cp, errmsg[STRERR_BUFSIZE];
271 struct bpf_insn insns[] = {268 struct bpf_insn insns[] = {
272 BPF_MOV64_IMM(BPF_REG_0, 0),269 BPF_MOV64_IMM(BPF_REG_0, 0),
273 BPF_EXIT_INSN(),270 BPF_EXIT_INSN(),
@@ -285,9 +282,8 @@ static int probe_prog_bind_map(int token_fd) · 11 unmodified lines
285 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, &map_opts);282 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, &map_opts);
286 if (map < 0) {283 if (map < 0) {
287 ret = -errno;284 ret = -errno;
288 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));285 pr_warn("Error in %s(): %s. Couldn't create simple array map.\n",
289 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",286 __func__, errstr(ret));
290 __func__, cp, -ret);
291 return ret;287 return ret;
292 }288 }
293 289
@@ -604,7 +600,8 @@ bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_ · 310 unmodified lines
604 } else if (ret == 0) {600 } else if (ret == 0) {
605 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING);601 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING);
606 } else {602 } else {
607 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);603 pr_warn("Detection of kernel %s support failed: %s\n",
604 feat->desc, errstr(ret));
608 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING);605 WRITE_ONCE(cache->res[feat_id], FEAT_MISSING);
609 }606 }
610 }607 }
tools/lib/bpf/gen_loader.c+137−53
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include "bpf_gen_internal.h"14#include "bpf_gen_internal.h"
15#include "skel_internal.h"15#include "skel_internal.h"
16#include <asm/byteorder.h>16#include <asm/byteorder.h>
17#include "str_error.h"
17 18
18#define MAX_USED_MAPS 6419#define MAX_USED_MAPS 64
19#define MAX_USED_PROGS 3220#define MAX_USED_PROGS 32
@@ -393,7 +394,7 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) · 373 unmodified lines
393 blob_fd_array_off(gen, i));394 blob_fd_array_off(gen, i));
394 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));395 emit(gen, BPF_MOV64_IMM(BPF_REG_0, 0));
395 emit(gen, BPF_EXIT_INSN());396 emit(gen, BPF_EXIT_INSN());
396 pr_debug("gen: finish %d\n", gen->error);397 pr_debug("gen: finish %s\n", errstr(gen->error));
397 if (!gen->error) {398 if (!gen->error) {
398 struct gen_loader_opts *opts = gen->opts;399 struct gen_loader_opts *opts = gen->opts;
399 400
@@ -401,6 +402,15 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps) · 1 unmodified line
401 opts->insns_sz = gen->insn_cur - gen->insn_start;402 opts->insns_sz = gen->insn_cur - gen->insn_start;
402 opts->data = gen->data_start;403 opts->data = gen->data_start;
403 opts->data_sz = gen->data_cur - gen->data_start;404 opts->data_sz = gen->data_cur - gen->data_start;
405
406 /* use target endianness for embedded loader */
407 if (gen->swapped_endian) {
408 struct bpf_insn *insn = (struct bpf_insn *)opts->insns;
409 int insn_cnt = opts->insns_sz / sizeof(struct bpf_insn);
410
411 for (i = 0; i < insn_cnt; i++)
412 bpf_insn_bswap(insn++);
413 }
404 }414 }
405 return gen->error;415 return gen->error;
406}416}
@@ -414,6 +424,28 @@ void bpf_gen__free(struct bpf_gen *gen) · 7 unmodified lines
414 free(gen);424 free(gen);
415}425}
416 426
427/*
428 * Fields of bpf_attr are set to values in native byte-order before being
429 * written to the target-bound data blob, and may need endian conversion.
430 * This macro allows providing the correct value in situ more simply than
431 * writing a separate converter for *all fields* of *all records* included
432 * in union bpf_attr. Note that sizeof(rval) should match the assignment
433 * target to avoid runtime problems.
434 */
435#define tgt_endian(rval) ({ \
436 typeof(rval) _val = (rval); \
437 if (gen->swapped_endian) { \
438 switch (sizeof(_val)) { \
439 case 1: break; \
440 case 2: _val = bswap_16(_val); break; \
441 case 4: _val = bswap_32(_val); break; \
442 case 8: _val = bswap_64(_val); break; \
443 default: pr_warn("unsupported bswap size!\n"); \
444 } \
445 } \
446 _val; \
447})
448
417void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,449void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data,
418 __u32 btf_raw_size)450 __u32 btf_raw_size)
419{451{
@@ -422,11 +454,12 @@ void bpf_gen__load_btf(struct bpf_gen *gen, const void *btf_raw_data, · 2 unmodified lines
422 union bpf_attr attr;454 union bpf_attr attr;
423 455
424 memset(&attr, 0, attr_size);456 memset(&attr, 0, attr_size);
425 pr_debug("gen: load_btf: size %d\n", btf_raw_size);
426 btf_data = add_data(gen, btf_raw_data, btf_raw_size);457 btf_data = add_data(gen, btf_raw_data, btf_raw_size);
427 458
428 attr.btf_size = btf_raw_size;459 attr.btf_size = tgt_endian(btf_raw_size);
429 btf_load_attr = add_data(gen, &attr, attr_size);460 btf_load_attr = add_data(gen, &attr, attr_size);
461 pr_debug("gen: load_btf: off %d size %d, attr: off %d size %d\n",
462 btf_data, btf_raw_size, btf_load_attr, attr_size);
430 463
431 /* populate union bpf_attr with user provided log details */464 /* populate union bpf_attr with user provided log details */
432 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,465 move_ctx2blob(gen, attr_field(btf_load_attr, btf_log_level), 4,
@@ -457,28 +490,29 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 24 unmodified lines
457 union bpf_attr attr;490 union bpf_attr attr;
458 491
459 memset(&attr, 0, attr_size);492 memset(&attr, 0, attr_size);
460 attr.map_type = map_type;493 attr.map_type = tgt_endian(map_type);
461 attr.key_size = key_size;494 attr.key_size = tgt_endian(key_size);
462 attr.value_size = value_size;495 attr.value_size = tgt_endian(value_size);
463 attr.map_flags = map_attr->map_flags;496 attr.map_flags = tgt_endian(map_attr->map_flags);
464 attr.map_extra = map_attr->map_extra;497 attr.map_extra = tgt_endian(map_attr->map_extra);
465 if (map_name)498 if (map_name)
466 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));499 libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
467 attr.numa_node = map_attr->numa_node;500 attr.numa_node = tgt_endian(map_attr->numa_node);
468 attr.map_ifindex = map_attr->map_ifindex;501 attr.map_ifindex = tgt_endian(map_attr->map_ifindex);
469 attr.max_entries = max_entries;502 attr.max_entries = tgt_endian(max_entries);
470 attr.btf_key_type_id = map_attr->btf_key_type_id;503 attr.btf_key_type_id = tgt_endian(map_attr->btf_key_type_id);
471 attr.btf_value_type_id = map_attr->btf_value_type_id;504 attr.btf_value_type_id = tgt_endian(map_attr->btf_value_type_id);
472
473 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d\n",
474 attr.map_name, map_idx, map_type, attr.btf_value_type_id);
475 505
476 map_create_attr = add_data(gen, &attr, attr_size);506 map_create_attr = add_data(gen, &attr, attr_size);
477 if (attr.btf_value_type_id)507 pr_debug("gen: map_create: %s idx %d type %d value_type_id %d, attr: off %d size %d\n",
508 map_name, map_idx, map_type, map_attr->btf_value_type_id,
509 map_create_attr, attr_size);
510
511 if (map_attr->btf_value_type_id)
478 /* populate union bpf_attr with btf_fd saved in the stack earlier */512 /* populate union bpf_attr with btf_fd saved in the stack earlier */
479 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,513 move_stack2blob(gen, attr_field(map_create_attr, btf_fd), 4,
480 stack_off(btf_fd));514 stack_off(btf_fd));
481 switch (attr.map_type) {515 switch (map_type) {
482 case BPF_MAP_TYPE_ARRAY_OF_MAPS:516 case BPF_MAP_TYPE_ARRAY_OF_MAPS:
483 case BPF_MAP_TYPE_HASH_OF_MAPS:517 case BPF_MAP_TYPE_HASH_OF_MAPS:
484 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,518 move_stack2blob(gen, attr_field(map_create_attr, inner_map_fd), 4,
@@ -498,8 +532,8 @@ void bpf_gen__map_create(struct bpf_gen *gen, · 13 unmodified lines
498 /* emit MAP_CREATE command */532 /* emit MAP_CREATE command */
499 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);533 emit_sys_bpf(gen, BPF_MAP_CREATE, map_create_attr, attr_size);
500 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",534 debug_ret(gen, "map_create %s idx %d type %d value_size %d value_btf_id %d",
501 attr.map_name, map_idx, map_type, value_size,535 map_name, map_idx, map_type, value_size,
502 attr.btf_value_type_id);536 map_attr->btf_value_type_id);
503 emit_check_err(gen);537 emit_check_err(gen);
504 /* remember map_fd in the stack, if successful */538 /* remember map_fd in the stack, if successful */
505 if (map_idx < 0) {539 if (map_idx < 0) {
@@ -784,12 +818,12 @@ log: · 278 unmodified lines
784 emit_ksym_relo_log(gen, relo, kdesc->ref);818 emit_ksym_relo_log(gen, relo, kdesc->ref);
785}819}
786 820
787static __u32 src_reg_mask(void)821static __u32 src_reg_mask(struct bpf_gen *gen)
788{822{
789#if defined(__LITTLE_ENDIAN_BITFIELD)823#if defined(__LITTLE_ENDIAN_BITFIELD) /* src_reg,dst_reg,... */
790 return 0x0f; /* src_reg,dst_reg,... */824 return gen->swapped_endian ? 0xf0 : 0x0f;
791#elif defined(__BIG_ENDIAN_BITFIELD)825#elif defined(__BIG_ENDIAN_BITFIELD) /* dst_reg,src_reg,... */
792 return 0xf0; /* dst_reg,src_reg,... */826 return gen->swapped_endian ? 0x0f : 0xf0;
793#else827#else
794#error "Unsupported bit endianness, cannot proceed"828#error "Unsupported bit endianness, cannot proceed"
795#endif829#endif
@@ -840,7 +874,7 @@ static void emit_relo_ksym_btf(struct bpf_gen *gen, struct ksym_relo_desc *relo, · 44 unmodified lines
840 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));874 emit(gen, BPF_JMP_IMM(BPF_JA, 0, 0, 3));
841clear_src_reg:875clear_src_reg:
842 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */876 /* clear bpf_object__relocate_data's src_reg assignment, otherwise we get a verifier failure */
843 reg_mask = src_reg_mask();877 reg_mask = src_reg_mask(gen);
844 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));878 emit(gen, BPF_LDX_MEM(BPF_B, BPF_REG_9, BPF_REG_8, offsetofend(struct bpf_insn, code)));
845 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));879 emit(gen, BPF_ALU32_IMM(BPF_AND, BPF_REG_9, reg_mask));
846 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));880 emit(gen, BPF_STX_MEM(BPF_B, BPF_REG_8, BPF_REG_9, offsetofend(struct bpf_insn, code)));
@@ -931,48 +965,94 @@ static void cleanup_relos(struct bpf_gen *gen, int insns) · 84 unmodified lines
931 cleanup_core_relo(gen);965 cleanup_core_relo(gen);
932}966}
933 967
968/* Convert func, line, and core relo info blobs to target endianness */
969static void info_blob_bswap(struct bpf_gen *gen, int func_info, int line_info,
970 int core_relos, struct bpf_prog_load_opts *load_attr)
971{
972 struct bpf_func_info *fi = gen->data_start + func_info;
973 struct bpf_line_info *li = gen->data_start + line_info;
974 struct bpf_core_relo *cr = gen->data_start + core_relos;
975 int i;
976
977 for (i = 0; i < load_attr->func_info_cnt; i++)
978 bpf_func_info_bswap(fi++);
979
980 for (i = 0; i < load_attr->line_info_cnt; i++)
981 bpf_line_info_bswap(li++);
982
983 for (i = 0; i < gen->core_relo_cnt; i++)
984 bpf_core_relo_bswap(cr++);
985}
986
934void bpf_gen__prog_load(struct bpf_gen *gen,987void bpf_gen__prog_load(struct bpf_gen *gen,
935 enum bpf_prog_type prog_type, const char *prog_name,988 enum bpf_prog_type prog_type, const char *prog_name,
936 const char *license, struct bpf_insn *insns, size_t insn_cnt,989 const char *license, struct bpf_insn *insns, size_t insn_cnt,
937 struct bpf_prog_load_opts *load_attr, int prog_idx)990 struct bpf_prog_load_opts *load_attr, int prog_idx)
938{991{
992 int func_info_tot_sz = load_attr->func_info_cnt *
993 load_attr->func_info_rec_size;
994 int line_info_tot_sz = load_attr->line_info_cnt *
995 load_attr->line_info_rec_size;
996 int core_relo_tot_sz = gen->core_relo_cnt *
997 sizeof(struct bpf_core_relo);
939 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;998 int prog_load_attr, license_off, insns_off, func_info, line_info, core_relos;
940 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);999 int attr_size = offsetofend(union bpf_attr, core_relo_rec_size);
941 union bpf_attr attr;1000 union bpf_attr attr;
942 1001
943 memset(&attr, 0, attr_size);1002 memset(&attr, 0, attr_size);
944 pr_debug("gen: prog_load: type %d insns_cnt %zd progi_idx %d\n",
945 prog_type, insn_cnt, prog_idx);
946 /* add license string to blob of bytes */1003 /* add license string to blob of bytes */
947 license_off = add_data(gen, license, strlen(license) + 1);1004 license_off = add_data(gen, license, strlen(license) + 1);
948 /* add insns to blob of bytes */1005 /* add insns to blob of bytes */
949 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));1006 insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn));
1007 pr_debug("gen: prog_load: prog_idx %d type %d insn off %d insns_cnt %zd license off %d\n",
1008 prog_idx, prog_type, insns_off, insn_cnt, license_off);
950 1009
951 attr.prog_type = prog_type;1010 /* convert blob insns to target endianness */
952 attr.expected_attach_type = load_attr->expected_attach_type;1011 if (gen->swapped_endian) {
953 attr.attach_btf_id = load_attr->attach_btf_id;1012 struct bpf_insn *insn = gen->data_start + insns_off;
954 attr.prog_ifindex = load_attr->prog_ifindex;1013 int i;
955 attr.kern_version = 0;
956 attr.insn_cnt = (__u32)insn_cnt;
957 attr.prog_flags = load_attr->prog_flags;
958
959 attr.func_info_rec_size = load_attr->func_info_rec_size;
960 attr.func_info_cnt = load_attr->func_info_cnt;
961 func_info = add_data(gen, load_attr->func_info,
962 attr.func_info_cnt * attr.func_info_rec_size);
963 1014
964 attr.line_info_rec_size = load_attr->line_info_rec_size;1015 for (i = 0; i < insn_cnt; i++, insn++)
965 attr.line_info_cnt = load_attr->line_info_cnt;1016 bpf_insn_bswap(insn);
966 line_info = add_data(gen, load_attr->line_info,1017 }
967 attr.line_info_cnt * attr.line_info_rec_size);
968 1018
969 attr.core_relo_rec_size = sizeof(struct bpf_core_relo);1019 attr.prog_type = tgt_endian(prog_type);
970 attr.core_relo_cnt = gen->core_relo_cnt;1020 attr.expected_attach_type = tgt_endian(load_attr->expected_attach_type);
971 core_relos = add_data(gen, gen->core_relos,1021 attr.attach_btf_id = tgt_endian(load_attr->attach_btf_id);
972 attr.core_relo_cnt * attr.core_relo_rec_size);1022 attr.prog_ifindex = tgt_endian(load_attr->prog_ifindex);
1023 attr.kern_version = 0;
1024 attr.insn_cnt = tgt_endian((__u32)insn_cnt);
1025 attr.prog_flags = tgt_endian(load_attr->prog_flags);
1026
1027 attr.func_info_rec_size = tgt_endian(load_attr->func_info_rec_size);
1028 attr.func_info_cnt = tgt_endian(load_attr->func_info_cnt);
1029 func_info = add_data(gen, load_attr->func_info, func_info_tot_sz);
1030 pr_debug("gen: prog_load: func_info: off %d cnt %d rec size %d\n",
1031 func_info, load_attr->func_info_cnt,
1032 load_attr->func_info_rec_size);
1033
1034 attr.line_info_rec_size = tgt_endian(load_attr->line_info_rec_size);
1035 attr.line_info_cnt = tgt_endian(load_attr->line_info_cnt);
1036 line_info = add_data(gen, load_attr->line_info, line_info_tot_sz);
1037 pr_debug("gen: prog_load: line_info: off %d cnt %d rec size %d\n",
1038 line_info, load_attr->line_info_cnt,
1039 load_attr->line_info_rec_size);
1040
1041 attr.core_relo_rec_size = tgt_endian((__u32)sizeof(struct bpf_core_relo));
1042 attr.core_relo_cnt = tgt_endian(gen->core_relo_cnt);
1043 core_relos = add_data(gen, gen->core_relos, core_relo_tot_sz);
1044 pr_debug("gen: prog_load: core_relos: off %d cnt %d rec size %zd\n",
1045 core_relos, gen->core_relo_cnt,
1046 sizeof(struct bpf_core_relo));
1047
1048 /* convert all info blobs to target endianness */
1049 if (gen->swapped_endian)
1050 info_blob_bswap(gen, func_info, line_info, core_relos, load_attr);
973 1051
974 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));1052 libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
975 prog_load_attr = add_data(gen, &attr, attr_size);1053 prog_load_attr = add_data(gen, &attr, attr_size);
1054 pr_debug("gen: prog_load: attr: off %d size %d\n",
1055 prog_load_attr, attr_size);
976 1056
977 /* populate union bpf_attr with a pointer to license */1057 /* populate union bpf_attr with a pointer to license */
978 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);1058 emit_rel_store(gen, attr_field(prog_load_attr, license), license_off);
@@ -1040,7 +1120,6 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, · 61 unmodified lines
1040 int zero = 0;1120 int zero = 0;
1041 1121
1042 memset(&attr, 0, attr_size);1122 memset(&attr, 0, attr_size);
1043 pr_debug("gen: map_update_elem: idx %d\n", map_idx);
1044 1123
1045 value = add_data(gen, pvalue, value_size);1124 value = add_data(gen, pvalue, value_size);
1046 key = add_data(gen, &zero, sizeof(zero));1125 key = add_data(gen, &zero, sizeof(zero));
@@ -1068,6 +1147,8 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, · 21 unmodified lines
1068 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));1147 emit(gen, BPF_EMIT_CALL(BPF_FUNC_probe_read_kernel));
1069 1148
1070 map_update_attr = add_data(gen, &attr, attr_size);1149 map_update_attr = add_data(gen, &attr, attr_size);
1150 pr_debug("gen: map_update_elem: idx %d, value: off %d size %d, attr: off %d size %d\n",
1151 map_idx, value, value_size, map_update_attr, attr_size);
1071 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,1152 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1072 blob_fd_array_off(gen, map_idx));1153 blob_fd_array_off(gen, map_idx));
1073 emit_rel_store(gen, attr_field(map_update_attr, key), key);1154 emit_rel_store(gen, attr_field(map_update_attr, key), key);
@@ -1084,14 +1165,16 @@ void bpf_gen__populate_outer_map(struct bpf_gen *gen, int outer_map_idx, int slo · 10 unmodified lines
1084 int attr_size = offsetofend(union bpf_attr, flags);1165 int attr_size = offsetofend(union bpf_attr, flags);
1085 int map_update_attr, key;1166 int map_update_attr, key;
1086 union bpf_attr attr;1167 union bpf_attr attr;
1168 int tgt_slot;
1087 1169
1088 memset(&attr, 0, attr_size);1170 memset(&attr, 0, attr_size);
1089 pr_debug("gen: populate_outer_map: outer %d key %d inner %d\n",
1090 outer_map_idx, slot, inner_map_idx);
1091 1171
1092 key = add_data(gen, &slot, sizeof(slot));1172 tgt_slot = tgt_endian(slot);
1173 key = add_data(gen, &tgt_slot, sizeof(tgt_slot));
1093 1174
1094 map_update_attr = add_data(gen, &attr, attr_size);1175 map_update_attr = add_data(gen, &attr, attr_size);
1176 pr_debug("gen: populate_outer_map: outer %d key %d inner %d, attr: off %d size %d\n",
1177 outer_map_idx, slot, inner_map_idx, map_update_attr, attr_size);
1095 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,1178 move_blob2blob(gen, attr_field(map_update_attr, map_fd), 4,
1096 blob_fd_array_off(gen, outer_map_idx));1179 blob_fd_array_off(gen, outer_map_idx));
1097 emit_rel_store(gen, attr_field(map_update_attr, key), key);1180 emit_rel_store(gen, attr_field(map_update_attr, key), key);
@@ -1112,8 +1195,9 @@ void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx) · 14 unmodified lines
1112 union bpf_attr attr;1195 union bpf_attr attr;
1113 1196
1114 memset(&attr, 0, attr_size);1197 memset(&attr, 0, attr_size);
1115 pr_debug("gen: map_freeze: idx %d\n", map_idx);
1116 map_freeze_attr = add_data(gen, &attr, attr_size);1198 map_freeze_attr = add_data(gen, &attr, attr_size);
1199 pr_debug("gen: map_freeze: idx %d, attr: off %d size %d\n",
1200 map_idx, map_freeze_attr, attr_size);
1117 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,1201 move_blob2blob(gen, attr_field(map_freeze_attr, map_fd), 4,
1118 blob_fd_array_off(gen, map_idx));1202 blob_fd_array_off(gen, map_idx));
1119 /* emit MAP_FREEZE command */1203 /* emit MAP_FREEZE command */
tools/lib/bpf/hashmap.h+10−10
@@ -166,8 +166,8 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); · 165 unmodified lines
166 * @bkt: integer used as a bucket loop cursor166 * @bkt: integer used as a bucket loop cursor
167 */167 */
168#define hashmap__for_each_entry(map, cur, bkt) \168#define hashmap__for_each_entry(map, cur, bkt) \
169 for (bkt = 0; bkt < map->cap; bkt++) \169 for (bkt = 0; bkt < (map)->cap; bkt++) \
170 for (cur = map->buckets[bkt]; cur; cur = cur->next)170 for (cur = (map)->buckets[bkt]; cur; cur = cur->next)
171 171
172/*172/*
173 * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe173 * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe
@@ -178,8 +178,8 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); · 4 unmodified lines
178 * @bkt: integer used as a bucket loop cursor178 * @bkt: integer used as a bucket loop cursor
179 */179 */
180#define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \180#define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \
181 for (bkt = 0; bkt < map->cap; bkt++) \181 for (bkt = 0; bkt < (map)->cap; bkt++) \
182 for (cur = map->buckets[bkt]; \182 for (cur = (map)->buckets[bkt]; \
183 cur && ({tmp = cur->next; true; }); \183 cur && ({tmp = cur->next; true; }); \
184 cur = tmp)184 cur = tmp)
185 185
@@ -190,19 +190,19 @@ bool hashmap_find(const struct hashmap *map, long key, long *value); · 4 unmodified lines
190 * @key: key to iterate entries for190 * @key: key to iterate entries for
191 */191 */
192#define hashmap__for_each_key_entry(map, cur, _key) \192#define hashmap__for_each_key_entry(map, cur, _key) \
193 for (cur = map->buckets \193 for (cur = (map)->buckets \
194 ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \194 ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \
195 : NULL; \195 : NULL; \
196 cur; \196 cur; \
197 cur = cur->next) \197 cur = cur->next) \
198 if (map->equal_fn(cur->key, (_key), map->ctx))198 if ((map)->equal_fn(cur->key, (_key), (map)->ctx))
199 199
200#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \200#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \
201 for (cur = map->buckets \201 for (cur = (map)->buckets \
202 ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \202 ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \
203 : NULL; \203 : NULL; \
204 cur && ({ tmp = cur->next; true; }); \204 cur && ({ tmp = cur->next; true; }); \
205 cur = tmp) \205 cur = tmp) \
206 if (map->equal_fn(cur->key, (_key), map->ctx))206 if ((map)->equal_fn(cur->key, (_key), (map)->ctx))
207 207
208#endif /* __LIBBPF_HASHMAP_H */208#endif /* __LIBBPF_HASHMAP_H */
tools/lib/bpf/libbpf.c+266−260
@@ -133,6 +133,7 @@ static const char * const attach_type_name[] = { · 132 unmodified lines
133 [BPF_NETKIT_PRIMARY] = "netkit_primary",133 [BPF_NETKIT_PRIMARY] = "netkit_primary",
134 [BPF_NETKIT_PEER] = "netkit_peer",134 [BPF_NETKIT_PEER] = "netkit_peer",
135 [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",135 [BPF_TRACE_KPROBE_SESSION] = "trace_kprobe_session",
136 [BPF_TRACE_UPROBE_SESSION] = "trace_uprobe_session",
136};137};
137 138
138static const char * const link_type_name[] = {139static const char * const link_type_name[] = {
@@ -694,6 +695,8 @@ struct bpf_object { · 555 unmodified lines
694 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */695 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
695 struct elf_state efile;696 struct elf_state efile;
696 697
698 unsigned char byteorder;
699
697 struct btf *btf;700 struct btf *btf;
698 struct btf_ext *btf_ext;701 struct btf_ext *btf_ext;
699 702
@@ -940,6 +943,20 @@ bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, · 240 unmodified lines
940 return 0;943 return 0;
941}944}
942 945
946static void bpf_object_bswap_progs(struct bpf_object *obj)
947{
948 struct bpf_program *prog = obj->programs;
949 struct bpf_insn *insn;
950 int p, i;
951
952 for (p = 0; p < obj->nr_programs; p++, prog++) {
953 insn = prog->insns;
954 for (i = 0; i < prog->insns_cnt; i++, insn++)
955 bpf_insn_bswap(insn);
956 }
957 pr_debug("converted %zu BPF programs to native byte order\n", obj->nr_programs);
958}
959
943static const struct btf_member *960static const struct btf_member *
944find_member_by_offset(const struct btf_type *t, __u32 bit_offset)961find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
945{962{
@@ -1506,6 +1523,7 @@ static void bpf_object__elf_finish(struct bpf_object *obj) · 560 unmodified lines
1506 1523
1507 elf_end(obj->efile.elf);1524 elf_end(obj->efile.elf);
1508 obj->efile.elf = NULL;1525 obj->efile.elf = NULL;
1526 obj->efile.ehdr = NULL;
1509 obj->efile.symbols = NULL;1527 obj->efile.symbols = NULL;
1510 obj->efile.arena_data = NULL;1528 obj->efile.arena_data = NULL;
1511 1529
@@ -1533,11 +1551,8 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 21 unmodified lines
1533 } else {1551 } else {
1534 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);1552 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1535 if (obj->efile.fd < 0) {1553 if (obj->efile.fd < 0) {
1536 char errmsg[STRERR_BUFSIZE], *cp;
1537
1538 err = -errno;1554 err = -errno;
1539 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));1555 pr_warn("elf: failed to open %s: %s\n", obj->path, errstr(err));
1540 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1541 return err;1556 return err;
1542 }1557 }
1543 1558
@@ -1571,6 +1586,16 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 27 unmodified lines
1571 goto errout;1586 goto errout;
1572 }1587 }
1573 1588
1589 /* Validate ELF object endianness... */
1590 if (ehdr->e_ident[EI_DATA] != ELFDATA2LSB &&
1591 ehdr->e_ident[EI_DATA] != ELFDATA2MSB) {
1592 err = -LIBBPF_ERRNO__ENDIAN;
1593 pr_warn("elf: '%s' has unknown byte order\n", obj->path);
1594 goto errout;
1595 }
1596 /* and save after bpf_object_open() frees ELF data */
1597 obj->byteorder = ehdr->e_ident[EI_DATA];
1598
1574 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {1599 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1575 pr_warn("elf: failed to get section names section index for %s: %s\n",1600 pr_warn("elf: failed to get section names section index for %s: %s\n",
1576 obj->path, elf_errmsg(-1));1601 obj->path, elf_errmsg(-1));
@@ -1599,19 +1624,15 @@ errout: · 22 unmodified lines
1599 return err;1624 return err;
1600}1625}
1601 1626
1602static int bpf_object__check_endianness(struct bpf_object *obj)1627static bool is_native_endianness(struct bpf_object *obj)
1603{1628{
1604#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__1629#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1605 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)1630 return obj->byteorder == ELFDATA2LSB;
1606 return 0;
1607#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__1631#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1608 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)1632 return obj->byteorder == ELFDATA2MSB;
1609 return 0;
1610#else1633#else
1611# error "Unrecognized __BYTE_ORDER__"1634# error "Unrecognized __BYTE_ORDER__"
1612#endif1635#endif
1613 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1614 return -LIBBPF_ERRNO__ENDIAN;
1615}1636}
1616 1637
1617static int1638static int
@@ -1937,8 +1958,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, · 319 unmodified lines
1937 if (map->mmaped == MAP_FAILED) {1958 if (map->mmaped == MAP_FAILED) {
1938 err = -errno;1959 err = -errno;
1939 map->mmaped = NULL;1960 map->mmaped = NULL;
1940 pr_warn("failed to alloc map '%s' content buffer: %d\n",1961 pr_warn("failed to alloc map '%s' content buffer: %s\n", map->name, errstr(err));
1941 map->name, err);
1942 zfree(&map->real_name);1962 zfree(&map->real_name);
1943 zfree(&map->name);1963 zfree(&map->name);
1944 return err;1964 return err;
@@ -2102,7 +2122,7 @@ static int parse_u64(const char *value, __u64 *res) · 157 unmodified lines
2102 *res = strtoull(value, &value_end, 0);2122 *res = strtoull(value, &value_end, 0);
2103 if (errno) {2123 if (errno) {
2104 err = -errno;2124 err = -errno;
2105 pr_warn("failed to parse '%s' as integer: %d\n", value, err);2125 pr_warn("failed to parse '%s': %s\n", value, errstr(err));
2106 return err;2126 return err;
2107 }2127 }
2108 if (*value_end) {2128 if (*value_end) {
@@ -2268,8 +2288,8 @@ static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) · 159 unmodified lines
2268 while (gzgets(file, buf, sizeof(buf))) {2288 while (gzgets(file, buf, sizeof(buf))) {
2269 err = bpf_object__process_kconfig_line(obj, buf, data);2289 err = bpf_object__process_kconfig_line(obj, buf, data);
2270 if (err) {2290 if (err) {
2271 pr_warn("error parsing system Kconfig line '%s': %d\n",2291 pr_warn("error parsing system Kconfig line '%s': %s\n",
2272 buf, err);2292 buf, errstr(err));
2273 goto out;2293 goto out;
2274 }2294 }
2275 }2295 }
@@ -2289,15 +2309,15 @@ static int bpf_object__read_kconfig_mem(struct bpf_object *obj, · 13 unmodified lines
2289 file = fmemopen((void *)config, strlen(config), "r");2309 file = fmemopen((void *)config, strlen(config), "r");
2290 if (!file) {2310 if (!file) {
2291 err = -errno;2311 err = -errno;
2292 pr_warn("failed to open in-memory Kconfig: %d\n", err);2312 pr_warn("failed to open in-memory Kconfig: %s\n", errstr(err));
2293 return err;2313 return err;
2294 }2314 }
2295 2315
2296 while (fgets(buf, sizeof(buf), file)) {2316 while (fgets(buf, sizeof(buf), file)) {
2297 err = bpf_object__process_kconfig_line(obj, buf, data);2317 err = bpf_object__process_kconfig_line(obj, buf, data);
2298 if (err) {2318 if (err) {
2299 pr_warn("error parsing in-memory Kconfig line '%s': %d\n",2319 pr_warn("error parsing in-memory Kconfig line '%s': %s\n",
2300 buf, err);2320 buf, errstr(err));
2301 break;2321 break;
2302 }2322 }
2303 }2323 }
@@ -3212,7 +3232,7 @@ static int bpf_object__init_btf(struct bpf_object *obj, · 908 unmodified lines
3212 err = libbpf_get_error(obj->btf);3232 err = libbpf_get_error(obj->btf);
3213 if (err) {3233 if (err) {
3214 obj->btf = NULL;3234 obj->btf = NULL;
3215 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);3235 pr_warn("Error loading ELF section %s: %s.\n", BTF_ELF_SEC, errstr(err));
3216 goto out;3236 goto out;
3217 }3237 }
3218 /* enforce 8-byte pointers for BPF-targeted BTFs */3238 /* enforce 8-byte pointers for BPF-targeted BTFs */
@@ -3230,8 +3250,8 @@ static int bpf_object__init_btf(struct bpf_object *obj, · 11 unmodified lines
3230 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);3250 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
3231 err = libbpf_get_error(obj->btf_ext);3251 err = libbpf_get_error(obj->btf_ext);
3232 if (err) {3252 if (err) {
3233 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",3253 pr_warn("Error loading ELF section %s: %s. Ignored and continue.\n",
3234 BTF_EXT_ELF_SEC, err);3254 BTF_EXT_ELF_SEC, errstr(err));
3235 obj->btf_ext = NULL;3255 obj->btf_ext = NULL;
3236 goto out;3256 goto out;
3237 }3257 }
@@ -3323,8 +3343,8 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, · 85 unmodified lines
3323 if (t->size == 0) {3343 if (t->size == 0) {
3324 err = find_elf_sec_sz(obj, sec_name, &size);3344 err = find_elf_sec_sz(obj, sec_name, &size);
3325 if (err || !size) {3345 if (err || !size) {
3326 pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",3346 pr_debug("sec '%s': failed to determine size from ELF: size %u, err %s\n",
3327 sec_name, size, err);3347 sec_name, size, errstr(err));
3328 return -ENOENT;3348 return -ENOENT;
3329 }3349 }
3330 3350
@@ -3478,7 +3498,7 @@ static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) · 147 unmodified lines
3478 obj->btf_vmlinux = btf__load_vmlinux_btf();3498 obj->btf_vmlinux = btf__load_vmlinux_btf();
3479 err = libbpf_get_error(obj->btf_vmlinux);3499 err = libbpf_get_error(obj->btf_vmlinux);
3480 if (err) {3500 if (err) {
3481 pr_warn("Error loading vmlinux BTF: %d\n", err);3501 pr_warn("Error loading vmlinux BTF: %s\n", errstr(err));
3482 obj->btf_vmlinux = NULL;3502 obj->btf_vmlinux = NULL;
3483 return err;3503 return err;
3484 }3504 }
@@ -3581,11 +3601,14 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) · 96 unmodified lines
3581report:3601report:
3582 if (err) {3602 if (err) {
3583 btf_mandatory = kernel_needs_btf(obj);3603 btf_mandatory = kernel_needs_btf(obj);
3584 pr_warn("Error loading .BTF into kernel: %d. %s\n", err,3604 if (btf_mandatory) {
3585 btf_mandatory ? "BTF is mandatory, can't proceed."3605 pr_warn("Error loading .BTF into kernel: %s. BTF is mandatory, can't proceed.\n",
3586 : "BTF is optional, ignoring.");3606 errstr(err));
3587 if (!btf_mandatory)3607 } else {
3608 pr_info("Error loading .BTF into kernel: %s. BTF is optional, ignoring.\n",
3609 errstr(err));
3588 err = 0;3610 err = 0;
3611 }
3589 }3612 }
3590 return err;3613 return err;
3591}3614}
@@ -3953,6 +3976,10 @@ static int bpf_object__elf_collect(struct bpf_object *obj) · 361 unmodified lines
3953 return -LIBBPF_ERRNO__FORMAT;3976 return -LIBBPF_ERRNO__FORMAT;
3954 }3977 }
3955 3978
3979 /* change BPF program insns to native endianness for introspection */
3980 if (!is_native_endianness(obj))
3981 bpf_object_bswap_progs(obj);
3982
3956 /* sort BPF programs by section name and in-section instruction offset3983 /* sort BPF programs by section name and in-section instruction offset
3957 * for faster search3984 * for faster search
3958 */3985 */
@@ -3985,7 +4012,7 @@ static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx) · 26 unmodified lines
3985 return true;4012 return true;
3986 4013
3987 /* global function */4014 /* global function */
3988 return bind == STB_GLOBAL && type == STT_FUNC;4015 return (bind == STB_GLOBAL || bind == STB_WEAK) && type == STT_FUNC;
3989}4016}
3990 4017
3991static int find_extern_btf_id(const struct btf *btf, const char *ext_name)4018static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
@@ -4389,7 +4416,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 397 unmodified lines
4389 4416
4390static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)4417static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
4391{4418{
4392 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;4419 return prog->sec_idx == obj->efile.text_shndx;
4393}4420}
4394 4421
4395struct bpf_program *4422struct bpf_program *
@@ -4783,8 +4810,8 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) · 387 unmodified lines
4783 fp = fopen(file, "re");4810 fp = fopen(file, "re");
4784 if (!fp) {4811 if (!fp) {
4785 err = -errno;4812 err = -errno;
4786 pr_warn("failed to open %s: %d. No procfs support?\n", file,4813 pr_warn("failed to open %s: %s. No procfs support?\n", file,
4787 err);4814 errstr(err));
4788 return err;4815 return err;
4789 }4816 }
4790 4817
@@ -4939,8 +4966,8 @@ static int bpf_object_prepare_token(struct bpf_object *obj) · 148 unmodified lines
4939 bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR);4966 bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR);
4940 if (bpffs_fd < 0) {4967 if (bpffs_fd < 0) {
4941 err = -errno;4968 err = -errno;
4942 __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n",4969 __pr(level, "object '%s': failed (%s) to open BPF FS mount at '%s'%s\n",
4943 obj->name, err, bpffs_path,4970 obj->name, errstr(err), bpffs_path,
4944 mandatory ? "" : ", skipping optional step...");4971 mandatory ? "" : ", skipping optional step...");
4945 return mandatory ? err : 0;4972 return mandatory ? err : 0;
4946 }4973 }
@@ -4974,7 +5001,6 @@ static int bpf_object_prepare_token(struct bpf_object *obj) · 27 unmodified lines
4974static int5001static int
4975bpf_object__probe_loading(struct bpf_object *obj)5002bpf_object__probe_loading(struct bpf_object *obj)
4976{5003{
4977 char *cp, errmsg[STRERR_BUFSIZE];
4978 struct bpf_insn insns[] = {5004 struct bpf_insn insns[] = {
4979 BPF_MOV64_IMM(BPF_REG_0, 0),5005 BPF_MOV64_IMM(BPF_REG_0, 0),
4980 BPF_EXIT_INSN(),5006 BPF_EXIT_INSN(),
@@ -4990,7 +5016,8 @@ bpf_object__probe_loading(struct bpf_object *obj) · 9 unmodified lines
4990 5016
4991 ret = bump_rlimit_memlock();5017 ret = bump_rlimit_memlock();
4992 if (ret)5018 if (ret)
4993 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);5019 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %s), you might need to do it explicitly!\n",
5020 errstr(ret));
4994 5021
4995 /* make sure basic loading works */5022 /* make sure basic loading works */
4996 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts);5023 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts);
@@ -4998,11 +5025,8 @@ bpf_object__probe_loading(struct bpf_object *obj) · 1 unmodified line
4998 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);5025 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
4999 if (ret < 0) {5026 if (ret < 0) {
5000 ret = errno;5027 ret = errno;
5001 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));5028 pr_warn("Error in %s(): %s. Couldn't load trivial BPF program. Make sure your kernel supports BPF (CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is set to big enough value.\n",
5002 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "5029 __func__, errstr(ret));
5003 "program. Make sure your kernel supports BPF "
5004 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
5005 "set to big enough value.\n", __func__, cp, ret);
5006 return -ret;5030 return -ret;
5007 }5031 }
5008 close(ret);5032 close(ret);
@@ -5027,7 +5051,6 @@ bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) · 18 unmodified lines
5027static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)5051static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
5028{5052{
5029 struct bpf_map_info map_info;5053 struct bpf_map_info map_info;
5030 char msg[STRERR_BUFSIZE];
5031 __u32 map_info_len = sizeof(map_info);5054 __u32 map_info_len = sizeof(map_info);
5032 int err;5055 int err;
5033 5056
@@ -5037,7 +5060,7 @@ static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) · 3 unmodified lines
5037 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);5060 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
5038 if (err) {5061 if (err) {
5039 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,5062 pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
5040 libbpf_strerror_r(errno, msg, sizeof(msg)));5063 errstr(err));
5041 return false;5064 return false;
5042 }5065 }
5043 5066
@@ -5052,7 +5075,6 @@ static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) · 8 unmodified lines
5052static int5075static int
5053bpf_object__reuse_map(struct bpf_map *map)5076bpf_object__reuse_map(struct bpf_map *map)
5054{5077{
5055 char *cp, errmsg[STRERR_BUFSIZE];
5056 int err, pin_fd;5078 int err, pin_fd;
5057 5079
5058 pin_fd = bpf_obj_get(map->pin_path);5080 pin_fd = bpf_obj_get(map->pin_path);
@@ -5064,9 +5086,8 @@ bpf_object__reuse_map(struct bpf_map *map) · 5 unmodified lines
5064 return 0;5086 return 0;
5065 }5087 }
5066 5088
5067 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5068 pr_warn("couldn't retrieve pinned map '%s': %s\n",5089 pr_warn("couldn't retrieve pinned map '%s': %s\n",
5069 map->pin_path, cp);5090 map->pin_path, errstr(err));
5070 return err;5091 return err;
5071 }5092 }
5072 5093
@@ -5092,8 +5113,8 @@ static int · 19 unmodified lines
5092bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)5113bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
5093{5114{
5094 enum libbpf_map_type map_type = map->libbpf_type;5115 enum libbpf_map_type map_type = map->libbpf_type;
5095 char *cp, errmsg[STRERR_BUFSIZE];
5096 int err, zero = 0;5116 int err, zero = 0;
5117 size_t mmap_sz;
5097 5118
5098 if (obj->gen_loader) {5119 if (obj->gen_loader) {
5099 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,5120 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
@@ -5106,9 +5127,8 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) · 6 unmodified lines
5106 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);5127 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5107 if (err) {5128 if (err) {
5108 err = -errno;5129 err = -errno;
5109 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5130 pr_warn("map '%s': failed to set initial contents: %s\n",
5110 pr_warn("Error setting initial map(%s) contents: %s\n",5131 bpf_map__name(map), errstr(err));
5111 map->name, cp);
5112 return err;5132 return err;
5113 }5133 }
5114 5134
@@ -5117,12 +5137,43 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) · 2 unmodified lines
5117 err = bpf_map_freeze(map->fd);5137 err = bpf_map_freeze(map->fd);
5118 if (err) {5138 if (err) {
5119 err = -errno;5139 err = -errno;
5120 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5140 pr_warn("map '%s': failed to freeze as read-only: %s\n",
5121 pr_warn("Error freezing map(%s) as read-only: %s\n",5141 bpf_map__name(map), errstr(err));
5122 map->name, cp);
5123 return err;5142 return err;
5124 }5143 }
5125 }5144 }
5145
5146 /* Remap anonymous mmap()-ed "map initialization image" as
5147 * a BPF map-backed mmap()-ed memory, but preserving the same
5148 * memory address. This will cause kernel to change process'
5149 * page table to point to a different piece of kernel memory,
5150 * but from userspace point of view memory address (and its
5151 * contents, being identical at this point) will stay the
5152 * same. This mapping will be released by bpf_object__close()
5153 * as per normal clean up procedure.
5154 */
5155 mmap_sz = bpf_map_mmap_sz(map);
5156 if (map->def.map_flags & BPF_F_MMAPABLE) {
5157 void *mmaped;
5158 int prot;
5159
5160 if (map->def.map_flags & BPF_F_RDONLY_PROG)
5161 prot = PROT_READ;
5162 else
5163 prot = PROT_READ | PROT_WRITE;
5164 mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map->fd, 0);
5165 if (mmaped == MAP_FAILED) {
5166 err = -errno;
5167 pr_warn("map '%s': failed to re-mmap() contents: %s\n",
5168 bpf_map__name(map), errstr(err));
5169 return err;
5170 }
5171 map->mmaped = mmaped;
5172 } else if (map->mmaped) {
5173 munmap(map->mmaped, mmap_sz);
5174 map->mmaped = NULL;
5175 }
5176
5126 return 0;5177 return 0;
5127}5178}
5128 5179
@@ -5171,8 +5222,8 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · 42 unmodified lines
5171 return err;5222 return err;
5172 err = bpf_object__create_map(obj, map->inner_map, true);5223 err = bpf_object__create_map(obj, map->inner_map, true);
5173 if (err) {5224 if (err) {
5174 pr_warn("map '%s': failed to create inner map: %d\n",5225 pr_warn("map '%s': failed to create inner map: %s\n",
5175 map->name, err);5226 map->name, errstr(err));
5176 return err;5227 return err;
5177 }5228 }
5178 map->inner_map_fd = map->inner_map->fd;5229 map->inner_map_fd = map->inner_map->fd;
@@ -5226,12 +5277,9 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b · 47 unmodified lines
5226 def->max_entries, &create_attr);5277 def->max_entries, &create_attr);
5227 }5278 }
5228 if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) {5279 if (map_fd < 0 && (create_attr.btf_key_type_id || create_attr.btf_value_type_id)) {
5229 char *cp, errmsg[STRERR_BUFSIZE];
5230
5231 err = -errno;5280 err = -errno;
5232 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5281 pr_warn("Error in bpf_create_map_xattr(%s): %s. Retrying without BTF.\n",
5233 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",5282 map->name, errstr(err));
5234 map->name, cp, err);
5235 create_attr.btf_fd = 0;5283 create_attr.btf_fd = 0;
5236 create_attr.btf_key_type_id = 0;5284 create_attr.btf_key_type_id = 0;
5237 create_attr.btf_value_type_id = 0;5285 create_attr.btf_value_type_id = 0;
@@ -5286,8 +5334,8 @@ static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) · 48 unmodified lines
5286 }5334 }
5287 if (err) {5335 if (err) {
5288 err = -errno;5336 err = -errno;
5289 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",5337 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %s\n",
5290 map->name, i, targ_map->name, fd, err);5338 map->name, i, targ_map->name, fd, errstr(err));
5291 return err;5339 return err;
5292 }5340 }
5293 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",5341 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
@@ -5319,8 +5367,8 @@ static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) · 25 unmodified lines
5319 err = bpf_map_update_elem(map->fd, &i, &fd, 0);5367 err = bpf_map_update_elem(map->fd, &i, &fd, 0);
5320 if (err) {5368 if (err) {
5321 err = -errno;5369 err = -errno;
5322 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",5370 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %s\n",
5323 map->name, i, targ_prog->name, fd, err);5371 map->name, i, targ_prog->name, fd, errstr(err));
5324 return err;5372 return err;
5325 }5373 }
5326 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",5374 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
@@ -5373,7 +5421,6 @@ static int · 46 unmodified lines
5373bpf_object__create_maps(struct bpf_object *obj)5421bpf_object__create_maps(struct bpf_object *obj)
5374{5422{
5375 struct bpf_map *map;5423 struct bpf_map *map;
5376 char *cp, errmsg[STRERR_BUFSIZE];
5377 unsigned int i, j;5424 unsigned int i, j;
5378 int err;5425 int err;
5379 bool retried;5426 bool retried;
@@ -5439,8 +5486,7 @@ retry: · 59 unmodified lines
5439 err = bpf_object__populate_internal_map(obj, map);5486 err = bpf_object__populate_internal_map(obj, map);
5440 if (err < 0)5487 if (err < 0)
5441 goto err_out;5488 goto err_out;
5442 }5489 } else if (map->def.type == BPF_MAP_TYPE_ARENA) {
5443 if (map->def.type == BPF_MAP_TYPE_ARENA) {
5444 map->mmaped = mmap((void *)(long)map->map_extra,5490 map->mmaped = mmap((void *)(long)map->map_extra,
5445 bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,5491 bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
5446 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,5492 map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,
@@ -5448,8 +5494,8 @@ retry: · 1 unmodified line
5448 if (map->mmaped == MAP_FAILED) {5494 if (map->mmaped == MAP_FAILED) {
5449 err = -errno;5495 err = -errno;
5450 map->mmaped = NULL;5496 map->mmaped = NULL;
5451 pr_warn("map '%s': failed to mmap arena: %d\n",5497 pr_warn("map '%s': failed to mmap arena: %s\n",
5452 map->name, err);5498 map->name, errstr(err));
5453 return err;5499 return err;
5454 }5500 }
5455 if (obj->arena_data) {5501 if (obj->arena_data) {
@@ -5471,8 +5517,8 @@ retry: · 15 unmodified lines
5471 retried = true;5517 retried = true;
5472 goto retry;5518 goto retry;
5473 }5519 }
5474 pr_warn("map '%s': failed to auto-pin at '%s': %d\n",5520 pr_warn("map '%s': failed to auto-pin at '%s': %s\n",
5475 map->name, map->pin_path, err);5521 map->name, map->pin_path, errstr(err));
5476 goto err_out;5522 goto err_out;
5477 }5523 }
5478 }5524 }
@@ -5481,8 +5527,7 @@ retry: · 2 unmodified lines
5481 return 0;5527 return 0;
5482 5528
5483err_out:5529err_out:
5484 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));5530 pr_warn("map '%s': failed to create: %s\n", map->name, errstr(err));
5485 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);
5486 pr_perm_msg(err);5531 pr_perm_msg(err);
5487 for (j = 0; j < i; j++)5532 for (j = 0; j < i; j++)
5488 zclose(obj->maps[j].fd);5533 zclose(obj->maps[j].fd);
@@ -5606,7 +5651,7 @@ static int load_module_btfs(struct bpf_object *obj) · 117 unmodified lines
5606 }5651 }
5607 if (err) {5652 if (err) {
5608 err = -errno;5653 err = -errno;
5609 pr_warn("failed to iterate BTF objects: %d\n", err);5654 pr_warn("failed to iterate BTF objects: %s\n", errstr(err));
5610 return err;5655 return err;
5611 }5656 }
5612 5657
@@ -5615,7 +5660,7 @@ static int load_module_btfs(struct bpf_object *obj) · 2 unmodified lines
5615 if (errno == ENOENT)5660 if (errno == ENOENT)
5616 continue; /* expected race: BTF was unloaded */5661 continue; /* expected race: BTF was unloaded */
5617 err = -errno;5662 err = -errno;
5618 pr_warn("failed to get BTF object #%d FD: %d\n", id, err);5663 pr_warn("failed to get BTF object #%d FD: %s\n", id, errstr(err));
5619 return err;5664 return err;
5620 }5665 }
5621 5666
@@ -5627,7 +5672,7 @@ static int load_module_btfs(struct bpf_object *obj) · 5 unmodified lines
5627 err = bpf_btf_get_info_by_fd(fd, &info, &len);5672 err = bpf_btf_get_info_by_fd(fd, &info, &len);
5628 if (err) {5673 if (err) {
5629 err = -errno;5674 err = -errno;
5630 pr_warn("failed to get BTF object #%d info: %d\n", id, err);5675 pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err));
5631 goto err_out;5676 goto err_out;
5632 }5677 }
5633 5678
@@ -5640,8 +5685,8 @@ static int load_module_btfs(struct bpf_object *obj) · 6 unmodified lines
5640 btf = btf_get_from_fd(fd, obj->btf_vmlinux);5685 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5641 err = libbpf_get_error(btf);5686 err = libbpf_get_error(btf);
5642 if (err) {5687 if (err) {
5643 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n",5688 pr_warn("failed to load module [%s]'s BTF object #%d: %s\n",
5644 name, id, err);5689 name, id, errstr(err));
5645 goto err_out;5690 goto err_out;
5646 }5691 }
5647 5692
@@ -5870,7 +5915,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) · 222 unmodified lines
5870 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);5915 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5871 err = libbpf_get_error(obj->btf_vmlinux_override);5916 err = libbpf_get_error(obj->btf_vmlinux_override);
5872 if (err) {5917 if (err) {
5873 pr_warn("failed to parse target BTF: %d\n", err);5918 pr_warn("failed to parse target BTF: %s\n", errstr(err));
5874 return err;5919 return err;
5875 }5920 }
5876 }5921 }
@@ -5930,8 +5975,8 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) · 53 unmodified lines
5930 5975
5931 err = record_relo_core(prog, rec, insn_idx);5976 err = record_relo_core(prog, rec, insn_idx);
5932 if (err) {5977 if (err) {
5933 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",5978 pr_warn("prog '%s': relo #%d: failed to record relocation: %s\n",
5934 prog->name, i, err);5979 prog->name, i, errstr(err));
5935 goto out;5980 goto out;
5936 }5981 }
5937 5982
@@ -5940,15 +5985,15 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) · 2 unmodified lines
5940 5985
5941 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);5986 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5942 if (err) {5987 if (err) {
5943 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",5988 pr_warn("prog '%s': relo #%d: failed to relocate: %s\n",
5944 prog->name, i, err);5989 prog->name, i, errstr(err));
5945 goto out;5990 goto out;
5946 }5991 }
5947 5992
5948 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);5993 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
5949 if (err) {5994 if (err) {
5950 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",5995 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %s\n",
5951 prog->name, i, insn_idx, err);5996 prog->name, i, insn_idx, errstr(err));
5952 goto out;5997 goto out;
5953 }5998 }
5954 }5999 }
@@ -6216,8 +6261,8 @@ reloc_prog_func_and_line_info(const struct bpf_object *obj, · 261 unmodified lines
6216 &main_prog->func_info_rec_size);6261 &main_prog->func_info_rec_size);
6217 if (err) {6262 if (err) {
6218 if (err != -ENOENT) {6263 if (err != -ENOENT) {
6219 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",6264 pr_warn("prog '%s': error relocating .BTF.ext function info: %s\n",
6220 prog->name, err);6265 prog->name, errstr(err));
6221 return err;6266 return err;
6222 }6267 }
6223 if (main_prog->func_info) {6268 if (main_prog->func_info) {
@@ -6244,8 +6289,8 @@ line_info: · 20 unmodified lines
6244 &main_prog->line_info_rec_size);6289 &main_prog->line_info_rec_size);
6245 if (err) {6290 if (err) {
6246 if (err != -ENOENT) {6291 if (err != -ENOENT) {
6247 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",6292 pr_warn("prog '%s': error relocating .BTF.ext line info: %s\n",
6248 prog->name, err);6293 prog->name, errstr(err));
6249 return err;6294 return err;
6250 }6295 }
6251 if (main_prog->line_info) {6296 if (main_prog->line_info) {
@@ -7009,8 +7054,8 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat · 757 unmodified lines
7009 if (obj->btf_ext) {7054 if (obj->btf_ext) {
7010 err = bpf_object__relocate_core(obj, targ_btf_path);7055 err = bpf_object__relocate_core(obj, targ_btf_path);
7011 if (err) {7056 if (err) {
7012 pr_warn("failed to perform CO-RE relocations: %d\n",7057 pr_warn("failed to perform CO-RE relocations: %s\n",
7013 err);7058 errstr(err));
7014 return err;7059 return err;
7015 }7060 }
7016 bpf_object__sort_relos(obj);7061 bpf_object__sort_relos(obj);
@@ -7054,8 +7099,8 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat · 37 unmodified lines
7054 7099
7055 err = bpf_object__relocate_calls(obj, prog);7100 err = bpf_object__relocate_calls(obj, prog);
7056 if (err) {7101 if (err) {
7057 pr_warn("prog '%s': failed to relocate calls: %d\n",7102 pr_warn("prog '%s': failed to relocate calls: %s\n",
7058 prog->name, err);7103 prog->name, errstr(err));
7059 return err;7104 return err;
7060 }7105 }
7061 7106
@@ -7091,16 +7136,16 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat · 29 unmodified lines
7091 /* Process data relos for main programs */7136 /* Process data relos for main programs */
7092 err = bpf_object__relocate_data(obj, prog);7137 err = bpf_object__relocate_data(obj, prog);
7093 if (err) {7138 if (err) {
7094 pr_warn("prog '%s': failed to relocate data references: %d\n",7139 pr_warn("prog '%s': failed to relocate data references: %s\n",
7095 prog->name, err);7140 prog->name, errstr(err));
7096 return err;7141 return err;
7097 }7142 }
7098 7143
7099 /* Fix up .BTF.ext information, if necessary */7144 /* Fix up .BTF.ext information, if necessary */
7100 err = bpf_program_fixup_func_info(obj, prog);7145 err = bpf_program_fixup_func_info(obj, prog);
7101 if (err) {7146 if (err) {
7102 pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %d\n",7147 pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %s\n",
7103 prog->name, err);7148 prog->name, errstr(err));
7104 return err;7149 return err;
7105 }7150 }
7106 }7151 }
@@ -7352,8 +7397,14 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, · 245 unmodified lines
7352 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;7397 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;
7353 7398
7354 /* special check for usdt to use uprobe_multi link */7399 /* special check for usdt to use uprobe_multi link */
7355 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))7400 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) {
7401 /* for BPF_TRACE_UPROBE_MULTI, user might want to query expected_attach_type
7402 * in prog, and expected_attach_type we set in kernel is from opts, so we
7403 * update both.
7404 */
7356 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;7405 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7406 opts->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7407 }
7357 7408
7358 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {7409 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7359 int btf_obj_fd = 0, btf_type_id = 0, err;7410 int btf_obj_fd = 0, btf_type_id = 0, err;
@@ -7403,7 +7454,6 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog · 43 unmodified lines
7403{7454{
7404 LIBBPF_OPTS(bpf_prog_load_opts, load_attr);7455 LIBBPF_OPTS(bpf_prog_load_opts, load_attr);
7405 const char *prog_name = NULL;7456 const char *prog_name = NULL;
7406 char *cp, errmsg[STRERR_BUFSIZE];
7407 size_t log_buf_size = 0;7457 size_t log_buf_size = 0;
7408 char *log_buf = NULL, *tmp;7458 char *log_buf = NULL, *tmp;
7409 bool own_log_buf = true;7459 bool own_log_buf = true;
@@ -7443,6 +7493,7 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog · 33 unmodified lines
7443 load_attr.attach_btf_id = prog->attach_btf_id;7493 load_attr.attach_btf_id = prog->attach_btf_id;
7444 load_attr.kern_version = kern_version;7494 load_attr.kern_version = kern_version;
7445 load_attr.prog_ifindex = prog->prog_ifindex;7495 load_attr.prog_ifindex = prog->prog_ifindex;
7496 load_attr.expected_attach_type = prog->expected_attach_type;
7446 7497
7447 /* specify func_info/line_info only if kernel supports them */7498 /* specify func_info/line_info only if kernel supports them */
7448 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {7499 if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
@@ -7466,17 +7517,14 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog · 17 unmodified lines
7466 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {7517 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7467 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);7518 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7468 if (err < 0) {7519 if (err < 0) {
7469 pr_warn("prog '%s': failed to prepare load attributes: %d\n",7520 pr_warn("prog '%s': failed to prepare load attributes: %s\n",
7470 prog->name, err);7521 prog->name, errstr(err));
7471 return err;7522 return err;
7472 }7523 }
7473 insns = prog->insns;7524 insns = prog->insns;
7474 insns_cnt = prog->insns_cnt;7525 insns_cnt = prog->insns_cnt;
7475 }7526 }
7476 7527
7477 /* allow prog_prepare_load_fn to change expected_attach_type */
7478 load_attr.expected_attach_type = prog->expected_attach_type;
7479
7480 if (obj->gen_loader) {7528 if (obj->gen_loader) {
7481 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,7529 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7482 license, insns, insns_cnt, &load_attr,7530 license, insns, insns_cnt, &load_attr,
@@ -7534,9 +7582,8 @@ retry_load: · 51 unmodified lines
7534 continue;7582 continue;
7535 7583
7536 if (bpf_prog_bind_map(ret, map->fd, NULL)) {7584 if (bpf_prog_bind_map(ret, map->fd, NULL)) {
7537 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
7538 pr_warn("prog '%s': failed to bind map '%s': %s\n",7585 pr_warn("prog '%s': failed to bind map '%s': %s\n",
7539 prog->name, map->real_name, cp);7586 prog->name, map->real_name, errstr(errno));
7540 /* Don't fail hard if can't bind rodata. */7587 /* Don't fail hard if can't bind rodata. */
7541 }7588 }
7542 }7589 }
@@ -7566,8 +7613,7 @@ retry_load: · 23 unmodified lines
7566 /* post-process verifier log to improve error descriptions */7613 /* post-process verifier log to improve error descriptions */
7567 fixup_verifier_log(prog, log_buf, log_buf_size);7614 fixup_verifier_log(prog, log_buf, log_buf_size);
7568 7615
7569 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));7616 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, errstr(errno));
7570 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7571 pr_perm_msg(ret);7617 pr_perm_msg(ret);
7572 7618
7573 if (own_log_buf && log_buf && log_buf[0] != '\0') {7619 if (own_log_buf && log_buf && log_buf[0] != '\0') {
@@ -7860,7 +7906,7 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) · 286 unmodified lines
7860 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,7906 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7861 obj->license, obj->kern_version, &prog->fd);7907 obj->license, obj->kern_version, &prog->fd);
7862 if (err) {7908 if (err) {
7863 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);7909 pr_warn("prog '%s': failed to load: %s\n", prog->name, errstr(err));
7864 return err;7910 return err;
7865 }7911 }
7866 }7912 }
@@ -7894,8 +7940,8 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object · 27 unmodified lines
7894 if (prog->sec_def->prog_setup_fn) {7940 if (prog->sec_def->prog_setup_fn) {
7895 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);7941 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7896 if (err < 0) {7942 if (err < 0) {
7897 pr_warn("prog '%s': failed to initialize: %d\n",7943 pr_warn("prog '%s': failed to initialize: %s\n",
7898 prog->name, err);7944 prog->name, errstr(err));
7899 return err;7945 return err;
7900 }7946 }
7901 }7947 }
@@ -7992,7 +8038,6 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, · 90 unmodified lines
7992 }8038 }
7993 8039
7994 err = bpf_object__elf_init(obj);8040 err = bpf_object__elf_init(obj);
7995 err = err ? : bpf_object__check_endianness(obj);
7996 err = err ? : bpf_object__elf_collect(obj);8041 err = err ? : bpf_object__elf_collect(obj);
7997 err = err ? : bpf_object__collect_externs(obj);8042 err = err ? : bpf_object__collect_externs(obj);
7998 err = err ? : bpf_object_fixup_btf(obj);8043 err = err ? : bpf_object_fixup_btf(obj);
@@ -8085,7 +8130,7 @@ static int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx) · 86 unmodified lines
8085 f = fopen("/proc/kallsyms", "re");8130 f = fopen("/proc/kallsyms", "re");
8086 if (!f) {8131 if (!f) {
8087 err = -errno;8132 err = -errno;
8088 pr_warn("failed to open /proc/kallsyms: %d\n", err);8133 pr_warn("failed to open /proc/kallsyms: %s\n", errstr(err));
8089 return err;8134 return err;
8090 }8135 }
8091 8136
@@ -8498,8 +8543,15 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch · 406 unmodified lines
8498 return libbpf_err(-EINVAL);8543 return libbpf_err(-EINVAL);
8499 }8544 }
8500 8545
8501 if (obj->gen_loader)8546 /* Disallow kernel loading programs of non-native endianness but
8547 * permit cross-endian creation of "light skeleton".
8548 */
8549 if (obj->gen_loader) {
8502 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);8550 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
8551 } else if (!is_native_endianness(obj)) {
8552 pr_warn("object '%s': loading non-native endianness is unsupported\n", obj->name);
8553 return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
8554 }
8503 8555
8504 err = bpf_object_prepare_token(obj);8556 err = bpf_object_prepare_token(obj);
8505 err = err ? : bpf_object__probe_loading(obj);8557 err = err ? : bpf_object__probe_loading(obj);
@@ -8562,7 +8614,6 @@ int bpf_object__load(struct bpf_object *obj) · 56 unmodified lines
8562 8614
8563static int make_parent_dir(const char *path)8615static int make_parent_dir(const char *path)
8564{8616{
8565 char *cp, errmsg[STRERR_BUFSIZE];
8566 char *dname, *dir;8617 char *dname, *dir;
8567 int err = 0;8618 int err = 0;
8568 8619
@@ -8576,15 +8627,13 @@ static int make_parent_dir(const char *path) · 7 unmodified lines
8576 8627
8577 free(dname);8628 free(dname);
8578 if (err) {8629 if (err) {
8579 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));8630 pr_warn("failed to mkdir %s: %s\n", path, errstr(err));
8580 pr_warn("failed to mkdir %s: %s\n", path, cp);
8581 }8631 }
8582 return err;8632 return err;
8583}8633}
8584 8634
8585static int check_path(const char *path)8635static int check_path(const char *path)
8586{8636{
8587 char *cp, errmsg[STRERR_BUFSIZE];
8588 struct statfs st_fs;8637 struct statfs st_fs;
8589 char *dname, *dir;8638 char *dname, *dir;
8590 int err = 0;8639 int err = 0;
@@ -8598,8 +8647,7 @@ static int check_path(const char *path) · 7 unmodified lines
8598 8647
8599 dir = dirname(dname);8648 dir = dirname(dname);
8600 if (statfs(dir, &st_fs)) {8649 if (statfs(dir, &st_fs)) {
8601 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));8650 pr_warn("failed to statfs %s: %s\n", dir, errstr(errno));
8602 pr_warn("failed to statfs %s: %s\n", dir, cp);
8603 err = -errno;8651 err = -errno;
8604 }8652 }
8605 free(dname);8653 free(dname);
@@ -8614,7 +8662,6 @@ static int check_path(const char *path) · 8 unmodified lines
8614 8662
8615int bpf_program__pin(struct bpf_program *prog, const char *path)8663int bpf_program__pin(struct bpf_program *prog, const char *path)
8616{8664{
8617 char *cp, errmsg[STRERR_BUFSIZE];
8618 int err;8665 int err;
8619 8666
8620 if (prog->fd < 0) {8667 if (prog->fd < 0) {
@@ -8632,8 +8679,7 @@ int bpf_program__pin(struct bpf_program *prog, const char *path) · 11 unmodified lines
8632 8679
8633 if (bpf_obj_pin(prog->fd, path)) {8680 if (bpf_obj_pin(prog->fd, path)) {
8634 err = -errno;8681 err = -errno;
8635 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));8682 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, errstr(err));
8636 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8637 return libbpf_err(err);8683 return libbpf_err(err);
8638 }8684 }
8639 8685
@@ -8664,7 +8710,6 @@ int bpf_program__unpin(struct bpf_program *prog, const char *path) · 24 unmodified lines
8664 8710
8665int bpf_map__pin(struct bpf_map *map, const char *path)8711int bpf_map__pin(struct bpf_map *map, const char *path)
8666{8712{
8667 char *cp, errmsg[STRERR_BUFSIZE];
8668 int err;8713 int err;
8669 8714
8670 if (map == NULL) {8715 if (map == NULL) {
@@ -8723,8 +8768,7 @@ int bpf_map__pin(struct bpf_map *map, const char *path) · 52 unmodified lines
8723 return 0;8768 return 0;
8724 8769
8725out_err:8770out_err:
8726 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));8771 pr_warn("failed to pin map: %s\n", errstr(err));
8727 pr_warn("failed to pin map: %s\n", cp);
8728 return libbpf_err(err);8772 return libbpf_err(err);
8729}8773}
8730 8774
@@ -9096,6 +9140,7 @@ int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) · 365 unmodified lines
9096 if (!gen)9140 if (!gen)
9097 return -ENOMEM;9141 return -ENOMEM;
9098 gen->opts = opts;9142 gen->opts = opts;
9143 gen->swapped_endian = !is_native_endianness(obj);
9099 obj->gen_loader = gen;9144 obj->gen_loader = gen;
9100 return 0;9145 return 0;
9101}9146}
@@ -9370,8 +9415,10 @@ static const struct bpf_sec_def section_defs[] = { · 268 unmodified lines
9370 SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),9415 SEC_DEF("kprobe.session+", KPROBE, BPF_TRACE_KPROBE_SESSION, SEC_NONE, attach_kprobe_session),
9371 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),9416 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9372 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),9417 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi),
9418 SEC_DEF("uprobe.session+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_NONE, attach_uprobe_multi),
9373 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),9419 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9374 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),9420 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi),
9421 SEC_DEF("uprobe.session.s+", KPROBE, BPF_TRACE_UPROBE_SESSION, SEC_SLEEPABLE, attach_uprobe_multi),
9375 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),9422 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
9376 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),9423 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall),
9377 SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),9424 SEC_DEF("usdt+", KPROBE, 0, SEC_USDT, attach_usdt),
@@ -9910,8 +9957,8 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) · 532 unmodified lines
9910 memset(&info, 0, info_len);9957 memset(&info, 0, info_len);
9911 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len);9958 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len);
9912 if (err) {9959 if (err) {
9913 pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n",9960 pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %s\n",
9914 attach_prog_fd, err);9961 attach_prog_fd, errstr(err));
9915 return err;9962 return err;
9916 }9963 }
9917 9964
@@ -9923,7 +9970,7 @@ static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) · 5 unmodified lines
9923 btf = btf__load_from_kernel_by_id(info.btf_id);9970 btf = btf__load_from_kernel_by_id(info.btf_id);
9924 err = libbpf_get_error(btf);9971 err = libbpf_get_error(btf);
9925 if (err) {9972 if (err) {
9926 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err);9973 pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err));
9927 goto out;9974 goto out;
9928 }9975 }
9929 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);9976 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC);
@@ -10005,8 +10052,8 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac · 75 unmodified lines
10005 }10052 }
10006 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);10053 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
10007 if (err < 0) {10054 if (err < 0) {
10008 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",10055 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n",
10009 prog->name, attach_prog_fd, attach_name, err);10056 prog->name, attach_prog_fd, attach_name, errstr(err));
10010 return err;10057 return err;
10011 }10058 }
10012 *btf_obj_fd = 0;10059 *btf_obj_fd = 0;
@@ -10025,8 +10072,8 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac · 12 unmodified lines
10025 btf_type_id);10072 btf_type_id);
10026 }10073 }
10027 if (err) {10074 if (err) {
10028 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",10075 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %s\n",
10029 prog->name, attach_name, err);10076 prog->name, attach_name, errstr(err));
10030 return err;10077 return err;
10031 }10078 }
10032 return 0;10079 return 0;
@@ -10254,14 +10301,14 @@ int bpf_map__set_value_size(struct bpf_map *map, __u32 size) · 221 unmodified lines
10254 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);10301 mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
10255 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz);10302 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz);
10256 if (err) {10303 if (err) {
10257 pr_warn("map '%s': failed to resize memory-mapped region: %d\n",10304 pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
10258 bpf_map__name(map), err);10305 bpf_map__name(map), errstr(err));
10259 return err;10306 return err;
10260 }10307 }
10261 err = map_btf_datasec_resize(map, size);10308 err = map_btf_datasec_resize(map, size);
10262 if (err && err != -ENOENT) {10309 if (err && err != -ENOENT) {
10263 pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n",10310 pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %s\n",
10264 bpf_map__name(map), err);10311 bpf_map__name(map), errstr(err));
10265 map->btf_value_type_id = 0;10312 map->btf_value_type_id = 0;
10266 map->btf_key_type_id = 0;10313 map->btf_key_type_id = 0;
10267 }10314 }
@@ -10752,7 +10799,6 @@ static void bpf_link_perf_dealloc(struct bpf_link *link) · 484 unmodified lines
10752struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,10799struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
10753 const struct bpf_perf_event_opts *opts)10800 const struct bpf_perf_event_opts *opts)
10754{10801{
10755 char errmsg[STRERR_BUFSIZE];
10756 struct bpf_link_perf *link;10802 struct bpf_link_perf *link;
10757 int prog_fd, link_fd = -1, err;10803 int prog_fd, link_fd = -1, err;
10758 bool force_ioctl_attach;10804 bool force_ioctl_attach;
@@ -10787,9 +10833,8 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p · 28 unmodified lines
10787 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);10833 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts);
10788 if (link_fd < 0) {10834 if (link_fd < 0) {
10789 err = -errno;10835 err = -errno;
10790 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",10836 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %s\n",
10791 prog->name, pfd,10837 prog->name, pfd, errstr(err));
10792 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10793 goto err_out;10838 goto err_out;
10794 }10839 }
10795 link->link.fd = link_fd;10840 link->link.fd = link_fd;
@@ -10803,7 +10848,7 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p · 7 unmodified lines
10803 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {10848 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
10804 err = -errno;10849 err = -errno;
10805 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",10850 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
10806 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));10851 prog->name, pfd, errstr(err));
10807 if (err == -EPROTO)10852 if (err == -EPROTO)
10808 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",10853 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
10809 prog->name, pfd);10854 prog->name, pfd);
@@ -10814,7 +10859,7 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p · 4 unmodified lines
10814 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {10859 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
10815 err = -errno;10860 err = -errno;
10816 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",10861 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
10817 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));10862 prog->name, pfd, errstr(err));
10818 goto err_out;10863 goto err_out;
10819 }10864 }
10820 10865
@@ -10838,22 +10883,19 @@ struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, · 17 unmodified lines
10838 */10883 */
10839static int parse_uint_from_file(const char *file, const char *fmt)10884static int parse_uint_from_file(const char *file, const char *fmt)
10840{10885{
10841 char buf[STRERR_BUFSIZE];
10842 int err, ret;10886 int err, ret;
10843 FILE *f;10887 FILE *f;
10844 10888
10845 f = fopen(file, "re");10889 f = fopen(file, "re");
10846 if (!f) {10890 if (!f) {
10847 err = -errno;10891 err = -errno;
10848 pr_debug("failed to open '%s': %s\n", file,10892 pr_debug("failed to open '%s': %s\n", file, errstr(err));
10849 libbpf_strerror_r(err, buf, sizeof(buf)));
10850 return err;10893 return err;
10851 }10894 }
10852 err = fscanf(f, fmt, &ret);10895 err = fscanf(f, fmt, &ret);
10853 if (err != 1) {10896 if (err != 1) {
10854 err = err == EOF ? -EIO : -errno;10897 err = err == EOF ? -EIO : -errno;
10855 pr_debug("failed to parse '%s': %s\n", file,10898 pr_debug("failed to parse '%s': %s\n", file, errstr(err));
10856 libbpf_strerror_r(err, buf, sizeof(buf)));
10857 fclose(f);10899 fclose(f);
10858 return err;10900 return err;
10859 }10901 }
@@ -10897,7 +10939,6 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 37 unmodified lines
10897{10939{
10898 const size_t attr_sz = sizeof(struct perf_event_attr);10940 const size_t attr_sz = sizeof(struct perf_event_attr);
10899 struct perf_event_attr attr;10941 struct perf_event_attr attr;
10900 char errmsg[STRERR_BUFSIZE];
10901 int type, pfd;10942 int type, pfd;
10902 10943
10903 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))10944 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
@@ -10910,7 +10951,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 6 unmodified lines
10910 if (type < 0) {10951 if (type < 0) {
10911 pr_warn("failed to determine %s perf type: %s\n",10952 pr_warn("failed to determine %s perf type: %s\n",
10912 uprobe ? "uprobe" : "kprobe",10953 uprobe ? "uprobe" : "kprobe",
10913 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));10954 errstr(type));
10914 return type;10955 return type;
10915 }10956 }
10916 if (retprobe) {10957 if (retprobe) {
@@ -10920,7 +10961,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 3 unmodified lines
10920 if (bit < 0) {10961 if (bit < 0) {
10921 pr_warn("failed to determine %s retprobe bit: %s\n",10962 pr_warn("failed to determine %s retprobe bit: %s\n",
10922 uprobe ? "uprobe" : "kprobe",10963 uprobe ? "uprobe" : "kprobe",
10923 libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));10964 errstr(bit));
10924 return bit;10965 return bit;
10925 }10966 }
10926 attr.config |= 1 << bit;10967 attr.config |= 1 << bit;
@@ -11049,14 +11090,13 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, · 122 unmodified lines
11049{11090{
11050 const size_t attr_sz = sizeof(struct perf_event_attr);11091 const size_t attr_sz = sizeof(struct perf_event_attr);
11051 struct perf_event_attr attr;11092 struct perf_event_attr attr;
11052 char errmsg[STRERR_BUFSIZE];
11053 int type, pfd, err;11093 int type, pfd, err;
11054 11094
11055 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);11095 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset);
11056 if (err < 0) {11096 if (err < 0) {
11057 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",11097 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n",
11058 kfunc_name, offset,11098 kfunc_name, offset,
11059 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11099 errstr(err));
11060 return err;11100 return err;
11061 }11101 }
11062 type = determine_kprobe_perf_type_legacy(probe_name, retprobe);11102 type = determine_kprobe_perf_type_legacy(probe_name, retprobe);
@@ -11064,7 +11104,7 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, · 1 unmodified line
11064 err = type;11104 err = type;
11065 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",11105 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n",
11066 kfunc_name, offset,11106 kfunc_name, offset,
11067 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11107 errstr(err));
11068 goto err_clean_legacy;11108 goto err_clean_legacy;
11069 }11109 }
11070 11110
@@ -11080,7 +11120,7 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, · 9 unmodified lines
11080 if (pfd < 0) {11120 if (pfd < 0) {
11081 err = -errno;11121 err = -errno;
11082 pr_warn("legacy kprobe perf_event_open() failed: %s\n",11122 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
11083 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11123 errstr(err));
11084 goto err_clean_legacy;11124 goto err_clean_legacy;
11085 }11125 }
11086 return pfd;11126 return pfd;
@@ -11156,7 +11196,6 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, · 69 unmodified lines
11156{11196{
11157 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);11197 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
11158 enum probe_attach_mode attach_mode;11198 enum probe_attach_mode attach_mode;
11159 char errmsg[STRERR_BUFSIZE];
11160 char *legacy_probe = NULL;11199 char *legacy_probe = NULL;
11161 struct bpf_link *link;11200 struct bpf_link *link;
11162 size_t offset;11201 size_t offset;
@@ -11214,7 +11253,7 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, · 51 unmodified lines
11214 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",11253 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
11215 prog->name, retprobe ? "kretprobe" : "kprobe",11254 prog->name, retprobe ? "kretprobe" : "kprobe",
11216 func_name, offset,11255 func_name, offset,
11217 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11256 errstr(err));
11218 goto err_out;11257 goto err_out;
11219 }11258 }
11220 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);11259 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
@@ -11224,7 +11263,7 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, · 3 unmodified lines
11224 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",11263 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
11225 prog->name, retprobe ? "kretprobe" : "kprobe",11264 prog->name, retprobe ? "kretprobe" : "kprobe",
11226 func_name, offset,11265 func_name, offset,
11227 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11266 errstr(err));
11228 goto err_clean_legacy;11267 goto err_clean_legacy;
11229 }11268 }
11230 if (legacy) {11269 if (legacy) {
@@ -11360,7 +11399,7 @@ static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) · 129 unmodified lines
11360 f = fopen(available_functions_file, "re");11399 f = fopen(available_functions_file, "re");
11361 if (!f) {11400 if (!f) {
11362 err = -errno;11401 err = -errno;
11363 pr_warn("failed to open %s: %d\n", available_functions_file, err);11402 pr_warn("failed to open %s: %s\n", available_functions_file, errstr(err));
11364 return err;11403 return err;
11365 }11404 }
11366 11405
@@ -11435,7 +11474,7 @@ static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) · 68 unmodified lines
11435 f = fopen(available_path, "re");11474 f = fopen(available_path, "re");
11436 if (!f) {11475 if (!f) {
11437 err = -errno;11476 err = -errno;
11438 pr_warn("failed to open %s: %d\n", available_path, err);11477 pr_warn("failed to open %s: %s\n", available_path, errstr(err));
11439 return err;11478 return err;
11440 }11479 }
11441 11480
@@ -11481,7 +11520,6 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, · 39 unmodified lines
11481 };11520 };
11482 enum bpf_attach_type attach_type;11521 enum bpf_attach_type attach_type;
11483 struct bpf_link *link = NULL;11522 struct bpf_link *link = NULL;
11484 char errmsg[STRERR_BUFSIZE];
11485 const unsigned long *addrs;11523 const unsigned long *addrs;
11486 int err, link_fd, prog_fd;11524 int err, link_fd, prog_fd;
11487 bool retprobe, session;11525 bool retprobe, session;
@@ -11549,7 +11587,7 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, · 61 unmodified lines
11549 if (link_fd < 0) {11587 if (link_fd < 0) {
11550 err = -errno;11588 err = -errno;
11551 pr_warn("prog '%s': failed to attach: %s\n",11589 pr_warn("prog '%s': failed to attach: %s\n",
11552 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));11590 prog->name, errstr(err));
11553 goto error;11591 goto error;
11554 }11592 }
11555 link->fd = link_fd;11593 link->fd = link_fd;
@@ -11693,7 +11731,9 @@ static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, stru · 137 unmodified lines
11693 ret = 0;11731 ret = 0;
11694 break;11732 break;
11695 case 3:11733 case 3:
11734 opts.session = str_has_pfx(probe_type, "uprobe.session");
11696 opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi");11735 opts.retprobe = str_has_pfx(probe_type, "uretprobe.multi");
11736
11697 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);11737 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11698 ret = libbpf_get_error(*link);11738 ret = libbpf_get_error(*link);
11699 break;11739 break;
@@ -11756,15 +11796,15 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, · 56 unmodified lines
11756 11796
11757 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);11797 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset);
11758 if (err < 0) {11798 if (err < 0) {
11759 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n",11799 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %s\n",
11760 binary_path, (size_t)offset, err);11800 binary_path, (size_t)offset, errstr(err));
11761 return err;11801 return err;
11762 }11802 }
11763 type = determine_uprobe_perf_type_legacy(probe_name, retprobe);11803 type = determine_uprobe_perf_type_legacy(probe_name, retprobe);
11764 if (type < 0) {11804 if (type < 0) {
11765 err = type;11805 err = type;
11766 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n",11806 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %s\n",
11767 binary_path, offset, err);11807 binary_path, offset, errstr(err));
11768 goto err_clean_legacy;11808 goto err_clean_legacy;
11769 }11809 }
11770 11810
@@ -11779,7 +11819,7 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, · 8 unmodified lines
11779 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);11819 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
11780 if (pfd < 0) {11820 if (pfd < 0) {
11781 err = -errno;11821 err = -errno;
11782 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err);11822 pr_warn("legacy uprobe perf_event_open() failed: %s\n", errstr(err));
11783 goto err_clean_legacy;11823 goto err_clean_legacy;
11784 }11824 }
11785 return pfd;11825 return pfd;
@@ -11942,10 +11982,11 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 156 unmodified lines
11942 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;11982 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL;
11943 LIBBPF_OPTS(bpf_link_create_opts, lopts);11983 LIBBPF_OPTS(bpf_link_create_opts, lopts);
11944 unsigned long *resolved_offsets = NULL;11984 unsigned long *resolved_offsets = NULL;
11985 enum bpf_attach_type attach_type;
11945 int err = 0, link_fd, prog_fd;11986 int err = 0, link_fd, prog_fd;
11946 struct bpf_link *link = NULL;11987 struct bpf_link *link = NULL;
11947 char errmsg[STRERR_BUFSIZE];
11948 char full_path[PATH_MAX];11988 char full_path[PATH_MAX];
11989 bool retprobe, session;
11949 const __u64 *cookies;11990 const __u64 *cookies;
11950 const char **syms;11991 const char **syms;
11951 size_t cnt;11992 size_t cnt;
@@ -11965,6 +12006,8 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 13 unmodified lines
11965 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);12006 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
11966 cookies = OPTS_GET(opts, cookies, NULL);12007 cookies = OPTS_GET(opts, cookies, NULL);
11967 cnt = OPTS_GET(opts, cnt, 0);12008 cnt = OPTS_GET(opts, cnt, 0);
12009 retprobe = OPTS_GET(opts, retprobe, false);
12010 session = OPTS_GET(opts, session, false);
11968 12011
11969 /*12012 /*
11970 * User can specify 2 mutually exclusive set of inputs:12013 * User can specify 2 mutually exclusive set of inputs:
@@ -11993,12 +12036,15 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 22 unmodified lines
11993 return libbpf_err_ptr(-EINVAL);12036 return libbpf_err_ptr(-EINVAL);
11994 }12037 }
11995 12038
12039 if (retprobe && session)
12040 return libbpf_err_ptr(-EINVAL);
12041
11996 if (func_pattern) {12042 if (func_pattern) {
11997 if (!strchr(path, '/')) {12043 if (!strchr(path, '/')) {
11998 err = resolve_full_path(path, full_path, sizeof(full_path));12044 err = resolve_full_path(path, full_path, sizeof(full_path));
11999 if (err) {12045 if (err) {
12000 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",12046 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12001 prog->name, path, err);12047 prog->name, path, errstr(err));
12002 return libbpf_err_ptr(err);12048 return libbpf_err_ptr(err);
12003 }12049 }
12004 path = full_path;12050 path = full_path;
@@ -12016,12 +12062,14 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 11 unmodified lines
12016 offsets = resolved_offsets;12062 offsets = resolved_offsets;
12017 }12063 }
12018 12064
12065 attach_type = session ? BPF_TRACE_UPROBE_SESSION : BPF_TRACE_UPROBE_MULTI;
12066
12019 lopts.uprobe_multi.path = path;12067 lopts.uprobe_multi.path = path;
12020 lopts.uprobe_multi.offsets = offsets;12068 lopts.uprobe_multi.offsets = offsets;
12021 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;12069 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets;
12022 lopts.uprobe_multi.cookies = cookies;12070 lopts.uprobe_multi.cookies = cookies;
12023 lopts.uprobe_multi.cnt = cnt;12071 lopts.uprobe_multi.cnt = cnt;
12024 lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0;12072 lopts.uprobe_multi.flags = retprobe ? BPF_F_UPROBE_MULTI_RETURN : 0;
12025 12073
12026 if (pid == 0)12074 if (pid == 0)
12027 pid = getpid();12075 pid = getpid();
@@ -12035,11 +12083,11 @@ bpf_program__attach_uprobe_multi(const struct bpf_program *prog, · 7 unmodified lines
12035 }12083 }
12036 link->detach = &bpf_link__detach_fd;12084 link->detach = &bpf_link__detach_fd;
12037 12085
12038 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);12086 link_fd = bpf_link_create(prog_fd, 0, attach_type, &lopts);
12039 if (link_fd < 0) {12087 if (link_fd < 0) {
12040 err = -errno;12088 err = -errno;
12041 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",12089 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
12042 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12090 prog->name, errstr(err));
12043 goto error;12091 goto error;
12044 }12092 }
12045 link->fd = link_fd;12093 link->fd = link_fd;
@@ -12058,7 +12106,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 12 unmodified lines
12058 const struct bpf_uprobe_opts *opts)12106 const struct bpf_uprobe_opts *opts)
12059{12107{
12060 const char *archive_path = NULL, *archive_sep = NULL;12108 const char *archive_path = NULL, *archive_sep = NULL;
12061 char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL;12109 char *legacy_probe = NULL;
12062 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);12110 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
12063 enum probe_attach_mode attach_mode;12111 enum probe_attach_mode attach_mode;
12064 char full_path[PATH_MAX];12112 char full_path[PATH_MAX];
@@ -12090,8 +12138,8 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 25 unmodified lines
12090 } else if (!strchr(binary_path, '/')) {12138 } else if (!strchr(binary_path, '/')) {
12091 err = resolve_full_path(binary_path, full_path, sizeof(full_path));12139 err = resolve_full_path(binary_path, full_path, sizeof(full_path));
12092 if (err) {12140 if (err) {
12093 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",12141 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12094 prog->name, binary_path, err);12142 prog->name, binary_path, errstr(err));
12095 return libbpf_err_ptr(err);12143 return libbpf_err_ptr(err);
12096 }12144 }
12097 binary_path = full_path;12145 binary_path = full_path;
@@ -12157,7 +12205,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 59 unmodified lines
12157 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",12205 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
12158 prog->name, retprobe ? "uretprobe" : "uprobe",12206 prog->name, retprobe ? "uretprobe" : "uprobe",
12159 binary_path, func_offset,12207 binary_path, func_offset,
12160 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12208 errstr(err));
12161 goto err_out;12209 goto err_out;
12162 }12210 }
12163 12211
@@ -12168,7 +12216,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, · 4 unmodified lines
12168 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",12216 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
12169 prog->name, retprobe ? "uretprobe" : "uprobe",12217 prog->name, retprobe ? "uretprobe" : "uprobe",
12170 binary_path, func_offset,12218 binary_path, func_offset,
12171 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12219 errstr(err));
12172 goto err_clean_legacy;12220 goto err_clean_legacy;
12173 }12221 }
12174 if (legacy) {12222 if (legacy) {
@@ -12289,8 +12337,8 @@ struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, · 114 unmodified lines
12289 if (!strchr(binary_path, '/')) {12337 if (!strchr(binary_path, '/')) {
12290 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));12338 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path));
12291 if (err) {12339 if (err) {
12292 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",12340 pr_warn("prog '%s': failed to resolve full path for '%s': %s\n",
12293 prog->name, binary_path, err);12341 prog->name, binary_path, errstr(err));
12294 return libbpf_err_ptr(err);12342 return libbpf_err_ptr(err);
12295 }12343 }
12296 binary_path = resolved_path;12344 binary_path = resolved_path;
@@ -12368,14 +12416,13 @@ static int perf_event_open_tracepoint(const char *tp_category, · 71 unmodified lines
12368{12416{
12369 const size_t attr_sz = sizeof(struct perf_event_attr);12417 const size_t attr_sz = sizeof(struct perf_event_attr);
12370 struct perf_event_attr attr;12418 struct perf_event_attr attr;
12371 char errmsg[STRERR_BUFSIZE];
12372 int tp_id, pfd, err;12419 int tp_id, pfd, err;
12373 12420
12374 tp_id = determine_tracepoint_id(tp_category, tp_name);12421 tp_id = determine_tracepoint_id(tp_category, tp_name);
12375 if (tp_id < 0) {12422 if (tp_id < 0) {
12376 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",12423 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
12377 tp_category, tp_name,12424 tp_category, tp_name,
12378 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));12425 errstr(tp_id));
12379 return tp_id;12426 return tp_id;
12380 }12427 }
12381 12428
@@ -12390,7 +12437,7 @@ static int perf_event_open_tracepoint(const char *tp_category, · 8 unmodified lines
12390 err = -errno;12437 err = -errno;
12391 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",12438 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
12392 tp_category, tp_name,12439 tp_category, tp_name,
12393 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12440 errstr(err));
12394 return err;12441 return err;
12395 }12442 }
12396 return pfd;12443 return pfd;
@@ -12402,7 +12449,6 @@ struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *p · 5 unmodified lines
12402 const struct bpf_tracepoint_opts *opts)12449 const struct bpf_tracepoint_opts *opts)
12403{12450{
12404 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);12451 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
12405 char errmsg[STRERR_BUFSIZE];
12406 struct bpf_link *link;12452 struct bpf_link *link;
12407 int pfd, err;12453 int pfd, err;
12408 12454
@@ -12415,7 +12461,7 @@ struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *p · 6 unmodified lines
12415 if (pfd < 0) {12461 if (pfd < 0) {
12416 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",12462 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
12417 prog->name, tp_category, tp_name,12463 prog->name, tp_category, tp_name,
12418 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));12464 errstr(pfd));
12419 return libbpf_err_ptr(pfd);12465 return libbpf_err_ptr(pfd);
12420 }12466 }
12421 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);12467 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
@@ -12424,7 +12470,7 @@ struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *p · 2 unmodified lines
12424 close(pfd);12470 close(pfd);
12425 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",12471 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
12426 prog->name, tp_category, tp_name,12472 prog->name, tp_category, tp_name,
12427 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));12473 errstr(err));
12428 return libbpf_err_ptr(err);12474 return libbpf_err_ptr(err);
12429 }12475 }
12430 return link;12476 return link;
@@ -12475,7 +12521,6 @@ bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, · 44 unmodified lines
12475 struct bpf_raw_tracepoint_opts *opts)12521 struct bpf_raw_tracepoint_opts *opts)
12476{12522{
12477 LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);12523 LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);
12478 char errmsg[STRERR_BUFSIZE];
12479 struct bpf_link *link;12524 struct bpf_link *link;
12480 int prog_fd, pfd;12525 int prog_fd, pfd;
12481 12526
@@ -12500,7 +12545,7 @@ bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog, · 18 unmodified lines
12500 pfd = -errno;12545 pfd = -errno;
12501 free(link);12546 free(link);
12502 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",12547 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
12503 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));12548 prog->name, tp_name, errstr(pfd));
12504 return libbpf_err_ptr(pfd);12549 return libbpf_err_ptr(pfd);
12505 }12550 }
12506 link->fd = pfd;12551 link->fd = pfd;
@@ -12559,7 +12604,6 @@ static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *pro · 52 unmodified lines
12559 const struct bpf_trace_opts *opts)12604 const struct bpf_trace_opts *opts)
12560{12605{
12561 LIBBPF_OPTS(bpf_link_create_opts, link_opts);12606 LIBBPF_OPTS(bpf_link_create_opts, link_opts);
12562 char errmsg[STRERR_BUFSIZE];
12563 struct bpf_link *link;12607 struct bpf_link *link;
12564 int prog_fd, pfd;12608 int prog_fd, pfd;
12565 12609
@@ -12584,7 +12628,7 @@ static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *pro · 18 unmodified lines
12584 pfd = -errno;12628 pfd = -errno;
12585 free(link);12629 free(link);
12586 pr_warn("prog '%s': failed to attach: %s\n",12630 pr_warn("prog '%s': failed to attach: %s\n",
12587 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));12631 prog->name, errstr(pfd));
12588 return libbpf_err_ptr(pfd);12632 return libbpf_err_ptr(pfd);
12589 }12633 }
12590 link->fd = pfd;12634 link->fd = pfd;
@@ -12625,7 +12669,6 @@ bpf_program_attach_fd(const struct bpf_program *prog, · 34 unmodified lines
12625 const struct bpf_link_create_opts *opts)12669 const struct bpf_link_create_opts *opts)
12626{12670{
12627 enum bpf_attach_type attach_type;12671 enum bpf_attach_type attach_type;
12628 char errmsg[STRERR_BUFSIZE];
12629 struct bpf_link *link;12672 struct bpf_link *link;
12630 int prog_fd, link_fd;12673 int prog_fd, link_fd;
12631 12674
@@ -12647,7 +12690,7 @@ bpf_program_attach_fd(const struct bpf_program *prog, · 15 unmodified lines
12647 free(link);12690 free(link);
12648 pr_warn("prog '%s': failed to attach to %s: %s\n",12691 pr_warn("prog '%s': failed to attach to %s: %s\n",
12649 prog->name, target_name,12692 prog->name, target_name,
12650 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));12693 errstr(link_fd));
12651 return libbpf_err_ptr(link_fd);12694 return libbpf_err_ptr(link_fd);
12652 }12695 }
12653 link->fd = link_fd;12696 link->fd = link_fd;
@@ -12760,7 +12803,7 @@ struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, · 106 unmodified lines
12760 }12803 }
12761 12804
12762 if (prog->type != BPF_PROG_TYPE_EXT) {12805 if (prog->type != BPF_PROG_TYPE_EXT) {
12763 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",12806 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace\n",
12764 prog->name);12807 prog->name);
12765 return libbpf_err_ptr(-EINVAL);12808 return libbpf_err_ptr(-EINVAL);
12766 }12809 }
@@ -12789,7 +12832,6 @@ bpf_program__attach_iter(const struct bpf_program *prog, · 22 unmodified lines
12789 const struct bpf_iter_attach_opts *opts)12832 const struct bpf_iter_attach_opts *opts)
12790{12833{
12791 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);12834 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
12792 char errmsg[STRERR_BUFSIZE];
12793 struct bpf_link *link;12835 struct bpf_link *link;
12794 int prog_fd, link_fd;12836 int prog_fd, link_fd;
12795 __u32 target_fd = 0;12837 __u32 target_fd = 0;
@@ -12817,7 +12859,7 @@ bpf_program__attach_iter(const struct bpf_program *prog, · 21 unmodified lines
12817 link_fd = -errno;12859 link_fd = -errno;
12818 free(link);12860 free(link);
12819 pr_warn("prog '%s': failed to attach to iterator: %s\n",12861 pr_warn("prog '%s': failed to attach to iterator: %s\n",
12820 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));12862 prog->name, errstr(link_fd));
12821 return libbpf_err_ptr(link_fd);12863 return libbpf_err_ptr(link_fd);
12822 }12864 }
12823 link->fd = link_fd;12865 link->fd = link_fd;
@@ -12859,12 +12901,10 @@ struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, · 35 unmodified lines
12859 12901
12860 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);12902 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts);
12861 if (link_fd < 0) {12903 if (link_fd < 0) {
12862 char errmsg[STRERR_BUFSIZE];
12863
12864 link_fd = -errno;12904 link_fd = -errno;
12865 free(link);12905 free(link);
12866 pr_warn("prog '%s': failed to attach to netfilter: %s\n",12906 pr_warn("prog '%s': failed to attach to netfilter: %s\n",
12867 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));12907 prog->name, errstr(link_fd));
12868 return libbpf_err_ptr(link_fd);12908 return libbpf_err_ptr(link_fd);
12869 }12909 }
12870 link->fd = link_fd;12910 link->fd = link_fd;
@@ -13149,7 +13189,6 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, · 278 unmodified lines
13149 int cpu, int map_key)13189 int cpu, int map_key)
13150{13190{
13151 struct perf_cpu_buf *cpu_buf;13191 struct perf_cpu_buf *cpu_buf;
13152 char msg[STRERR_BUFSIZE];
13153 int err;13192 int err;
13154 13193
13155 cpu_buf = calloc(1, sizeof(*cpu_buf));13194 cpu_buf = calloc(1, sizeof(*cpu_buf));
@@ -13165,7 +13204,7 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, · 9 unmodified lines
13165 if (cpu_buf->fd < 0) {13204 if (cpu_buf->fd < 0) {
13166 err = -errno;13205 err = -errno;
13167 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",13206 pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
13168 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));13207 cpu, errstr(err));
13169 goto error;13208 goto error;
13170 }13209 }
13171 13210
@@ -13176,14 +13215,14 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, · 4 unmodified lines
13176 cpu_buf->base = NULL;13215 cpu_buf->base = NULL;
13177 err = -errno;13216 err = -errno;
13178 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",13217 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
13179 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));13218 cpu, errstr(err));
13180 goto error;13219 goto error;
13181 }13220 }
13182 13221
13183 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {13222 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
13184 err = -errno;13223 err = -errno;
13185 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",13224 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
13186 cpu, libbpf_strerror_r(err, msg, sizeof(msg)));13225 cpu, errstr(err));
13187 goto error;13226 goto error;
13188 }13227 }
13189 13228
@@ -13259,7 +13298,6 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 69 unmodified lines
13259{13298{
13260 const char *online_cpus_file = "/sys/devices/system/cpu/online";13299 const char *online_cpus_file = "/sys/devices/system/cpu/online";
13261 struct bpf_map_info map;13300 struct bpf_map_info map;
13262 char msg[STRERR_BUFSIZE];
13263 struct perf_buffer *pb;13301 struct perf_buffer *pb;
13264 bool *online = NULL;13302 bool *online = NULL;
13265 __u32 map_info_len;13303 __u32 map_info_len;
@@ -13282,7 +13320,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 16 unmodified lines
13282 */13320 */
13283 if (err != -EINVAL) {13321 if (err != -EINVAL) {
13284 pr_warn("failed to get map info for map FD %d: %s\n",13322 pr_warn("failed to get map info for map FD %d: %s\n",
13285 map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));13323 map_fd, errstr(err));
13286 return ERR_PTR(err);13324 return ERR_PTR(err);
13287 }13325 }
13288 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",13326 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n",
@@ -13312,7 +13350,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 23 unmodified lines
13312 if (pb->epoll_fd < 0) {13350 if (pb->epoll_fd < 0) {
13313 err = -errno;13351 err = -errno;
13314 pr_warn("failed to create epoll instance: %s\n",13352 pr_warn("failed to create epoll instance: %s\n",
13315 libbpf_strerror_r(err, msg, sizeof(msg)));13353 errstr(err));
13316 goto error;13354 goto error;
13317 }13355 }
13318 13356
@@ -13343,7 +13381,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 24 unmodified lines
13343 13381
13344 err = parse_cpu_mask_file(online_cpus_file, &online, &n);13382 err = parse_cpu_mask_file(online_cpus_file, &online, &n);
13345 if (err) {13383 if (err) {
13346 pr_warn("failed to get online CPU mask: %d\n", err);13384 pr_warn("failed to get online CPU mask: %s\n", errstr(err));
13347 goto error;13385 goto error;
13348 }13386 }
13349 13387
@@ -13374,7 +13412,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 24 unmodified lines
13374 err = -errno;13412 err = -errno;
13375 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",13413 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
13376 cpu, map_key, cpu_buf->fd,13414 cpu, map_key, cpu_buf->fd,
13377 libbpf_strerror_r(err, msg, sizeof(msg)));13415 errstr(err));
13378 goto error;13416 goto error;
13379 }13417 }
13380 13418
@@ -13385,7 +13423,7 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 4 unmodified lines
13385 err = -errno;13423 err = -errno;
13386 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",13424 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
13387 cpu, cpu_buf->fd,13425 cpu, cpu_buf->fd,
13388 libbpf_strerror_r(err, msg, sizeof(msg)));13426 errstr(err));
13389 goto error;13427 goto error;
13390 }13428 }
13391 j++;13429 j++;
@@ -13480,7 +13518,7 @@ int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) · 88 unmodified lines
13480 13518
13481 err = perf_buffer__process_records(pb, cpu_buf);13519 err = perf_buffer__process_records(pb, cpu_buf);
13482 if (err) {13520 if (err) {
13483 pr_warn("error while processing records: %d\n", err);13521 pr_warn("error while processing records: %s\n", errstr(err));
13484 return libbpf_err(err);13522 return libbpf_err(err);
13485 }13523 }
13486 }13524 }
@@ -13564,7 +13602,8 @@ int perf_buffer__consume(struct perf_buffer *pb) · 77 unmodified lines
13564 13602
13565 err = perf_buffer__process_records(pb, cpu_buf);13603 err = perf_buffer__process_records(pb, cpu_buf);
13566 if (err) {13604 if (err) {
13567 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err);13605 pr_warn("perf_buffer: failed to process records in buffer #%d: %s\n",
13606 i, errstr(err));
13568 return libbpf_err(err);13607 return libbpf_err(err);
13569 }13608 }
13570 }13609 }
@@ -13675,14 +13714,14 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) · 104 unmodified lines
13675 fd = open(fcpu, O_RDONLY | O_CLOEXEC);13714 fd = open(fcpu, O_RDONLY | O_CLOEXEC);
13676 if (fd < 0) {13715 if (fd < 0) {
13677 err = -errno;13716 err = -errno;
13678 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err);13717 pr_warn("Failed to open cpu mask file %s: %s\n", fcpu, errstr(err));
13679 return err;13718 return err;
13680 }13719 }
13681 len = read(fd, buf, sizeof(buf));13720 len = read(fd, buf, sizeof(buf));
13682 close(fd);13721 close(fd);
13683 if (len <= 0) {13722 if (len <= 0) {
13684 err = len ? -errno : -EINVAL;13723 err = len ? -errno : -EINVAL;
13685 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err);13724 pr_warn("Failed to read cpu mask from %s: %s\n", fcpu, errstr(err));
13686 return err;13725 return err;
13687 }13726 }
13688 if (len >= sizeof(buf)) {13727 if (len >= sizeof(buf)) {
@@ -13774,20 +13813,21 @@ int bpf_object__open_skeleton(struct bpf_object_skeleton *s, · 85 unmodified lines
13774 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);13813 obj = bpf_object_open(NULL, s->data, s->data_sz, s->name, opts);
13775 if (IS_ERR(obj)) {13814 if (IS_ERR(obj)) {
13776 err = PTR_ERR(obj);13815 err = PTR_ERR(obj);
13777 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", s->name, err);13816 pr_warn("failed to initialize skeleton BPF object '%s': %s\n",
13817 s->name, errstr(err));
13778 return libbpf_err(err);13818 return libbpf_err(err);
13779 }13819 }
13780 13820
13781 *s->obj = obj;13821 *s->obj = obj;
13782 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);13822 err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);
13783 if (err) {13823 if (err) {
13784 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);13824 pr_warn("failed to populate skeleton maps for '%s': %s\n", s->name, errstr(err));
13785 return libbpf_err(err);13825 return libbpf_err(err);
13786 }13826 }
13787 13827
13788 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);13828 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13789 if (err) {13829 if (err) {
13790 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);13830 pr_warn("failed to populate skeleton progs for '%s': %s\n", s->name, errstr(err));
13791 return libbpf_err(err);13831 return libbpf_err(err);
13792 }13832 }
13793 13833
@@ -13817,13 +13857,13 @@ int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) · 23 unmodified lines
13817 13857
13818 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);13858 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);
13819 if (err) {13859 if (err) {
13820 pr_warn("failed to populate subskeleton maps: %d\n", err);13860 pr_warn("failed to populate subskeleton maps: %s\n", errstr(err));
13821 return libbpf_err(err);13861 return libbpf_err(err);
13822 }13862 }
13823 13863
13824 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);13864 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);
13825 if (err) {13865 if (err) {
13826 pr_warn("failed to populate subskeleton maps: %d\n", err);13866 pr_warn("failed to populate subskeleton maps: %s\n", errstr(err));
13827 return libbpf_err(err);13867 return libbpf_err(err);
13828 }13868 }
13829 13869
@@ -13834,7 +13874,7 @@ int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) · 4 unmodified lines
13834 map_type = btf__type_by_id(btf, map_type_id);13874 map_type = btf__type_by_id(btf, map_type_id);
13835 13875
13836 if (!btf_is_datasec(map_type)) {13876 if (!btf_is_datasec(map_type)) {
13837 pr_warn("type for map '%1$s' is not a datasec: %2$s",13877 pr_warn("type for map '%1$s' is not a datasec: %2$s\n",
13838 bpf_map__name(map),13878 bpf_map__name(map),
13839 __btf_kind_str(btf_kind(map_type)));13879 __btf_kind_str(btf_kind(map_type)));
13840 return libbpf_err(-EINVAL);13880 return libbpf_err(-EINVAL);
@@ -13870,53 +13910,18 @@ int bpf_object__load_skeleton(struct bpf_object_skeleton *s) · 29 unmodified lines
13870 13910
13871 err = bpf_object__load(*s->obj);13911 err = bpf_object__load(*s->obj);
13872 if (err) {13912 if (err) {
13873 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);13913 pr_warn("failed to load BPF skeleton '%s': %s\n", s->name, errstr(err));
13874 return libbpf_err(err);13914 return libbpf_err(err);
13875 }13915 }
13876 13916
13877 for (i = 0; i < s->map_cnt; i++) {13917 for (i = 0; i < s->map_cnt; i++) {
13878 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;13918 struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;
13879 struct bpf_map *map = *map_skel->map;13919 struct bpf_map *map = *map_skel->map;
13880 size_t mmap_sz = bpf_map_mmap_sz(map);
13881 int prot, map_fd = map->fd;
13882 void **mmaped = map_skel->mmaped;
13883 13920
13884 if (!mmaped)13921 if (!map_skel->mmaped)
13885 continue;13922 continue;
13886 13923
13887 if (!(map->def.map_flags & BPF_F_MMAPABLE)) {13924 *map_skel->mmaped = map->mmaped;
13888 *mmaped = NULL;
13889 continue;
13890 }
13891
13892 if (map->def.type == BPF_MAP_TYPE_ARENA) {
13893 *mmaped = map->mmaped;
13894 continue;
13895 }
13896
13897 if (map->def.map_flags & BPF_F_RDONLY_PROG)
13898 prot = PROT_READ;
13899 else
13900 prot = PROT_READ | PROT_WRITE;
13901
13902 /* Remap anonymous mmap()-ed "map initialization image" as
13903 * a BPF map-backed mmap()-ed memory, but preserving the same
13904 * memory address. This will cause kernel to change process'
13905 * page table to point to a different piece of kernel memory,
13906 * but from userspace point of view memory address (and its
13907 * contents, being identical at this point) will stay the
13908 * same. This mapping will be released by bpf_object__close()
13909 * as per normal clean up procedure, so we don't need to worry
13910 * about it from skeleton's clean up perspective.
13911 */
13912 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0);
13913 if (*mmaped == MAP_FAILED) {
13914 err = -errno;
13915 *mmaped = NULL;
13916 pr_warn("failed to re-mmap() map '%s': %d\n",
13917 bpf_map__name(map), err);
13918 return libbpf_err(err);
13919 }
13920 }13925 }
13921 13926
13922 return 0;13927 return 0;
@@ -13944,8 +13949,8 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) · 21 unmodified lines
13944 13949
13945 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);13950 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
13946 if (err) {13951 if (err) {
13947 pr_warn("prog '%s': failed to auto-attach: %d\n",13952 pr_warn("prog '%s': failed to auto-attach: %s\n",
13948 bpf_program__name(prog), err);13953 bpf_program__name(prog), errstr(err));
13949 return libbpf_err(err);13954 return libbpf_err(err);
13950 }13955 }
13951 13956
@@ -13988,7 +13993,8 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) · 36 unmodified lines
13988 *link = bpf_map__attach_struct_ops(map);13993 *link = bpf_map__attach_struct_ops(map);
13989 if (!*link) {13994 if (!*link) {
13990 err = -errno;13995 err = -errno;
13991 pr_warn("map '%s': failed to auto-attach: %d\n", bpf_map__name(map), err);13996 pr_warn("map '%s': failed to auto-attach: %s\n",
13997 bpf_map__name(map), errstr(err));
13992 return libbpf_err(err);13998 return libbpf_err(err);
13993 }13999 }
13994 }14000 }
tools/lib/bpf/libbpf.h+3−1
@@ -577,10 +577,12 @@ struct bpf_uprobe_multi_opts { · 576 unmodified lines
577 size_t cnt;577 size_t cnt;
578 /* create return uprobes */578 /* create return uprobes */
579 bool retprobe;579 bool retprobe;
580 /* create session kprobes */
581 bool session;
580 size_t :0;582 size_t :0;
581};583};
582 584
583#define bpf_uprobe_multi_opts__last_field retprobe585#define bpf_uprobe_multi_opts__last_field session
584 586
585/**587/**
586 * @brief **bpf_program__attach_uprobe_multi()** attaches a BPF program588 * @brief **bpf_program__attach_uprobe_multi()** attaches a BPF program
tools/lib/bpf/libbpf.map+5−0
@@ -421,6 +421,8 @@ LIBBPF_1.5.0 { · 420 unmodified lines
421 global:421 global:
422 btf__distill_base;422 btf__distill_base;
423 btf__relocate;423 btf__relocate;
424 btf_ext__endianness;
425 btf_ext__set_endianness;
424 bpf_map__autoattach;426 bpf_map__autoattach;
425 bpf_map__set_autoattach;427 bpf_map__set_autoattach;
426 bpf_object__token_fd;428 bpf_object__token_fd;
@@ -428,3 +430,6 @@ LIBBPF_1.5.0 { · 1 unmodified line
428 ring__consume_n;430 ring__consume_n;
429 ring_buffer__consume_n;431 ring_buffer__consume_n;
430} LIBBPF_1.4.0;432} LIBBPF_1.4.0;
433
434LIBBPF_1.6.0 {
435} LIBBPF_1.5.0;
tools/lib/bpf/libbpf_internal.h+41−2
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#define __LIBBPF_LIBBPF_INTERNAL_H10#define __LIBBPF_LIBBPF_INTERNAL_H
11 11
12#include <stdlib.h>12#include <stdlib.h>
13#include <byteswap.h>
13#include <limits.h>14#include <limits.h>
14#include <errno.h>15#include <errno.h>
15#include <linux/err.h>16#include <linux/err.h>
@@ -448,11 +449,11 @@ struct btf_ext_info { · 432 unmodified lines
448 *449 *
449 * The func_info subsection layout:450 * The func_info subsection layout:
450 * record size for struct bpf_func_info in the func_info subsection451 * record size for struct bpf_func_info in the func_info subsection
451 * struct btf_sec_func_info for section #1452 * struct btf_ext_info_sec for section #1
452 * a list of bpf_func_info records for section #1453 * a list of bpf_func_info records for section #1
453 * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h454 * where struct bpf_func_info mimics one in include/uapi/linux/bpf.h
454 * but may not be identical455 * but may not be identical
455 * struct btf_sec_func_info for section #2456 * struct btf_ext_info_sec for section #2
456 * a list of bpf_func_info records for section #2457 * a list of bpf_func_info records for section #2
457 * ......458 * ......
458 *459 *
@@ -484,6 +485,8 @@ struct btf_ext { · 25 unmodified lines
484 struct btf_ext_header *hdr;485 struct btf_ext_header *hdr;
485 void *data;486 void *data;
486 };487 };
488 void *data_swapped;
489 bool swapped_endian;
487 struct btf_ext_info func_info;490 struct btf_ext_info func_info;
488 struct btf_ext_info line_info;491 struct btf_ext_info line_info;
489 struct btf_ext_info core_relo_info;492 struct btf_ext_info core_relo_info;
@@ -511,6 +514,32 @@ struct bpf_line_info_min { · 21 unmodified lines
511 __u32 line_col;514 __u32 line_col;
512};515};
513 516
517/* Functions to byte-swap info records */
518
519typedef void (*info_rec_bswap_fn)(void *);
520
521static inline void bpf_func_info_bswap(struct bpf_func_info *i)
522{
523 i->insn_off = bswap_32(i->insn_off);
524 i->type_id = bswap_32(i->type_id);
525}
526
527static inline void bpf_line_info_bswap(struct bpf_line_info *i)
528{
529 i->insn_off = bswap_32(i->insn_off);
530 i->file_name_off = bswap_32(i->file_name_off);
531 i->line_off = bswap_32(i->line_off);
532 i->line_col = bswap_32(i->line_col);
533}
534
535static inline void bpf_core_relo_bswap(struct bpf_core_relo *i)
536{
537 i->insn_off = bswap_32(i->insn_off);
538 i->type_id = bswap_32(i->type_id);
539 i->access_str_off = bswap_32(i->access_str_off);
540 i->kind = bswap_32(i->kind);
541}
542
514enum btf_field_iter_kind {543enum btf_field_iter_kind {
515 BTF_FIELD_ITER_IDS,544 BTF_FIELD_ITER_IDS,
516 BTF_FIELD_ITER_STRS,545 BTF_FIELD_ITER_STRS,
@@ -588,6 +617,16 @@ static inline bool is_ldimm64_insn(struct bpf_insn *insn) · 71 unmodified lines
588 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);617 return insn->code == (BPF_LD | BPF_IMM | BPF_DW);
589}618}
590 619
620static inline void bpf_insn_bswap(struct bpf_insn *insn)
621{
622 __u8 tmp_reg = insn->dst_reg;
623
624 insn->dst_reg = insn->src_reg;
625 insn->src_reg = tmp_reg;
626 insn->off = bswap_16(insn->off);
627 insn->imm = bswap_32(insn->imm);
628}
629
591/* Unconditionally dup FD, ensuring it doesn't use [0, 2] range.630/* Unconditionally dup FD, ensuring it doesn't use [0, 2] range.
592 * Original FD is not closed or altered in any other way.631 * Original FD is not closed or altered in any other way.
593 * Preserves original FD value, if it's invalid (negative).632 * Preserves original FD value, if it's invalid (negative).
tools/lib/bpf/libbpf_version.h+1−1
@@ -4,6 +4,6 @@ · 3 unmodified lines
4#define __LIBBPF_VERSION_H4#define __LIBBPF_VERSION_H
5 5
6#define LIBBPF_MAJOR_VERSION 16#define LIBBPF_MAJOR_VERSION 1
7#define LIBBPF_MINOR_VERSION 57#define LIBBPF_MINOR_VERSION 6
8 8
9#endif /* __LIBBPF_VERSION_H */9#endif /* __LIBBPF_VERSION_H */
tools/lib/bpf/linker.c+80−25
@@ -20,6 +20,7 @@ · 19 unmodified lines
20#include "btf.h"20#include "btf.h"
21#include "libbpf_internal.h"21#include "libbpf_internal.h"
22#include "strset.h"22#include "strset.h"
23#include "str_error.h"
23 24
24#define BTF_EXTERN_SEC ".extern"25#define BTF_EXTERN_SEC ".extern"
25 26
@@ -135,6 +136,7 @@ struct bpf_linker { · 109 unmodified lines
135 int fd;136 int fd;
136 Elf *elf;137 Elf *elf;
137 Elf64_Ehdr *elf_hdr;138 Elf64_Ehdr *elf_hdr;
139 bool swapped_endian;
138 140
139 /* Output sections metadata */141 /* Output sections metadata */
140 struct dst_sec *secs;142 struct dst_sec *secs;
@@ -305,7 +307,7 @@ static int init_output_elf(struct bpf_linker *linker, const char *file) · 164 unmodified lines
305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);307 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
306 if (linker->fd < 0) {308 if (linker->fd < 0) {
307 err = -errno;309 err = -errno;
308 pr_warn("failed to create '%s': %d\n", file, err);310 pr_warn("failed to create '%s': %s\n", file, errstr(err));
309 return err;311 return err;
310 }312 }
311 313
@@ -324,13 +326,8 @@ static int init_output_elf(struct bpf_linker *linker, const char *file) · 12 unmodified lines
324 326
325 linker->elf_hdr->e_machine = EM_BPF;327 linker->elf_hdr->e_machine = EM_BPF;
326 linker->elf_hdr->e_type = ET_REL;328 linker->elf_hdr->e_type = ET_REL;
327#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__329 /* Set unknown ELF endianness, assign later from input files */
328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB;330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATANONE;
329#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB;
331#else
332#error "Unknown __BYTE_ORDER__"
333#endif
334 331
335 /* STRTAB */332 /* STRTAB */
336 /* initialize strset with an empty string to conform to ELF */333 /* initialize strset with an empty string to conform to ELF */
@@ -396,6 +393,8 @@ static int init_output_elf(struct bpf_linker *linker, const char *file) · 59 unmodified lines
396 pr_warn_elf("failed to create SYMTAB data");393 pr_warn_elf("failed to create SYMTAB data");
397 return -EINVAL;394 return -EINVAL;
398 }395 }
396 /* Ensure libelf translates byte-order of symbol records */
397 sec->data->d_type = ELF_T_SYM;
399 398
400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);399 str_off = strset__add_str(linker->strtab_strs, sec->sec_name);
401 if (str_off < 0)400 if (str_off < 0)
@@ -539,19 +538,21 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 137 unmodified lines
539 const struct bpf_linker_file_opts *opts,538 const struct bpf_linker_file_opts *opts,
540 struct src_obj *obj)539 struct src_obj *obj)
541{540{
542#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
543 const int host_endianness = ELFDATA2LSB;
544#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
545 const int host_endianness = ELFDATA2MSB;
546#else
547#error "Unknown __BYTE_ORDER__"
548#endif
549 int err = 0;541 int err = 0;
550 Elf_Scn *scn;542 Elf_Scn *scn;
551 Elf_Data *data;543 Elf_Data *data;
552 Elf64_Ehdr *ehdr;544 Elf64_Ehdr *ehdr;
553 Elf64_Shdr *shdr;545 Elf64_Shdr *shdr;
554 struct src_sec *sec;546 struct src_sec *sec;
547 unsigned char obj_byteorder;
548 unsigned char link_byteorder = linker->elf_hdr->e_ident[EI_DATA];
549#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
550 const unsigned char host_byteorder = ELFDATA2LSB;
551#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
552 const unsigned char host_byteorder = ELFDATA2MSB;
553#else
554#error "Unknown __BYTE_ORDER__"
555#endif
555 556
556 pr_debug("linker: adding object file '%s'...\n", filename);557 pr_debug("linker: adding object file '%s'...\n", filename);
557 558
@@ -560,7 +561,7 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 2 unmodified lines
560 obj->fd = open(filename, O_RDONLY | O_CLOEXEC);561 obj->fd = open(filename, O_RDONLY | O_CLOEXEC);
561 if (obj->fd < 0) {562 if (obj->fd < 0) {
562 err = -errno;563 err = -errno;
563 pr_warn("failed to open file '%s': %d\n", filename, err);564 pr_warn("failed to open file '%s': %s\n", filename, errstr(err));
564 return err;565 return err;
565 }566 }
566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);567 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL);
@@ -577,11 +578,25 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 10 unmodified lines
577 pr_warn_elf("failed to get ELF header for %s", filename);578 pr_warn_elf("failed to get ELF header for %s", filename);
578 return err;579 return err;
579 }580 }
580 if (ehdr->e_ident[EI_DATA] != host_endianness) {581
582 /* Linker output endianness set by first input object */
583 obj_byteorder = ehdr->e_ident[EI_DATA];
584 if (obj_byteorder != ELFDATA2LSB && obj_byteorder != ELFDATA2MSB) {
581 err = -EOPNOTSUPP;585 err = -EOPNOTSUPP;
582 pr_warn_elf("unsupported byte order of ELF file %s", filename);586 pr_warn("unknown byte order of ELF file %s\n", filename);
583 return err;587 return err;
584 }588 }
589 if (link_byteorder == ELFDATANONE) {
590 linker->elf_hdr->e_ident[EI_DATA] = obj_byteorder;
591 linker->swapped_endian = obj_byteorder != host_byteorder;
592 pr_debug("linker: set %s-endian output byte order\n",
593 obj_byteorder == ELFDATA2MSB ? "big" : "little");
594 } else if (link_byteorder != obj_byteorder) {
595 err = -EOPNOTSUPP;
596 pr_warn("byte order mismatch with ELF file %s\n", filename);
597 return err;
598 }
599
585 if (ehdr->e_type != ET_REL600 if (ehdr->e_type != ET_REL
586 || ehdr->e_machine != EM_BPF601 || ehdr->e_machine != EM_BPF
587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {602 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
@@ -656,7 +671,8 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 68 unmodified lines
656 obj->btf = btf__new(data->d_buf, shdr->sh_size);671 obj->btf = btf__new(data->d_buf, shdr->sh_size);
657 err = libbpf_get_error(obj->btf);672 err = libbpf_get_error(obj->btf);
658 if (err) {673 if (err) {
659 pr_warn("failed to parse .BTF from %s: %d\n", filename, err);674 pr_warn("failed to parse .BTF from %s: %s\n",
675 filename, errstr(err));
660 return err;676 return err;
661 }677 }
662 sec->skipped = true;678 sec->skipped = true;
@@ -666,7 +682,8 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, · 3 unmodified lines
666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);682 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size);
667 err = libbpf_get_error(obj->btf_ext);683 err = libbpf_get_error(obj->btf_ext);
668 if (err) {684 if (err) {
669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err);685 pr_warn("failed to parse .BTF.ext from '%s': %s\n",
686 filename, errstr(err));
670 return err;687 return err;
671 }688 }
672 sec->skipped = true;689 sec->skipped = true;
@@ -1109,6 +1126,24 @@ static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec · 436 unmodified lines
1109 return true;1126 return true;
1110}1127}
1111 1128
1129static bool is_exec_sec(struct dst_sec *sec)
1130{
1131 if (!sec || sec->ephemeral)
1132 return false;
1133 return (sec->shdr->sh_type == SHT_PROGBITS) &&
1134 (sec->shdr->sh_flags & SHF_EXECINSTR);
1135}
1136
1137static void exec_sec_bswap(void *raw_data, int size)
1138{
1139 const int insn_cnt = size / sizeof(struct bpf_insn);
1140 struct bpf_insn *insn = raw_data;
1141 int i;
1142
1143 for (i = 0; i < insn_cnt; i++, insn++)
1144 bpf_insn_bswap(insn);
1145}
1146
1112static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)1147static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src)
1113{1148{
1114 void *tmp;1149 void *tmp;
@@ -1168,6 +1203,10 @@ static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src · 53 unmodified lines
1168 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);1203 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz);
1169 /* now copy src data at a properly aligned offset */1204 /* now copy src data at a properly aligned offset */
1170 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);1205 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size);
1206
1207 /* convert added bpf insns to native byte-order */
1208 if (linker->swapped_endian && is_exec_sec(dst))
1209 exec_sec_bswap(dst->raw_data + dst_align_sz, src->shdr->sh_size);
1171 }1210 }
1172 1211
1173 dst->sec_sz = dst_final_sz;1212 dst->sec_sz = dst_final_sz;
@@ -2415,6 +2454,10 @@ static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) · 1241 unmodified lines
2415 if (glob_sym && glob_sym->var_idx >= 0) {2454 if (glob_sym && glob_sym->var_idx >= 0) {
2416 __s64 sz;2455 __s64 sz;
2417 2456
2457 /* FUNCs don't have size, nothing to update */
2458 if (btf_is_func(t))
2459 continue;
2460
2418 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];2461 dst_var = &dst_sec->sec_vars[glob_sym->var_idx];
2419 /* Because underlying BTF type might have2462 /* Because underlying BTF type might have
2420 * changed, so might its size have changed, so2463 * changed, so might its size have changed, so
@@ -2628,6 +2671,10 @@ int bpf_linker__finalize(struct bpf_linker *linker) · 207 unmodified lines
2628 if (!sec->scn)2671 if (!sec->scn)
2629 continue;2672 continue;
2630 2673
2674 /* restore sections with bpf insns to target byte-order */
2675 if (linker->swapped_endian && is_exec_sec(sec))
2676 exec_sec_bswap(sec->raw_data, sec->sec_sz);
2677
2631 sec->data->d_buf = sec->raw_data;2678 sec->data->d_buf = sec->raw_data;
2632 }2679 }
2633 2680
@@ -2696,6 +2743,7 @@ static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, · 62 unmodified lines
2696 2743
2697static int finalize_btf(struct bpf_linker *linker)2744static int finalize_btf(struct bpf_linker *linker)
2698{2745{
2746 enum btf_endianness link_endianness;
2699 LIBBPF_OPTS(btf_dedup_opts, opts);2747 LIBBPF_OPTS(btf_dedup_opts, opts);
2700 struct btf *btf = linker->btf;2748 struct btf *btf = linker->btf;
2701 const void *raw_data;2749 const void *raw_data;
@@ -2729,17 +2777,24 @@ static int finalize_btf(struct bpf_linker *linker) · 27 unmodified lines
2729 2777
2730 err = finalize_btf_ext(linker);2778 err = finalize_btf_ext(linker);
2731 if (err) {2779 if (err) {
2732 pr_warn(".BTF.ext generation failed: %d\n", err);2780 pr_warn(".BTF.ext generation failed: %s\n", errstr(err));
2733 return err;2781 return err;
2734 }2782 }
2735 2783
2736 opts.btf_ext = linker->btf_ext;2784 opts.btf_ext = linker->btf_ext;
2737 err = btf__dedup(linker->btf, &opts);2785 err = btf__dedup(linker->btf, &opts);
2738 if (err) {2786 if (err) {
2739 pr_warn("BTF dedup failed: %d\n", err);2787 pr_warn("BTF dedup failed: %s\n", errstr(err));
2740 return err;2788 return err;
2741 }2789 }
2742 2790
2791 /* Set .BTF and .BTF.ext output byte order */
2792 link_endianness = linker->elf_hdr->e_ident[EI_DATA] == ELFDATA2MSB ?
2793 BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
2794 btf__set_endianness(linker->btf, link_endianness);
2795 if (linker->btf_ext)
2796 btf_ext__set_endianness(linker->btf_ext, link_endianness);
2797
2743 /* Emit .BTF section */2798 /* Emit .BTF section */
2744 raw_data = btf__raw_data(linker->btf, &raw_sz);2799 raw_data = btf__raw_data(linker->btf, &raw_sz);
2745 if (!raw_data)2800 if (!raw_data)
@@ -2747,7 +2802,7 @@ static int finalize_btf(struct bpf_linker *linker) · 1 unmodified line
2747 2802
2748 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);2803 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz);
2749 if (err) {2804 if (err) {
2750 pr_warn("failed to write out .BTF ELF section: %d\n", err);2805 pr_warn("failed to write out .BTF ELF section: %s\n", errstr(err));
2751 return err;2806 return err;
2752 }2807 }
2753 2808
@@ -2759,7 +2814,7 @@ static int finalize_btf(struct bpf_linker *linker) · 5 unmodified lines
2759 2814
2760 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);2815 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz);
2761 if (err) {2816 if (err) {
2762 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err);2817 pr_warn("failed to write out .BTF.ext ELF section: %s\n", errstr(err));
2763 return err;2818 return err;
2764 }2819 }
2765 }2820 }
@@ -2935,7 +2990,7 @@ static int finalize_btf_ext(struct bpf_linker *linker) · 169 unmodified lines
2935 err = libbpf_get_error(linker->btf_ext);2990 err = libbpf_get_error(linker->btf_ext);
2936 if (err) {2991 if (err) {
2937 linker->btf_ext = NULL;2992 linker->btf_ext = NULL;
2938 pr_warn("failed to parse final .BTF.ext data: %d\n", err);2993 pr_warn("failed to parse final .BTF.ext data: %s\n", errstr(err));
2939 goto out;2994 goto out;
2940 }2995 }
2941 2996
tools/lib/bpf/relo_core.c+1−1
@@ -1339,7 +1339,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, · 1338 unmodified lines
1339 cands->cands[i].id, cand_spec);1339 cands->cands[i].id, cand_spec);
1340 if (err < 0) {1340 if (err < 0) {
1341 bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);1341 bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec);
1342 pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n ",1342 pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n",
1343 prog_name, relo_idx, i, spec_buf, err);1343 prog_name, relo_idx, i, spec_buf, err);
1344 return err;1344 return err;
1345 }1345 }
tools/lib/bpf/ringbuf.c+18−16
@@ -21,6 +21,7 @@ · 20 unmodified lines
21#include "libbpf.h"21#include "libbpf.h"
22#include "libbpf_internal.h"22#include "libbpf_internal.h"
23#include "bpf.h"23#include "bpf.h"
24#include "str_error.h"
24 25
25struct ring {26struct ring {
26 ring_buffer_sample_fn sample_cb;27 ring_buffer_sample_fn sample_cb;
@@ -88,8 +89,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 61 unmodified lines
88 err = bpf_map_get_info_by_fd(map_fd, &info, &len);89 err = bpf_map_get_info_by_fd(map_fd, &info, &len);
89 if (err) {90 if (err) {
90 err = -errno;91 err = -errno;
91 pr_warn("ringbuf: failed to get map info for fd=%d: %d\n",92 pr_warn("ringbuf: failed to get map info for fd=%d: %s\n",
92 map_fd, err);93 map_fd, errstr(err));
93 return libbpf_err(err);94 return libbpf_err(err);
94 }95 }
95 96
@@ -123,8 +124,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 27 unmodified lines
123 tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0);124 tmp = mmap(NULL, rb->page_size, PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, 0);
124 if (tmp == MAP_FAILED) {125 if (tmp == MAP_FAILED) {
125 err = -errno;126 err = -errno;
126 pr_warn("ringbuf: failed to mmap consumer page for map fd=%d: %d\n",127 pr_warn("ringbuf: failed to mmap consumer page for map fd=%d: %s\n",
127 map_fd, err);128 map_fd, errstr(err));
128 goto err_out;129 goto err_out;
129 }130 }
130 r->consumer_pos = tmp;131 r->consumer_pos = tmp;
@@ -142,8 +143,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 11 unmodified lines
142 tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ, MAP_SHARED, map_fd, rb->page_size);143 tmp = mmap(NULL, (size_t)mmap_sz, PROT_READ, MAP_SHARED, map_fd, rb->page_size);
143 if (tmp == MAP_FAILED) {144 if (tmp == MAP_FAILED) {
144 err = -errno;145 err = -errno;
145 pr_warn("ringbuf: failed to mmap data pages for map fd=%d: %d\n",146 pr_warn("ringbuf: failed to mmap data pages for map fd=%d: %s\n",
146 map_fd, err);147 map_fd, errstr(err));
147 goto err_out;148 goto err_out;
148 }149 }
149 r->producer_pos = tmp;150 r->producer_pos = tmp;
@@ -156,8 +157,8 @@ int ring_buffer__add(struct ring_buffer *rb, int map_fd, · 6 unmodified lines
156 e->data.fd = rb->ring_cnt;157 e->data.fd = rb->ring_cnt;
157 if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) {158 if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, e) < 0) {
158 err = -errno;159 err = -errno;
159 pr_warn("ringbuf: failed to epoll add map fd=%d: %d\n",160 pr_warn("ringbuf: failed to epoll add map fd=%d: %s\n",
160 map_fd, err);161 map_fd, errstr(err));
161 goto err_out;162 goto err_out;
162 }163 }
163 164
@@ -205,7 +206,7 @@ ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, · 41 unmodified lines
205 rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);206 rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
206 if (rb->epoll_fd < 0) {207 if (rb->epoll_fd < 0) {
207 err = -errno;208 err = -errno;
208 pr_warn("ringbuf: failed to create epoll instance: %d\n", err);209 pr_warn("ringbuf: failed to create epoll instance: %s\n", errstr(err));
209 goto err_out;210 goto err_out;
210 }211 }
211 212
@@ -458,7 +459,8 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 246 unmodified lines
458 err = bpf_map_get_info_by_fd(map_fd, &info, &len);459 err = bpf_map_get_info_by_fd(map_fd, &info, &len);
459 if (err) {460 if (err) {
460 err = -errno;461 err = -errno;
461 pr_warn("user ringbuf: failed to get map info for fd=%d: %d\n", map_fd, err);462 pr_warn("user ringbuf: failed to get map info for fd=%d: %s\n",
463 map_fd, errstr(err));
462 return err;464 return err;
463 }465 }
464 466
@@ -474,8 +476,8 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 9 unmodified lines
474 tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0);476 tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0);
475 if (tmp == MAP_FAILED) {477 if (tmp == MAP_FAILED) {
476 err = -errno;478 err = -errno;
477 pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %d\n",479 pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %s\n",
478 map_fd, err);480 map_fd, errstr(err));
479 return err;481 return err;
480 }482 }
481 rb->consumer_pos = tmp;483 rb->consumer_pos = tmp;
@@ -494,8 +496,8 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 12 unmodified lines
494 map_fd, rb->page_size);496 map_fd, rb->page_size);
495 if (tmp == MAP_FAILED) {497 if (tmp == MAP_FAILED) {
496 err = -errno;498 err = -errno;
497 pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %d\n",499 pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %s\n",
498 map_fd, err);500 map_fd, errstr(err));
499 return err;501 return err;
500 }502 }
501 503
@@ -506,7 +508,7 @@ static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd) · 4 unmodified lines
506 rb_epoll->events = EPOLLOUT;508 rb_epoll->events = EPOLLOUT;
507 if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) {509 if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) {
508 err = -errno;510 err = -errno;
509 pr_warn("user ringbuf: failed to epoll add map fd=%d: %d\n", map_fd, err);511 pr_warn("user ringbuf: failed to epoll add map fd=%d: %s\n", map_fd, errstr(err));
510 return err;512 return err;
511 }513 }
512 514
@@ -531,7 +533,7 @@ user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts) · 18 unmodified lines
531 rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);533 rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
532 if (rb->epoll_fd < 0) {534 if (rb->epoll_fd < 0) {
533 err = -errno;535 err = -errno;
534 pr_warn("user ringbuf: failed to create epoll instance: %d\n", err);536 pr_warn("user ringbuf: failed to create epoll instance: %s\n", errstr(err));
535 goto err_out;537 goto err_out;
536 }538 }
537 539
tools/lib/bpf/skel_internal.h+2−1
@@ -351,10 +351,11 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) · 350 unmodified lines
351 attr.test.ctx_size_in = opts->ctx->sz;351 attr.test.ctx_size_in = opts->ctx->sz;
352 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz);352 err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz);
353 if (err < 0 || (int)attr.test.retval < 0) {353 if (err < 0 || (int)attr.test.retval < 0) {
354 opts->errstr = "failed to execute loader prog";
355 if (err < 0) {354 if (err < 0) {
355 opts->errstr = "failed to execute loader prog";
356 set_err;356 set_err;
357 } else {357 } else {
358 opts->errstr = "error returned by loader prog";
358 err = (int)attr.test.retval;359 err = (int)attr.test.retval;
359#ifndef __KERNEL__360#ifndef __KERNEL__
360 errno = -err;361 errno = -err;
tools/lib/bpf/str_error.c+71−0
@@ -5,6 +5,10 @@ · 4 unmodified lines
5#include <errno.h>5#include <errno.h>
6#include "str_error.h"6#include "str_error.h"
7 7
8#ifndef ENOTSUPP
9#define ENOTSUPP 524
10#endif
11
8/* make sure libbpf doesn't use kernel-only integer typedefs */12/* make sure libbpf doesn't use kernel-only integer typedefs */
9#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s6413#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64
10 14
@@ -31,3 +35,70 @@ char *libbpf_strerror_r(int err, char *dst, int len) · 20 unmodified lines
31 }35 }
32 return dst;36 return dst;
33}37}
38
39const char *errstr(int err)
40{
41 static __thread char buf[12];
42
43 if (err > 0)
44 err = -err;
45
46 switch (err) {
47 case -E2BIG: return "-E2BIG";
48 case -EACCES: return "-EACCES";
49 case -EADDRINUSE: return "-EADDRINUSE";
50 case -EADDRNOTAVAIL: return "-EADDRNOTAVAIL";
51 case -EAGAIN: return "-EAGAIN";
52 case -EALREADY: return "-EALREADY";
53 case -EBADF: return "-EBADF";
54 case -EBADFD: return "-EBADFD";
55 case -EBUSY: return "-EBUSY";
56 case -ECANCELED: return "-ECANCELED";
57 case -ECHILD: return "-ECHILD";
58 case -EDEADLK: return "-EDEADLK";
59 case -EDOM: return "-EDOM";
60 case -EEXIST: return "-EEXIST";
61 case -EFAULT: return "-EFAULT";
62 case -EFBIG: return "-EFBIG";
63 case -EILSEQ: return "-EILSEQ";
64 case -EINPROGRESS: return "-EINPROGRESS";
65 case -EINTR: return "-EINTR";
66 case -EINVAL: return "-EINVAL";
67 case -EIO: return "-EIO";
68 case -EISDIR: return "-EISDIR";
69 case -ELOOP: return "-ELOOP";
70 case -EMFILE: return "-EMFILE";
71 case -EMLINK: return "-EMLINK";
72 case -EMSGSIZE: return "-EMSGSIZE";
73 case -ENAMETOOLONG: return "-ENAMETOOLONG";
74 case -ENFILE: return "-ENFILE";
75 case -ENODATA: return "-ENODATA";
76 case -ENODEV: return "-ENODEV";
77 case -ENOENT: return "-ENOENT";
78 case -ENOEXEC: return "-ENOEXEC";
79 case -ENOLINK: return "-ENOLINK";
80 case -ENOMEM: return "-ENOMEM";
81 case -ENOSPC: return "-ENOSPC";
82 case -ENOTBLK: return "-ENOTBLK";
83 case -ENOTDIR: return "-ENOTDIR";
84 case -ENOTSUPP: return "-ENOTSUPP";
85 case -ENOTTY: return "-ENOTTY";
86 case -ENXIO: return "-ENXIO";
87 case -EOPNOTSUPP: return "-EOPNOTSUPP";
88 case -EOVERFLOW: return "-EOVERFLOW";
89 case -EPERM: return "-EPERM";
90 case -EPIPE: return "-EPIPE";
91 case -EPROTO: return "-EPROTO";
92 case -EPROTONOSUPPORT: return "-EPROTONOSUPPORT";
93 case -ERANGE: return "-ERANGE";
94 case -EROFS: return "-EROFS";
95 case -ESPIPE: return "-ESPIPE";
96 case -ESRCH: return "-ESRCH";
97 case -ETXTBSY: return "-ETXTBSY";
98 case -EUCLEAN: return "-EUCLEAN";
99 case -EXDEV: return "-EXDEV";
100 default:
101 snprintf(buf, sizeof(buf), "%d", err);
102 return buf;
103 }
104}
tools/lib/bpf/str_error.h+7−0
@@ -6,4 +6,11 @@ · 5 unmodified lines
6 6
7char *libbpf_strerror_r(int err, char *dst, int len);7char *libbpf_strerror_r(int err, char *dst, int len);
8 8
9/**
10 * @brief **errstr()** returns string corresponding to numeric errno
11 * @param err negative numeric errno
12 * @return pointer to string representation of the errno, that is invalidated
13 * upon the next call.
14 */
15const char *errstr(int err);
9#endif /* __LIBBPF_STR_ERROR_H */16#endif /* __LIBBPF_STR_ERROR_H */
tools/lib/bpf/usdt.c+17−15
@@ -20,6 +20,7 @@ · 19 unmodified lines
20#include "libbpf_common.h"20#include "libbpf_common.h"
21#include "libbpf_internal.h"21#include "libbpf_internal.h"
22#include "hashmap.h"22#include "hashmap.h"
23#include "str_error.h"
23 24
24/* libbpf's USDT support consists of BPF-side state/code and user-space25/* libbpf's USDT support consists of BPF-side state/code and user-space
25 * state/code working together in concert. BPF-side parts are defined in26 * state/code working together in concert. BPF-side parts are defined in
@@ -465,8 +466,8 @@ static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, · 439 unmodified lines
465 goto proceed;466 goto proceed;
466 467
467 if (!realpath(lib_path, path)) {468 if (!realpath(lib_path, path)) {
468 pr_warn("usdt: failed to get absolute path of '%s' (err %d), using path as is...\n",469 pr_warn("usdt: failed to get absolute path of '%s' (err %s), using path as is...\n",
469 lib_path, -errno);470 lib_path, errstr(-errno));
470 libbpf_strlcpy(path, lib_path, sizeof(path));471 libbpf_strlcpy(path, lib_path, sizeof(path));
471 }472 }
472 473
@@ -475,8 +476,8 @@ proceed: · 2 unmodified lines
475 f = fopen(line, "re");476 f = fopen(line, "re");
476 if (!f) {477 if (!f) {
477 err = -errno;478 err = -errno;
478 pr_warn("usdt: failed to open '%s' to get base addr of '%s': %d\n",479 pr_warn("usdt: failed to open '%s' to get base addr of '%s': %s\n",
479 line, lib_path, err);480 line, lib_path, errstr(err));
480 return err;481 return err;
481 }482 }
482 483
@@ -606,7 +607,8 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * · 123 unmodified lines
606 607
607 err = parse_elf_segs(elf, path, &segs, &seg_cnt);608 err = parse_elf_segs(elf, path, &segs, &seg_cnt);
608 if (err) {609 if (err) {
609 pr_warn("usdt: failed to process ELF program segments for '%s': %d\n", path, err);610 pr_warn("usdt: failed to process ELF program segments for '%s': %s\n",
611 path, errstr(err));
610 goto err_out;612 goto err_out;
611 }613 }
612 614
@@ -708,8 +710,8 @@ static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char * · 95 unmodified lines
708 if (vma_seg_cnt == 0) {710 if (vma_seg_cnt == 0) {
709 err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt);711 err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt);
710 if (err) {712 if (err) {
711 pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %d\n",713 pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %s\n",
712 pid, path, err);714 pid, path, errstr(err));
713 goto err_out;715 goto err_out;
714 }716 }
715 }717 }
@@ -1047,8 +1049,8 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 331 unmodified lines
1047 1049
1048 if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) {1050 if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) {
1049 err = -errno;1051 err = -errno;
1050 pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %d\n",1052 pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %s\n",
1051 spec_id, usdt_provider, usdt_name, path, err);1053 spec_id, usdt_provider, usdt_name, path, errstr(err));
1052 goto err_out;1054 goto err_out;
1053 }1055 }
1054 if (!man->has_bpf_cookie &&1056 if (!man->has_bpf_cookie &&
@@ -1058,9 +1060,9 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 3 unmodified lines
1058 pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n",1060 pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n",
1059 spec_id, usdt_provider, usdt_name, path);1061 spec_id, usdt_provider, usdt_name, path);
1060 } else {1062 } else {
1061 pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %d\n",1063 pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n",
1062 target->abs_ip, spec_id, usdt_provider, usdt_name,1064 target->abs_ip, spec_id, usdt_provider, usdt_name,
1063 path, err);1065 path, errstr(err));
1064 }1066 }
1065 goto err_out;1067 goto err_out;
1066 }1068 }
@@ -1076,8 +1078,8 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 9 unmodified lines
1076 target->rel_ip, &opts);1078 target->rel_ip, &opts);
1077 err = libbpf_get_error(uprobe_link);1079 err = libbpf_get_error(uprobe_link);
1078 if (err) {1080 if (err) {
1079 pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n",1081 pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %s\n",
1080 i, usdt_provider, usdt_name, path, err);1082 i, usdt_provider, usdt_name, path, errstr(err));
1081 goto err_out;1083 goto err_out;
1082 }1084 }
1083 1085
@@ -1099,8 +1101,8 @@ struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct · 15 unmodified lines
1099 NULL, &opts_multi);1101 NULL, &opts_multi);
1100 if (!link->multi_link) {1102 if (!link->multi_link) {
1101 err = -errno;1103 err = -errno;
1102 pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %d\n",1104 pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %s\n",
1103 usdt_provider, usdt_name, path, err);1105 usdt_provider, usdt_name, path, errstr(err));
1104 goto err_out;1106 goto err_out;
1105 }1107 }
1106 1108
tools/lib/bpf/zip.c+1−1
@@ -223,7 +223,7 @@ struct zip_archive *zip_archive_open(const char *path) · 222 unmodified lines
223 if (!archive) {223 if (!archive) {
224 munmap(data, size);224 munmap(data, size);
225 return ERR_PTR(-ENOMEM);225 return ERR_PTR(-ENOMEM);
226 };226 }
227 227
228 archive->data = data;228 archive->data = data;
229 archive->size = size;229 archive->size = size;
tools/lib/subcmd/parse-options.c+1−1
@@ -807,7 +807,7 @@ static int option__cmp(const void *va, const void *vb) · 806 unmodified lines
807static struct option *options__order(const struct option *opts)807static struct option *options__order(const struct option *opts)
808{808{
809 int nr_opts = 0, nr_group = 0, nr_parent = 0, len;809 int nr_opts = 0, nr_group = 0, nr_parent = 0, len;
810 const struct option *o, *p = opts;810 const struct option *o = NULL, *p = opts;
811 struct option *opt, *ordered = NULL, *group;811 struct option *opt, *ordered = NULL, *group;
812 812
813 /* flatten the options that have parents */813 /* flatten the options that have parents */
tools/testing/selftests/bpf/.gitignore+0−1
@@ -16,7 +16,6 @@ fixdep · 15 unmodified lines
16/test_progs-cpuv416/test_progs-cpuv4
17test_verifier_log17test_verifier_log
18feature18feature
19test_sock
20urandom_read19urandom_read
21test_sockmap20test_sockmap
22test_lirc_mode2_user21test_lirc_mode2_user
tools/testing/selftests/bpf/Makefile+38−22
@@ -10,6 +10,7 @@ TOOLSDIR := $(abspath ../../..) · 9 unmodified lines
10LIBDIR := $(TOOLSDIR)/lib10LIBDIR := $(TOOLSDIR)/lib
11BPFDIR := $(LIBDIR)/bpf11BPFDIR := $(LIBDIR)/bpf
12TOOLSINCDIR := $(TOOLSDIR)/include12TOOLSINCDIR := $(TOOLSDIR)/include
13TOOLSARCHINCDIR := $(TOOLSDIR)/arch/$(SRCARCH)/include
13BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool14BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
14APIDIR := $(TOOLSINCDIR)/uapi15APIDIR := $(TOOLSINCDIR)/uapi
15ifneq ($(O),)16ifneq ($(O),)
@@ -44,7 +45,7 @@ CFLAGS += -g $(OPT_FLAGS) -rdynamic \ · 28 unmodified lines
44 -Wall -Werror -fno-omit-frame-pointer \45 -Wall -Werror -fno-omit-frame-pointer \
45 $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \46 $(GENFLAGS) $(SAN_CFLAGS) $(LIBELF_CFLAGS) \
46 -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \47 -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
47 -I$(TOOLSINCDIR) -I$(APIDIR) -I$(OUTPUT)48 -I$(TOOLSINCDIR) -I$(TOOLSARCHINCDIR) -I$(APIDIR) -I$(OUTPUT)
48LDFLAGS += $(SAN_LDFLAGS)49LDFLAGS += $(SAN_LDFLAGS)
49LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread50LDLIBS += $(LIBELF_LIBS) -lz -lrt -lpthread
50 51
@@ -83,7 +84,7 @@ endif · 32 unmodified lines
83 84
84# Order correspond to 'make run_tests' order85# Order correspond to 'make run_tests' order
85TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \86TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
86 test_sock test_sockmap \87 test_sockmap \
87 test_tcpnotify_user test_sysctl \88 test_tcpnotify_user test_sysctl \
88 test_progs-no_alu3289 test_progs-no_alu32
89TEST_INST_SUBDIRS := no_alu3290TEST_INST_SUBDIRS := no_alu32
@@ -132,7 +133,6 @@ TEST_PROGS := test_kmod.sh \ · 42 unmodified lines
132 test_tunnel.sh \133 test_tunnel.sh \
133 test_lwt_seg6local.sh \134 test_lwt_seg6local.sh \
134 test_lirc_mode2.sh \135 test_lirc_mode2.sh \
135 test_skb_cgroup_id.sh \
136 test_flow_dissector.sh \136 test_flow_dissector.sh \
137 test_xdp_vlan_mode_generic.sh \137 test_xdp_vlan_mode_generic.sh \
138 test_xdp_vlan_mode_native.sh \138 test_xdp_vlan_mode_native.sh \
@@ -274,6 +274,7 @@ $(OUTPUT)/liburandom_read.so: urandom_read_lib1.c urandom_read_lib2.c liburandom · 135 unmodified lines
274 $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \274 $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \
275 $(filter-out -static,$(CFLAGS) $(LDFLAGS)) \275 $(filter-out -static,$(CFLAGS) $(LDFLAGS)) \
276 $(filter %.c,$^) $(filter-out -static,$(LDLIBS)) \276 $(filter %.c,$^) $(filter-out -static,$(LDLIBS)) \
277 -Wno-unused-command-line-argument \
277 -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \278 -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \
278 -Wl,--version-script=liburandom_read.map \279 -Wl,--version-script=liburandom_read.map \
279 -fPIC -shared -o $@280 -fPIC -shared -o $@
@@ -282,6 +283,7 @@ $(OUTPUT)/urandom_read: urandom_read.c urandom_read_aux.c $(OUTPUT)/liburandom_r · 2 unmodified lines
282 $(call msg,BINARY,,$@)283 $(call msg,BINARY,,$@)
283 $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \284 $(Q)$(CLANG) $(CLANG_TARGET_ARCH) \
284 $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $(filter %.c,$^) \285 $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $(filter %.c,$^) \
286 -Wno-unused-command-line-argument \
285 -lurandom_read $(filter-out -static,$(LDLIBS)) -L$(OUTPUT) \287 -lurandom_read $(filter-out -static,$(LDLIBS)) -L$(OUTPUT) \
286 -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \288 -fuse-ld=$(LLD) -Wl,-znoseparate-code -Wl,--build-id=sha1 \
287 -Wl,-rpath=. -o $@289 -Wl,-rpath=. -o $@
@@ -295,25 +297,33 @@ $(OUTPUT)/sign-file: ../../../../scripts/sign-file.c · 7 unmodified lines
295$(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch])297$(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch])
296 $(call msg,MOD,,$@)298 $(call msg,MOD,,$@)
297 $(Q)$(RM) bpf_testmod/bpf_testmod.ko # force re-compilation299 $(Q)$(RM) bpf_testmod/bpf_testmod.ko # force re-compilation
298 $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_testmod300 $(Q)$(MAKE) $(submake_extras) -C bpf_testmod \
301 RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
302 EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
299 $(Q)cp bpf_testmod/bpf_testmod.ko $@303 $(Q)cp bpf_testmod/bpf_testmod.ko $@
300 304
301$(OUTPUT)/bpf_test_no_cfi.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_no_cfi/Makefile bpf_test_no_cfi/*.[ch])305$(OUTPUT)/bpf_test_no_cfi.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_no_cfi/Makefile bpf_test_no_cfi/*.[ch])
302 $(call msg,MOD,,$@)306 $(call msg,MOD,,$@)
303 $(Q)$(RM) bpf_test_no_cfi/bpf_test_no_cfi.ko # force re-compilation307 $(Q)$(RM) bpf_test_no_cfi/bpf_test_no_cfi.ko # force re-compilation
304 $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_test_no_cfi308 $(Q)$(MAKE) $(submake_extras) -C bpf_test_no_cfi \
309 RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
310 EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
305 $(Q)cp bpf_test_no_cfi/bpf_test_no_cfi.ko $@311 $(Q)cp bpf_test_no_cfi/bpf_test_no_cfi.ko $@
306 312
307$(OUTPUT)/bpf_test_modorder_x.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_x/Makefile bpf_test_modorder_x/*.[ch])313$(OUTPUT)/bpf_test_modorder_x.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_x/Makefile bpf_test_modorder_x/*.[ch])
308 $(call msg,MOD,,$@)314 $(call msg,MOD,,$@)
309 $(Q)$(RM) bpf_test_modorder_x/bpf_test_modorder_x.ko # force re-compilation315 $(Q)$(RM) bpf_test_modorder_x/bpf_test_modorder_x.ko # force re-compilation
310 $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_test_modorder_x316 $(Q)$(MAKE) $(submake_extras) -C bpf_test_modorder_x \
317 RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
318 EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
311 $(Q)cp bpf_test_modorder_x/bpf_test_modorder_x.ko $@319 $(Q)cp bpf_test_modorder_x/bpf_test_modorder_x.ko $@
312 320
313$(OUTPUT)/bpf_test_modorder_y.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_y/Makefile bpf_test_modorder_y/*.[ch])321$(OUTPUT)/bpf_test_modorder_y.ko: $(VMLINUX_BTF) $(RESOLVE_BTFIDS) $(wildcard bpf_test_modorder_y/Makefile bpf_test_modorder_y/*.[ch])
314 $(call msg,MOD,,$@)322 $(call msg,MOD,,$@)
315 $(Q)$(RM) bpf_test_modorder_y/bpf_test_modorder_y.ko # force re-compilation323 $(Q)$(RM) bpf_test_modorder_y/bpf_test_modorder_y.ko # force re-compilation
316 $(Q)$(MAKE) $(submake_extras) RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) -C bpf_test_modorder_y324 $(Q)$(MAKE) $(submake_extras) -C bpf_test_modorder_y \
325 RESOLVE_BTFIDS=$(RESOLVE_BTFIDS) \
326 EXTRA_CFLAGS='' EXTRA_LDFLAGS=''
317 $(Q)cp bpf_test_modorder_y/bpf_test_modorder_y.ko $@327 $(Q)cp bpf_test_modorder_y/bpf_test_modorder_y.ko $@
318 328
319 329
@@ -333,8 +343,8 @@ $(OUTPUT)/runqslower: $(BPFOBJ) | $(DEFAULT_BPFTOOL) $(RUNQSLOWER_OUTPUT) · 13 unmodified lines
333 BPFTOOL_OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \343 BPFTOOL_OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
334 BPFOBJ_OUTPUT=$(BUILD_DIR)/libbpf/ \344 BPFOBJ_OUTPUT=$(BUILD_DIR)/libbpf/ \
335 BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR) \345 BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR) \
336 EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS)' \346 EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS) $(EXTRA_CFLAGS)' \
337 EXTRA_LDFLAGS='$(SAN_LDFLAGS)' && \347 EXTRA_LDFLAGS='$(SAN_LDFLAGS) $(EXTRA_LDFLAGS)' && \
338 cp $(RUNQSLOWER_OUTPUT)runqslower $@348 cp $(RUNQSLOWER_OUTPUT)runqslower $@
339 349
340TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL)350TEST_GEN_PROGS_EXTENDED += $(TRUNNER_BPFTOOL)
@@ -349,7 +359,6 @@ JSON_WRITER := $(OUTPUT)/json_writer.o · 8 unmodified lines
349CAP_HELPERS := $(OUTPUT)/cap_helpers.o359CAP_HELPERS := $(OUTPUT)/cap_helpers.o
350NETWORK_HELPERS := $(OUTPUT)/network_helpers.o360NETWORK_HELPERS := $(OUTPUT)/network_helpers.o
351 361
352$(OUTPUT)/test_sock: $(CGROUP_HELPERS) $(TESTING_HELPERS)
353$(OUTPUT)/test_sockmap: $(CGROUP_HELPERS) $(TESTING_HELPERS)362$(OUTPUT)/test_sockmap: $(CGROUP_HELPERS) $(TESTING_HELPERS)
354$(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS)363$(OUTPUT)/test_tcpnotify_user: $(CGROUP_HELPERS) $(TESTING_HELPERS) $(TRACE_HELPERS)
355$(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS)364$(OUTPUT)/test_sock_fields: $(CGROUP_HELPERS) $(TESTING_HELPERS)
@@ -368,7 +377,8 @@ $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ · 12 unmodified lines
368 $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool377 $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool
369 $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \378 $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
370 ARCH= CROSS_COMPILE= CC="$(HOSTCC)" LD="$(HOSTLD)" \379 ARCH= CROSS_COMPILE= CC="$(HOSTCC)" LD="$(HOSTLD)" \
371 EXTRA_CFLAGS='-g $(OPT_FLAGS)' \380 EXTRA_CFLAGS='-g $(OPT_FLAGS) $(EXTRA_CFLAGS)' \
381 EXTRA_LDFLAGS='$(EXTRA_LDFLAGS)' \
372 OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \382 OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
373 LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \383 LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
374 LIBBPF_DESTDIR=$(HOST_SCRATCH_DIR)/ \384 LIBBPF_DESTDIR=$(HOST_SCRATCH_DIR)/ \
@@ -379,7 +389,8 @@ $(CROSS_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ · 4 unmodified lines
379 $(BPFOBJ) | $(BUILD_DIR)/bpftool389 $(BPFOBJ) | $(BUILD_DIR)/bpftool
380 $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \390 $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
381 ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) \391 ARCH=$(ARCH) CROSS_COMPILE=$(CROSS_COMPILE) \
382 EXTRA_CFLAGS='-g $(OPT_FLAGS)' \392 EXTRA_CFLAGS='-g $(OPT_FLAGS) $(EXTRA_CFLAGS)' \
393 EXTRA_LDFLAGS='$(EXTRA_LDFLAGS)' \
383 OUTPUT=$(BUILD_DIR)/bpftool/ \394 OUTPUT=$(BUILD_DIR)/bpftool/ \
384 LIBBPF_OUTPUT=$(BUILD_DIR)/libbpf/ \395 LIBBPF_OUTPUT=$(BUILD_DIR)/libbpf/ \
385 LIBBPF_DESTDIR=$(SCRATCH_DIR)/ \396 LIBBPF_DESTDIR=$(SCRATCH_DIR)/ \
@@ -402,8 +413,8 @@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ · 16 unmodified lines
402 $(APIDIR)/linux/bpf.h \413 $(APIDIR)/linux/bpf.h \
403 | $(BUILD_DIR)/libbpf414 | $(BUILD_DIR)/libbpf
404 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/ \415 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/ \
405 EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS)' \416 EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS) $(EXTRA_CFLAGS)' \
406 EXTRA_LDFLAGS='$(SAN_LDFLAGS)' \417 EXTRA_LDFLAGS='$(SAN_LDFLAGS) $(EXTRA_LDFLAGS)' \
407 DESTDIR=$(SCRATCH_DIR) prefix= all install_headers418 DESTDIR=$(SCRATCH_DIR) prefix= all install_headers
408 419
409ifneq ($(BPFOBJ),$(HOST_BPFOBJ))420ifneq ($(BPFOBJ),$(HOST_BPFOBJ))
@@ -411,7 +422,9 @@ $(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ · 1 unmodified line
411 $(APIDIR)/linux/bpf.h \422 $(APIDIR)/linux/bpf.h \
412 | $(HOST_BUILD_DIR)/libbpf423 | $(HOST_BUILD_DIR)/libbpf
413 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \424 $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \
414 EXTRA_CFLAGS='-g $(OPT_FLAGS)' ARCH= CROSS_COMPILE= \425 ARCH= CROSS_COMPILE= \
426 EXTRA_CFLAGS='-g $(OPT_FLAGS) $(EXTRA_CFLAGS)' \
427 EXTRA_LDFLAGS='$(EXTRA_LDFLAGS)' \
415 OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \428 OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
416 CC="$(HOSTCC)" LD="$(HOSTLD)" \429 CC="$(HOSTCC)" LD="$(HOSTLD)" \
417 DESTDIR=$(HOST_SCRATCH_DIR)/ prefix= all install_headers430 DESTDIR=$(HOST_SCRATCH_DIR)/ prefix= all install_headers
@@ -460,6 +473,7 @@ endef · 42 unmodified lines
460IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \473IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \
461 grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')474 grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
462MENDIAN=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian)475MENDIAN=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian)
476BPF_TARGET_ENDIAN=$(if $(IS_LITTLE_ENDIAN),--target=bpfel,--target=bpfeb)
463 477
464ifneq ($(CROSS_COMPILE),)478ifneq ($(CROSS_COMPILE),)
465CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))479CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
@@ -487,17 +501,17 @@ $(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h · 21 unmodified lines
487# $4 - binary name501# $4 - binary name
488define CLANG_BPF_BUILD_RULE502define CLANG_BPF_BUILD_RULE
489 $(call msg,CLNG-BPF,$4,$2)503 $(call msg,CLNG-BPF,$4,$2)
490 $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v3 -o $2504 $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v3 -o $2
491endef505endef
492# Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32506# Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32
493define CLANG_NOALU32_BPF_BUILD_RULE507define CLANG_NOALU32_BPF_BUILD_RULE
494 $(call msg,CLNG-BPF,$4,$2)508 $(call msg,CLNG-BPF,$4,$2)
495 $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v2 -o $2509 $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v2 -o $2
496endef510endef
497# Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4511# Similar to CLANG_BPF_BUILD_RULE, but with cpu-v4
498define CLANG_CPUV4_BPF_BUILD_RULE512define CLANG_CPUV4_BPF_BUILD_RULE
499 $(call msg,CLNG-BPF,$4,$2)513 $(call msg,CLNG-BPF,$4,$2)
500 $(Q)$(CLANG) $3 -O2 --target=bpf -c $1 -mcpu=v4 -o $2514 $(Q)$(CLANG) $3 -O2 $(BPF_TARGET_ENDIAN) -c $1 -mcpu=v4 -o $2
501endef515endef
502# Build BPF object using GCC516# Build BPF object using GCC
503define GCC_BPF_BUILD_RULE517define GCC_BPF_BUILD_RULE
@@ -637,10 +651,11 @@ $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_OUTPUT)/%: $$$$(%-deps) $(BPFTOOL) | $(TR · 133 unmodified lines
637 651
638# When the compiler generates a %.d file, only skel basenames (not652# When the compiler generates a %.d file, only skel basenames (not
639# full paths) are specified as prerequisites for corresponding %.o653# full paths) are specified as prerequisites for corresponding %.o
640# file. This target makes %.skel.h basename dependent on full paths,654# file. vpath directives below instruct make to search for skel files
641# linking generated %.d dependency with actual %.skel.h files.655# in TRUNNER_OUTPUT, if they are not present in the working directory.
642$(notdir %.skel.h): $(TRUNNER_OUTPUT)/%.skel.h656vpath %.skel.h $(TRUNNER_OUTPUT)
643 @true657vpath %.lskel.h $(TRUNNER_OUTPUT)
658vpath %.subskel.h $(TRUNNER_OUTPUT)
644 659
645endif660endif
646 661
@@ -727,6 +742,7 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \ · 80 unmodified lines
727 unpriv_helpers.c \742 unpriv_helpers.c \
728 netlink_helpers.c \743 netlink_helpers.c \
729 jit_disasm_helpers.c \744 jit_disasm_helpers.c \
745 io_helpers.c \
730 test_loader.c \746 test_loader.c \
731 xsk.c \747 xsk.c \
732 disasm.c \748 disasm.c \
tools/testing/selftests/bpf/benchs/bench_trigger.c+2−1
@@ -4,6 +4,7 @@ · 3 unmodified lines
4#include <argp.h>4#include <argp.h>
5#include <unistd.h>5#include <unistd.h>
6#include <stdint.h>6#include <stdint.h>
7#include "bpf_util.h"
7#include "bench.h"8#include "bench.h"
8#include "trigger_bench.skel.h"9#include "trigger_bench.skel.h"
9#include "trace_helpers.h"10#include "trace_helpers.h"
@@ -72,7 +73,7 @@ static __always_inline void inc_counter(struct counter *counters) · 62 unmodified lines
72 unsigned slot;73 unsigned slot;
73 74
74 if (unlikely(tid == 0))75 if (unlikely(tid == 0))
75 tid = syscall(SYS_gettid);76 tid = sys_gettid();
76 77
77 /* multiplicative hashing, it's fast */78 /* multiplicative hashing, it's fast */
78 slot = 2654435769U * tid;79 slot = 2654435769U * tid;
tools/testing/selftests/bpf/bpf_experimental.h+6−0
@@ -582,4 +582,10 @@ extern int bpf_wq_set_callback_impl(struct bpf_wq *wq, · 581 unmodified lines
582 unsigned int flags__k, void *aux__ign) __ksym;582 unsigned int flags__k, void *aux__ign) __ksym;
583#define bpf_wq_set_callback(timer, cb, flags) \583#define bpf_wq_set_callback(timer, cb, flags) \
584 bpf_wq_set_callback_impl(timer, cb, flags, NULL)584 bpf_wq_set_callback_impl(timer, cb, flags, NULL)
585
586struct bpf_iter_kmem_cache;
587extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym;
588extern struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it) __weak __ksym;
589extern void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it) __weak __ksym;
590
585#endif591#endif
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod-events.h+8−0
@@ -40,6 +40,14 @@ DECLARE_TRACE(bpf_testmod_test_nullable_bare, · 39 unmodified lines
40 TP_ARGS(ctx__nullable)40 TP_ARGS(ctx__nullable)
41);41);
42 42
43struct sk_buff;
44
45DECLARE_TRACE(bpf_testmod_test_raw_tp_null,
46 TP_PROTO(struct sk_buff *skb),
47 TP_ARGS(skb)
48);
49
50
43#undef BPF_TESTMOD_DECLARE_TRACE51#undef BPF_TESTMOD_DECLARE_TRACE
44#ifdef DECLARE_TRACE_WRITABLE52#ifdef DECLARE_TRACE_WRITABLE
45#define BPF_TESTMOD_DECLARE_TRACE(call, proto, args, size) \53#define BPF_TESTMOD_DECLARE_TRACE(call, proto, args, size) \
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c+106−0
@@ -245,6 +245,39 @@ __bpf_kfunc void bpf_testmod_ctx_release(struct bpf_testmod_ctx *ctx) · 244 unmodified lines
245 call_rcu(&ctx->rcu, testmod_free_cb);245 call_rcu(&ctx->rcu, testmod_free_cb);
246}246}
247 247
248static struct bpf_testmod_ops3 *st_ops3;
249
250static int bpf_testmod_test_3(void)
251{
252 return 0;
253}
254
255static int bpf_testmod_test_4(void)
256{
257 return 0;
258}
259
260static struct bpf_testmod_ops3 __bpf_testmod_ops3 = {
261 .test_1 = bpf_testmod_test_3,
262 .test_2 = bpf_testmod_test_4,
263};
264
265static void bpf_testmod_test_struct_ops3(void)
266{
267 if (st_ops3)
268 st_ops3->test_1();
269}
270
271__bpf_kfunc void bpf_testmod_ops3_call_test_1(void)
272{
273 st_ops3->test_1();
274}
275
276__bpf_kfunc void bpf_testmod_ops3_call_test_2(void)
277{
278 st_ops3->test_2();
279}
280
248struct bpf_testmod_btf_type_tag_1 {281struct bpf_testmod_btf_type_tag_1 {
249 int a;282 int a;
250};283};
@@ -380,6 +413,10 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, · 129 unmodified lines
380 413
381 (void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2);414 (void)bpf_testmod_test_arg_ptr_to_struct(&struct_arg1_2);
382 415
416 (void)trace_bpf_testmod_test_raw_tp_null(NULL);
417
418 bpf_testmod_test_struct_ops3();
419
383 struct_arg3 = kmalloc((sizeof(struct bpf_testmod_struct_arg_3) +420 struct_arg3 = kmalloc((sizeof(struct bpf_testmod_struct_arg_3) +
384 sizeof(int)), GFP_KERNEL);421 sizeof(int)), GFP_KERNEL);
385 if (struct_arg3 != NULL) {422 if (struct_arg3 != NULL) {
@@ -584,6 +621,8 @@ BTF_ID_FLAGS(func, bpf_kfunc_trusted_num_test, KF_TRUSTED_ARGS) · 198 unmodified lines
584BTF_ID_FLAGS(func, bpf_kfunc_rcu_task_test, KF_RCU)621BTF_ID_FLAGS(func, bpf_kfunc_rcu_task_test, KF_RCU)
585BTF_ID_FLAGS(func, bpf_testmod_ctx_create, KF_ACQUIRE | KF_RET_NULL)622BTF_ID_FLAGS(func, bpf_testmod_ctx_create, KF_ACQUIRE | KF_RET_NULL)
586BTF_ID_FLAGS(func, bpf_testmod_ctx_release, KF_RELEASE)623BTF_ID_FLAGS(func, bpf_testmod_ctx_release, KF_RELEASE)
624BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_1)
625BTF_ID_FLAGS(func, bpf_testmod_ops3_call_test_2)
587BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids)626BTF_KFUNCS_END(bpf_testmod_common_kfunc_ids)
588 627
589BTF_ID_LIST(bpf_testmod_dtor_ids)628BTF_ID_LIST(bpf_testmod_dtor_ids)
@@ -1094,6 +1133,10 @@ static const struct bpf_verifier_ops bpf_testmod_verifier_ops = { · 504 unmodified lines
1094 .is_valid_access = bpf_testmod_ops_is_valid_access,1133 .is_valid_access = bpf_testmod_ops_is_valid_access,
1095};1134};
1096 1135
1136static const struct bpf_verifier_ops bpf_testmod_verifier_ops3 = {
1137 .is_valid_access = bpf_testmod_ops_is_valid_access,
1138};
1139
1097static int bpf_dummy_reg(void *kdata, struct bpf_link *link)1140static int bpf_dummy_reg(void *kdata, struct bpf_link *link)
1098{1141{
1099 struct bpf_testmod_ops *ops = kdata;1142 struct bpf_testmod_ops *ops = kdata;
@@ -1173,6 +1216,68 @@ struct bpf_struct_ops bpf_testmod_ops2 = { · 73 unmodified lines
1173 .owner = THIS_MODULE,1216 .owner = THIS_MODULE,
1174};1217};
1175 1218
1219static int st_ops3_reg(void *kdata, struct bpf_link *link)
1220{
1221 int err = 0;
1222
1223 mutex_lock(&st_ops_mutex);
1224 if (st_ops3) {
1225 pr_err("st_ops has already been registered\n");
1226 err = -EEXIST;
1227 goto unlock;
1228 }
1229 st_ops3 = kdata;
1230
1231unlock:
1232 mutex_unlock(&st_ops_mutex);
1233 return err;
1234}
1235
1236static void st_ops3_unreg(void *kdata, struct bpf_link *link)
1237{
1238 mutex_lock(&st_ops_mutex);
1239 st_ops3 = NULL;
1240 mutex_unlock(&st_ops_mutex);
1241}
1242
1243static void test_1_recursion_detected(struct bpf_prog *prog)
1244{
1245 struct bpf_prog_stats *stats;
1246
1247 stats = this_cpu_ptr(prog->stats);
1248 printk("bpf_testmod: oh no, recursing into test_1, recursion_misses %llu",
1249 u64_stats_read(&stats->misses));
1250}
1251
1252static int st_ops3_check_member(const struct btf_type *t,
1253 const struct btf_member *member,
1254 const struct bpf_prog *prog)
1255{
1256 u32 moff = __btf_member_bit_offset(t, member) / 8;
1257
1258 switch (moff) {
1259 case offsetof(struct bpf_testmod_ops3, test_1):
1260 prog->aux->priv_stack_requested = true;
1261 prog->aux->recursion_detected = test_1_recursion_detected;
1262 fallthrough;
1263 default:
1264 break;
1265 }
1266 return 0;
1267}
1268
1269struct bpf_struct_ops bpf_testmod_ops3 = {
1270 .verifier_ops = &bpf_testmod_verifier_ops3,
1271 .init = bpf_testmod_ops_init,
1272 .init_member = bpf_testmod_ops_init_member,
1273 .reg = st_ops3_reg,
1274 .unreg = st_ops3_unreg,
1275 .check_member = st_ops3_check_member,
1276 .cfi_stubs = &__bpf_testmod_ops3,
1277 .name = "bpf_testmod_ops3",
1278 .owner = THIS_MODULE,
1279};
1280
1176static int bpf_test_mod_st_ops__test_prologue(struct st_ops_args *args)1281static int bpf_test_mod_st_ops__test_prologue(struct st_ops_args *args)
1177{1282{
1178 return 0;1283 return 0;
@@ -1331,6 +1436,7 @@ static int bpf_testmod_init(void) · 152 unmodified lines
1331 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_testmod_kfunc_set);1436 ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_testmod_kfunc_set);
1332 ret = ret ?: register_bpf_struct_ops(&bpf_bpf_testmod_ops, bpf_testmod_ops);1437 ret = ret ?: register_bpf_struct_ops(&bpf_bpf_testmod_ops, bpf_testmod_ops);
1333 ret = ret ?: register_bpf_struct_ops(&bpf_testmod_ops2, bpf_testmod_ops2);1438 ret = ret ?: register_bpf_struct_ops(&bpf_testmod_ops2, bpf_testmod_ops2);
1439 ret = ret ?: register_bpf_struct_ops(&bpf_testmod_ops3, bpf_testmod_ops3);
1334 ret = ret ?: register_bpf_struct_ops(&testmod_st_ops, bpf_testmod_st_ops);1440 ret = ret ?: register_bpf_struct_ops(&testmod_st_ops, bpf_testmod_st_ops);
1335 ret = ret ?: register_btf_id_dtor_kfuncs(bpf_testmod_dtors,1441 ret = ret ?: register_btf_id_dtor_kfuncs(bpf_testmod_dtors,
1336 ARRAY_SIZE(bpf_testmod_dtors),1442 ARRAY_SIZE(bpf_testmod_dtors),
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.h+5−0
@@ -94,6 +94,11 @@ struct bpf_testmod_ops2 { · 93 unmodified lines
94 int (*test_1)(void);94 int (*test_1)(void);
95};95};
96 96
97struct bpf_testmod_ops3 {
98 int (*test_1)(void);
99 int (*test_2)(void);
100};
101
97struct st_ops_args {102struct st_ops_args {
98 u64 a;103 u64 a;
99};104};
tools/testing/selftests/bpf/bpf_util.h+12−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#include <stdlib.h>6#include <stdlib.h>
7#include <string.h>7#include <string.h>
8#include <errno.h>8#include <errno.h>
9#include <syscall.h>
9#include <bpf/libbpf.h> /* libbpf_num_possible_cpus */10#include <bpf/libbpf.h> /* libbpf_num_possible_cpus */
10 11
11static inline unsigned int bpf_num_possible_cpus(void)12static inline unsigned int bpf_num_possible_cpus(void)
@@ -59,4 +60,15 @@ static inline void bpf_strlcpy(char *dst, const char *src, size_t sz) · 47 unmodified lines
59 (offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))60 (offsetof(TYPE, MEMBER) + sizeof_field(TYPE, MEMBER))
60#endif61#endif
61 62
63/* Availability of gettid across glibc versions is hit-and-miss, therefore
64 * fallback to syscall in this macro and use it everywhere.
65 */
66#ifndef sys_gettid
67#define sys_gettid() syscall(SYS_gettid)
68#endif
69
70#ifndef ENOTSUPP
71#define ENOTSUPP 524
72#endif
73
62#endif /* __BPF_UTIL__ */74#endif /* __BPF_UTIL__ */
tools/testing/selftests/bpf/config.vm+5−2
@@ -1,12 +1,15 @@
1CONFIG_9P_FS=y
2CONFIG_9P_FS_POSIX_ACL=y1CONFIG_9P_FS_POSIX_ACL=y
3CONFIG_9P_FS_SECURITY=y2CONFIG_9P_FS_SECURITY=y
3CONFIG_9P_FS=y
4CONFIG_CRYPTO_DEV_VIRTIO=y4CONFIG_CRYPTO_DEV_VIRTIO=y
5CONFIG_NET_9P=y5CONFIG_FUSE_FS=y
6CONFIG_FUSE_PASSTHROUGH=y
6CONFIG_NET_9P_VIRTIO=y7CONFIG_NET_9P_VIRTIO=y
8CONFIG_NET_9P=y
7CONFIG_VIRTIO_BALLOON=y9CONFIG_VIRTIO_BALLOON=y
8CONFIG_VIRTIO_BLK=y10CONFIG_VIRTIO_BLK=y
9CONFIG_VIRTIO_CONSOLE=y11CONFIG_VIRTIO_CONSOLE=y
12CONFIG_VIRTIO_FS=y
10CONFIG_VIRTIO_NET=y13CONFIG_VIRTIO_NET=y
11CONFIG_VIRTIO_PCI=y14CONFIG_VIRTIO_PCI=y
12CONFIG_VIRTIO_VSOCKETS_COMMON=y15CONFIG_VIRTIO_VSOCKETS_COMMON=y
tools/testing/selftests/bpf/io_helpers.c+21−0
@@ -0,0 +1,21 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <sys/select.h>
3#include <unistd.h>
4#include <errno.h>
5
6int read_with_timeout(int fd, char *buf, size_t count, long usec)
7{
8 const long M = 1000 * 1000;
9 struct timeval tv = { usec / M, usec % M };
10 fd_set fds;
11 int err;
12
13 FD_ZERO(&fds);
14 FD_SET(fd, &fds);
15 err = select(fd + 1, &fds, NULL, NULL, &tv);
16 if (err < 0)
17 return err;
18 if (FD_ISSET(fd, &fds))
19 return read(fd, buf, count);
20 return -EAGAIN;
21}
tools/testing/selftests/bpf/io_helpers.h+7−0
@@ -0,0 +1,7 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <unistd.h>
3
4/* As a regular read(2), but allows to specify a timeout in micro-seconds.
5 * Returns -EAGAIN on timeout.
6 */
7int read_with_timeout(int fd, char *buf, size_t count, long usec);
tools/testing/selftests/bpf/map_tests/task_storage_map.c+2−1
@@ -12,6 +12,7 @@ · 11 unmodified lines
12#include <bpf/bpf.h>12#include <bpf/bpf.h>
13#include <bpf/libbpf.h>13#include <bpf/libbpf.h>
14 14
15#include "bpf_util.h"
15#include "test_maps.h"16#include "test_maps.h"
16#include "task_local_storage_helpers.h"17#include "task_local_storage_helpers.h"
17#include "read_bpf_task_storage_busy.skel.h"18#include "read_bpf_task_storage_busy.skel.h"
@@ -115,7 +116,7 @@ void test_task_storage_map_stress_lookup(void) · 97 unmodified lines
115 CHECK(err, "attach", "error %d\n", err);116 CHECK(err, "attach", "error %d\n", err);
116 117
117 /* Trigger program */118 /* Trigger program */
118 syscall(SYS_gettid);119 sys_gettid();
119 skel->bss->pid = 0;120 skel->bss->pid = 0;
120 121
121 CHECK(skel->bss->busy != 0, "bad bpf_task_storage_busy", "got %d\n", skel->bss->busy);122 CHECK(skel->bss->busy != 0, "bad bpf_task_storage_busy", "got %d\n", skel->bss->busy);
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c+1−1
@@ -690,7 +690,7 @@ void test_bpf_cookie(void) · 689 unmodified lines
690 if (!ASSERT_OK_PTR(skel, "skel_open"))690 if (!ASSERT_OK_PTR(skel, "skel_open"))
691 return;691 return;
692 692
693 skel->bss->my_tid = syscall(SYS_gettid);693 skel->bss->my_tid = sys_gettid();
694 694
695 if (test__start_subtest("kprobe"))695 if (test__start_subtest("kprobe"))
696 kprobe_subtest(skel);696 kprobe_subtest(skel);
tools/testing/selftests/bpf/prog_tests/bpf_iter.c+7−7
@@ -226,7 +226,7 @@ static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts, · 225 unmodified lines
226 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),226 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
227 "pthread_create");227 "pthread_create");
228 228
229 skel->bss->tid = gettid();229 skel->bss->tid = sys_gettid();
230 230
231 do_dummy_read_opts(skel->progs.dump_task, opts);231 do_dummy_read_opts(skel->progs.dump_task, opts);
232 232
@@ -255,20 +255,20 @@ static void *run_test_task_tid(void *arg) · 22 unmodified lines
255 union bpf_iter_link_info linfo;255 union bpf_iter_link_info linfo;
256 int num_unknown_tid, num_known_tid;256 int num_unknown_tid, num_known_tid;
257 257
258 ASSERT_NEQ(getpid(), gettid(), "check_new_thread_id");258 ASSERT_NEQ(getpid(), sys_gettid(), "check_new_thread_id");
259 259
260 memset(&linfo, 0, sizeof(linfo));260 memset(&linfo, 0, sizeof(linfo));
261 linfo.task.tid = gettid();261 linfo.task.tid = sys_gettid();
262 opts.link_info = &linfo;262 opts.link_info = &linfo;
263 opts.link_info_len = sizeof(linfo);263 opts.link_info_len = sizeof(linfo);
264 test_task_common(&opts, 0, 1);264 test_task_common(&opts, 0, 1);
265 265
266 linfo.task.tid = 0;266 linfo.task.tid = 0;
267 linfo.task.pid = getpid();267 linfo.task.pid = getpid();
268 /* This includes the parent thread, this thread,268 /* This includes the parent thread, this thread, watchdog timer thread
269 * and the do_nothing_wait thread269 * and the do_nothing_wait thread
270 */270 */
271 test_task_common(&opts, 2, 1);271 test_task_common(&opts, 3, 1);
272 272
273 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);273 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
274 ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");274 ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");
@@ -297,7 +297,7 @@ static void test_task_pid(void) · 22 unmodified lines
297 opts.link_info = &linfo;297 opts.link_info = &linfo;
298 opts.link_info_len = sizeof(linfo);298 opts.link_info_len = sizeof(linfo);
299 299
300 test_task_common(&opts, 1, 1);300 test_task_common(&opts, 2, 1);
301}301}
302 302
303static void test_task_pidfd(void)303static void test_task_pidfd(void)
@@ -315,7 +315,7 @@ static void test_task_pidfd(void) · 11 unmodified lines
315 opts.link_info = &linfo;315 opts.link_info = &linfo;
316 opts.link_info_len = sizeof(linfo);316 opts.link_info_len = sizeof(linfo);
317 317
318 test_task_common(&opts, 1, 1);318 test_task_common(&opts, 2, 1);
319 319
320 close(pidfd);320 close(pidfd);
321}321}
tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+0−4
@@ -16,10 +16,6 @@ · 15 unmodified lines
16#include "tcp_ca_kfunc.skel.h"16#include "tcp_ca_kfunc.skel.h"
17#include "bpf_cc_cubic.skel.h"17#include "bpf_cc_cubic.skel.h"
18 18
19#ifndef ENOTSUPP
20#define ENOTSUPP 524
21#endif
22
23static const unsigned int total_bytes = 10 * 1024 * 1024;19static const unsigned int total_bytes = 10 * 1024 * 1024;
24static int expected_stg = 0xeB9F;20static int expected_stg = 0xeB9F;
25 21
tools/testing/selftests/bpf/prog_tests/cb_refs.c+2−2
@@ -11,8 +11,8 @@ struct { · 10 unmodified lines
11 const char *prog_name;11 const char *prog_name;
12 const char *err_msg;12 const char *err_msg;
13} cb_refs_tests[] = {13} cb_refs_tests[] = {
14 { "underflow_prog", "reference has not been acquired before" },14 { "underflow_prog", "must point to scalar, or struct with scalar" },
15 { "leak_prog", "Unreleased reference" },15 { "leak_prog", "Possibly NULL pointer passed to helper arg2" },
16 { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */16 { "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */
17 { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */17 { "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */
18};18};
tools/testing/selftests/bpf/prog_tests/cgrp_local_storage.c+5−5
@@ -63,14 +63,14 @@ static void test_tp_btf(int cgroup_fd) · 62 unmodified lines
63 if (!ASSERT_OK(err, "map_delete_elem"))63 if (!ASSERT_OK(err, "map_delete_elem"))
64 goto out;64 goto out;
65 65
66 skel->bss->target_pid = syscall(SYS_gettid);66 skel->bss->target_pid = sys_gettid();
67 67
68 err = cgrp_ls_tp_btf__attach(skel);68 err = cgrp_ls_tp_btf__attach(skel);
69 if (!ASSERT_OK(err, "skel_attach"))69 if (!ASSERT_OK(err, "skel_attach"))
70 goto out;70 goto out;
71 71
72 syscall(SYS_gettid);72 sys_gettid();
73 syscall(SYS_gettid);73 sys_gettid();
74 74
75 skel->bss->target_pid = 0;75 skel->bss->target_pid = 0;
76 76
@@ -154,7 +154,7 @@ static void test_recursion(int cgroup_fd) · 77 unmodified lines
154 goto out;154 goto out;
155 155
156 /* trigger sys_enter, make sure it does not cause deadlock */156 /* trigger sys_enter, make sure it does not cause deadlock */
157 syscall(SYS_gettid);157 sys_gettid();
158 158
159out:159out:
160 cgrp_ls_recursion__destroy(skel);160 cgrp_ls_recursion__destroy(skel);
@@ -224,7 +224,7 @@ static void test_yes_rcu_lock(__u64 cgroup_id) · 63 unmodified lines
224 return;224 return;
225 225
226 CGROUP_MODE_SET(skel);226 CGROUP_MODE_SET(skel);
227 skel->bss->target_pid = syscall(SYS_gettid);227 skel->bss->target_pid = sys_gettid();
228 228
229 bpf_program__set_autoload(skel->progs.yes_rcu_lock, true);229 bpf_program__set_autoload(skel->progs.yes_rcu_lock, true);
230 err = cgrp_ls_sleepable__load(skel);230 err = cgrp_ls_sleepable__load(skel);
tools/testing/selftests/bpf/prog_tests/core_reloc.c+1−1
@@ -1010,7 +1010,7 @@ static void run_core_reloc_tests(bool use_btfgen) · 1009 unmodified lines
1010 struct data *data;1010 struct data *data;
1011 void *mmap_data = NULL;1011 void *mmap_data = NULL;
1012 1012
1013 my_pid_tgid = getpid() | ((uint64_t)syscall(SYS_gettid) << 32);1013 my_pid_tgid = getpid() | ((uint64_t)sys_gettid() << 32);
1014 1014
1015 for (i = 0; i < ARRAY_SIZE(test_cases); i++) {1015 for (i = 0; i < ARRAY_SIZE(test_cases); i++) {
1016 char btf_file[] = "/tmp/core_reloc.btf.XXXXXX";1016 char btf_file[] = "/tmp/core_reloc.btf.XXXXXX";
tools/testing/selftests/bpf/prog_tests/iters.c+2−2
@@ -192,8 +192,8 @@ static void subtest_task_iters(void) · 191 unmodified lines
192 syscall(SYS_getpgid);192 syscall(SYS_getpgid);
193 iters_task__detach(skel);193 iters_task__detach(skel);
194 ASSERT_EQ(skel->bss->procs_cnt, 1, "procs_cnt");194 ASSERT_EQ(skel->bss->procs_cnt, 1, "procs_cnt");
195 ASSERT_EQ(skel->bss->threads_cnt, thread_num + 1, "threads_cnt");195 ASSERT_EQ(skel->bss->threads_cnt, thread_num + 2, "threads_cnt");
196 ASSERT_EQ(skel->bss->proc_threads_cnt, thread_num + 1, "proc_threads_cnt");196 ASSERT_EQ(skel->bss->proc_threads_cnt, thread_num + 2, "proc_threads_cnt");
197 ASSERT_EQ(skel->bss->invalid_cnt, 0, "invalid_cnt");197 ASSERT_EQ(skel->bss->invalid_cnt, 0, "invalid_cnt");
198 pthread_mutex_unlock(&do_nothing_mutex);198 pthread_mutex_unlock(&do_nothing_mutex);
199 for (int i = 0; i < thread_num; i++)199 for (int i = 0; i < thread_num; i++)
tools/testing/selftests/bpf/prog_tests/kmem_cache_iter.c+126−0
@@ -0,0 +1,126 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Google */
3
4#include <test_progs.h>
5#include <bpf/libbpf.h>
6#include <bpf/btf.h>
7#include "kmem_cache_iter.skel.h"
8
9#define SLAB_NAME_MAX 32
10
11struct kmem_cache_result {
12 char name[SLAB_NAME_MAX];
13 long obj_size;
14};
15
16static void subtest_kmem_cache_iter_check_task_struct(struct kmem_cache_iter *skel)
17{
18 LIBBPF_OPTS(bpf_test_run_opts, opts,
19 .flags = 0, /* Run it with the current task */
20 );
21 int prog_fd = bpf_program__fd(skel->progs.check_task_struct);
22
23 /* Get task_struct and check it if's from a slab cache */
24 ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &opts), "prog_test_run");
25
26 /* The BPF program should set 'found' variable */
27 ASSERT_EQ(skel->bss->task_struct_found, 1, "task_struct_found");
28}
29
30static void subtest_kmem_cache_iter_check_slabinfo(struct kmem_cache_iter *skel)
31{
32 FILE *fp;
33 int map_fd;
34 char name[SLAB_NAME_MAX];
35 unsigned long objsize;
36 char rest_of_line[1000];
37 struct kmem_cache_result r;
38 int seen = 0;
39
40 fp = fopen("/proc/slabinfo", "r");
41 if (fp == NULL) {
42 /* CONFIG_SLUB_DEBUG is not enabled */
43 return;
44 }
45
46 map_fd = bpf_map__fd(skel->maps.slab_result);
47
48 /* Ignore first two lines for header */
49 fscanf(fp, "slabinfo - version: %*d.%*d\n");
50 fscanf(fp, "# %*s %*s %*s %*s %*s %*s : %[^\n]\n", rest_of_line);
51
52 /* Compare name and objsize only - others can be changes frequently */
53 while (fscanf(fp, "%s %*u %*u %lu %*u %*u : %[^\n]\n",
54 name, &objsize, rest_of_line) == 3) {
55 int ret = bpf_map_lookup_elem(map_fd, &seen, &r);
56
57 if (!ASSERT_OK(ret, "kmem_cache_lookup"))
58 break;
59
60 ASSERT_STREQ(r.name, name, "kmem_cache_name");
61 ASSERT_EQ(r.obj_size, objsize, "kmem_cache_objsize");
62
63 seen++;
64 }
65
66 ASSERT_EQ(skel->bss->kmem_cache_seen, seen, "kmem_cache_seen_eq");
67
68 fclose(fp);
69}
70
71static void subtest_kmem_cache_iter_open_coded(struct kmem_cache_iter *skel)
72{
73 LIBBPF_OPTS(bpf_test_run_opts, topts);
74 int err, fd;
75
76 /* No need to attach it, just run it directly */
77 fd = bpf_program__fd(skel->progs.open_coded_iter);
78
79 err = bpf_prog_test_run_opts(fd, &topts);
80 if (!ASSERT_OK(err, "test_run_opts err"))
81 return;
82 if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
83 return;
84
85 /* It should be same as we've seen from the explicit iterator */
86 ASSERT_EQ(skel->bss->open_coded_seen, skel->bss->kmem_cache_seen, "open_code_seen_eq");
87}
88
89void test_kmem_cache_iter(void)
90{
91 struct kmem_cache_iter *skel = NULL;
92 char buf[256];
93 int iter_fd;
94
95 skel = kmem_cache_iter__open_and_load();
96 if (!ASSERT_OK_PTR(skel, "kmem_cache_iter__open_and_load"))
97 return;
98
99 if (!ASSERT_OK(kmem_cache_iter__attach(skel), "skel_attach"))
100 goto destroy;
101
102 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.slab_info_collector));
103 if (!ASSERT_GE(iter_fd, 0, "iter_create"))
104 goto destroy;
105
106 memset(buf, 0, sizeof(buf));
107 while (read(iter_fd, buf, sizeof(buf) > 0)) {
108 /* Read out all contents */
109 printf("%s", buf);
110 }
111
112 /* Next reads should return 0 */
113 ASSERT_EQ(read(iter_fd, buf, sizeof(buf)), 0, "read");
114
115 if (test__start_subtest("check_task_struct"))
116 subtest_kmem_cache_iter_check_task_struct(skel);
117 if (test__start_subtest("check_slabinfo"))
118 subtest_kmem_cache_iter_check_slabinfo(skel);
119 if (test__start_subtest("open_coded_iter"))
120 subtest_kmem_cache_iter_open_coded(skel);
121
122 close(iter_fd);
123
124destroy:
125 kmem_cache_iter__destroy(skel);
126}
tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c+2−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
6#include "kprobe_multi_override.skel.h"6#include "kprobe_multi_override.skel.h"
7#include "kprobe_multi_session.skel.h"7#include "kprobe_multi_session.skel.h"
8#include "kprobe_multi_session_cookie.skel.h"8#include "kprobe_multi_session_cookie.skel.h"
9#include "kprobe_multi_verifier.skel.h"
9#include "bpf/libbpf_internal.h"10#include "bpf/libbpf_internal.h"
10#include "bpf/hashmap.h"11#include "bpf/hashmap.h"
11 12
@@ -764,4 +765,5 @@ void test_kprobe_multi_test(void) · 752 unmodified lines
764 test_session_skel_api();765 test_session_skel_api();
765 if (test__start_subtest("session_cookie"))766 if (test__start_subtest("session_cookie"))
766 test_session_cookie_skel_api();767 test_session_cookie_skel_api();
768 RUN_TESTS(kprobe_multi_verifier);
767}769}
tools/testing/selftests/bpf/prog_tests/linked_funcs.c+1−1
@@ -20,7 +20,7 @@ void test_linked_funcs(void) · 19 unmodified lines
20 bpf_program__set_autoload(skel->progs.handler1, true);20 bpf_program__set_autoload(skel->progs.handler1, true);
21 bpf_program__set_autoload(skel->progs.handler2, true);21 bpf_program__set_autoload(skel->progs.handler2, true);
22 22
23 skel->rodata->my_tid = syscall(SYS_gettid);23 skel->rodata->my_tid = sys_gettid();
24 skel->bss->syscall_id = SYS_getpgid;24 skel->bss->syscall_id = SYS_getpgid;
25 25
26 err = linked_funcs__load(skel);26 err = linked_funcs__load(skel);
tools/testing/selftests/bpf/prog_tests/log_buf.c+0−3
@@ -169,7 +169,6 @@ static void bpf_prog_load_log_buf(void) · 168 unmodified lines
169 ASSERT_GE(fd, 0, "good_fd1");169 ASSERT_GE(fd, 0, "good_fd1");
170 if (fd >= 0)170 if (fd >= 0)
171 close(fd);171 close(fd);
172 fd = -1;
173 172
174 /* log_level == 2 should always fill log_buf, even for good prog */173 /* log_level == 2 should always fill log_buf, even for good prog */
175 log_buf[0] = '\0';174 log_buf[0] = '\0';
@@ -180,7 +179,6 @@ static void bpf_prog_load_log_buf(void) · 4 unmodified lines
180 ASSERT_GE(fd, 0, "good_fd2");179 ASSERT_GE(fd, 0, "good_fd2");
181 if (fd >= 0)180 if (fd >= 0)
182 close(fd);181 close(fd);
183 fd = -1;
184 182
185 /* log_level == 0 should fill log_buf for bad prog */183 /* log_level == 0 should fill log_buf for bad prog */
186 log_buf[0] = '\0';184 log_buf[0] = '\0';
@@ -191,7 +189,6 @@ static void bpf_prog_load_log_buf(void) · 4 unmodified lines
191 ASSERT_LT(fd, 0, "bad_fd");189 ASSERT_LT(fd, 0, "bad_fd");
192 if (fd >= 0)190 if (fd >= 0)
193 close(fd);191 close(fd);
194 fd = -1;
195 192
196 free(log_buf);193 free(log_buf);
197}194}
tools/testing/selftests/bpf/prog_tests/lsm_cgroup.c+0−4
@@ -10,10 +10,6 @@ · 9 unmodified lines
10#include "cgroup_helpers.h"10#include "cgroup_helpers.h"
11#include "network_helpers.h"11#include "network_helpers.h"
12 12
13#ifndef ENOTSUPP
14#define ENOTSUPP 524
15#endif
16
17static struct btf *btf;13static struct btf *btf;
18 14
19static __u32 query_prog_cnt(int cgroup_fd, const char *attach_func)15static __u32 query_prog_cnt(int cgroup_fd, const char *attach_func)
tools/testing/selftests/bpf/prog_tests/map_in_map.c+131−1
@@ -5,7 +5,9 @@ · 4 unmodified lines
5#include <sys/syscall.h>5#include <sys/syscall.h>
6#include <test_progs.h>6#include <test_progs.h>
7#include <bpf/btf.h>7#include <bpf/btf.h>
8
8#include "access_map_in_map.skel.h"9#include "access_map_in_map.skel.h"
10#include "update_map_in_htab.skel.h"
9 11
10struct thread_ctx {12struct thread_ctx {
11 pthread_barrier_t barrier;13 pthread_barrier_t barrier;
@@ -127,6 +129,131 @@ out: · 115 unmodified lines
127 access_map_in_map__destroy(skel);129 access_map_in_map__destroy(skel);
128}130}
129 131
132static void add_del_fd_htab(int outer_fd)
133{
134 int inner_fd, err;
135 int key = 1;
136
137 inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
138 if (!ASSERT_OK_FD(inner_fd, "inner1"))
139 return;
140 err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_NOEXIST);
141 close(inner_fd);
142 if (!ASSERT_OK(err, "add"))
143 return;
144
145 /* Delete */
146 err = bpf_map_delete_elem(outer_fd, &key);
147 ASSERT_OK(err, "del");
148}
149
150static void overwrite_fd_htab(int outer_fd)
151{
152 int inner_fd, err;
153 int key = 1;
154
155 inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
156 if (!ASSERT_OK_FD(inner_fd, "inner1"))
157 return;
158 err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_NOEXIST);
159 close(inner_fd);
160 if (!ASSERT_OK(err, "add"))
161 return;
162
163 /* Overwrite */
164 inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr2", 4, 4, 1, NULL);
165 if (!ASSERT_OK_FD(inner_fd, "inner2"))
166 goto out;
167 err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_EXIST);
168 close(inner_fd);
169 if (!ASSERT_OK(err, "overwrite"))
170 goto out;
171
172 err = bpf_map_delete_elem(outer_fd, &key);
173 ASSERT_OK(err, "del");
174 return;
175out:
176 bpf_map_delete_elem(outer_fd, &key);
177}
178
179static void lookup_delete_fd_htab(int outer_fd)
180{
181 int key = 1, value;
182 int inner_fd, err;
183
184 inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
185 if (!ASSERT_OK_FD(inner_fd, "inner1"))
186 return;
187 err = bpf_map_update_elem(outer_fd, &key, &inner_fd, BPF_NOEXIST);
188 close(inner_fd);
189 if (!ASSERT_OK(err, "add"))
190 return;
191
192 /* lookup_and_delete is not supported for htab of maps */
193 err = bpf_map_lookup_and_delete_elem(outer_fd, &key, &value);
194 ASSERT_EQ(err, -ENOTSUPP, "lookup_del");
195
196 err = bpf_map_delete_elem(outer_fd, &key);
197 ASSERT_OK(err, "del");
198}
199
200static void batched_lookup_delete_fd_htab(int outer_fd)
201{
202 int keys[2] = {1, 2}, values[2];
203 unsigned int cnt, batch;
204 int inner_fd, err;
205
206 inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr1", 4, 4, 1, NULL);
207 if (!ASSERT_OK_FD(inner_fd, "inner1"))
208 return;
209
210 err = bpf_map_update_elem(outer_fd, &keys[0], &inner_fd, BPF_NOEXIST);
211 close(inner_fd);
212 if (!ASSERT_OK(err, "add1"))
213 return;
214
215 inner_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "arr2", 4, 4, 1, NULL);
216 if (!ASSERT_OK_FD(inner_fd, "inner2"))
217 goto out;
218 err = bpf_map_update_elem(outer_fd, &keys[1], &inner_fd, BPF_NOEXIST);
219 close(inner_fd);
220 if (!ASSERT_OK(err, "add2"))
221 goto out;
222
223 /* batched lookup_and_delete */
224 cnt = ARRAY_SIZE(keys);
225 err = bpf_map_lookup_and_delete_batch(outer_fd, NULL, &batch, keys, values, &cnt, NULL);
226 ASSERT_TRUE((!err || err == -ENOENT), "delete_batch ret");
227 ASSERT_EQ(cnt, ARRAY_SIZE(keys), "delete_batch cnt");
228
229out:
230 bpf_map_delete_elem(outer_fd, &keys[0]);
231}
232
233static void test_update_map_in_htab(bool preallocate)
234{
235 struct update_map_in_htab *skel;
236 int err, fd;
237
238 skel = update_map_in_htab__open();
239 if (!ASSERT_OK_PTR(skel, "open"))
240 return;
241
242 err = update_map_in_htab__load(skel);
243 if (!ASSERT_OK(err, "load"))
244 goto out;
245
246 fd = preallocate ? bpf_map__fd(skel->maps.outer_htab_map) :
247 bpf_map__fd(skel->maps.outer_alloc_htab_map);
248
249 add_del_fd_htab(fd);
250 overwrite_fd_htab(fd);
251 lookup_delete_fd_htab(fd);
252 batched_lookup_delete_fd_htab(fd);
253out:
254 update_map_in_htab__destroy(skel);
255}
256
130void test_map_in_map(void)257void test_map_in_map(void)
131{258{
132 if (test__start_subtest("acc_map_in_array"))259 if (test__start_subtest("acc_map_in_array"))
@@ -137,5 +264,8 @@ void test_map_in_map(void) · 4 unmodified lines
137 test_map_in_map_access("access_map_in_htab", "outer_htab_map");264 test_map_in_map_access("access_map_in_htab", "outer_htab_map");
138 if (test__start_subtest("sleepable_acc_map_in_htab"))265 if (test__start_subtest("sleepable_acc_map_in_htab"))
139 test_map_in_map_access("sleepable_access_map_in_htab", "outer_htab_map");266 test_map_in_map_access("sleepable_access_map_in_htab", "outer_htab_map");
267 if (test__start_subtest("update_map_in_htab"))
268 test_update_map_in_htab(true);
269 if (test__start_subtest("update_map_in_alloc_htab"))
270 test_update_map_in_htab(false);
140}271}
141
tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c+1−1
@@ -23,7 +23,7 @@ static int get_pid_tgid(pid_t *pid, pid_t *tgid, · 22 unmodified lines
23 struct stat st;23 struct stat st;
24 int err;24 int err;
25 25
26 *pid = syscall(SYS_gettid);26 *pid = sys_gettid();
27 *tgid = getpid();27 *tgid = getpid();
28 28
29 err = stat("/proc/self/ns/pid", &st);29 err = stat("/proc/self/ns/pid", &st);
tools/testing/selftests/bpf/prog_tests/perf_link.c+13−2
@@ -4,8 +4,12 @@ · 3 unmodified lines
4#include <pthread.h>4#include <pthread.h>
5#include <sched.h>5#include <sched.h>
6#include <test_progs.h>6#include <test_progs.h>
7#include "testing_helpers.h"
7#include "test_perf_link.skel.h"8#include "test_perf_link.skel.h"
8 9
10#define BURN_TIMEOUT_MS 100
11#define BURN_TIMEOUT_NS BURN_TIMEOUT_MS * 1000000
12
9static void burn_cpu(void)13static void burn_cpu(void)
10{14{
11 volatile int j = 0;15 volatile int j = 0;
@@ -32,6 +36,7 @@ void serial_test_perf_link(void) · 20 unmodified lines
32 int run_cnt_before, run_cnt_after;36 int run_cnt_before, run_cnt_after;
33 struct bpf_link_info info;37 struct bpf_link_info info;
34 __u32 info_len = sizeof(info);38 __u32 info_len = sizeof(info);
39 __u64 timeout_time_ns;
35 40
36 /* create perf event */41 /* create perf event */
37 memset(&attr, 0, sizeof(attr));42 memset(&attr, 0, sizeof(attr));
@@ -63,8 +68,14 @@ void serial_test_perf_link(void) · 25 unmodified lines
63 ASSERT_GT(info.prog_id, 0, "link_prog_id");68 ASSERT_GT(info.prog_id, 0, "link_prog_id");
64 69
65 /* ensure we get at least one perf_event prog execution */70 /* ensure we get at least one perf_event prog execution */
66 burn_cpu();71 timeout_time_ns = get_time_ns() + BURN_TIMEOUT_NS;
67 ASSERT_GT(skel->bss->run_cnt, 0, "run_cnt");72 while (true) {
73 burn_cpu();
74 if (skel->bss->run_cnt > 0)
75 break;
76 if (!ASSERT_LT(get_time_ns(), timeout_time_ns, "run_cnt_timeout"))
77 break;
78 }
68 79
69 /* perf_event is still active, but we close link and BPF program80 /* perf_event is still active, but we close link and BPF program
70 * shouldn't be executed anymore81 * shouldn't be executed anymore
tools/testing/selftests/bpf/prog_tests/raw_tp_null.c+25−0
@@ -0,0 +1,25 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#include <test_progs.h>
5#include "raw_tp_null.skel.h"
6
7void test_raw_tp_null(void)
8{
9 struct raw_tp_null *skel;
10
11 skel = raw_tp_null__open_and_load();
12 if (!ASSERT_OK_PTR(skel, "raw_tp_null__open_and_load"))
13 return;
14
15 skel->bss->tid = sys_gettid();
16
17 if (!ASSERT_OK(raw_tp_null__attach(skel), "raw_tp_null__attach"))
18 goto end;
19
20 ASSERT_OK(trigger_module_test_read(2), "trigger testmod read");
21 ASSERT_EQ(skel->bss->i, 3, "invocations");
22
23end:
24 raw_tp_null__destroy(skel);
25}
tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c+2−2
@@ -21,7 +21,7 @@ static void test_success(void) · 20 unmodified lines
21 if (!ASSERT_OK_PTR(skel, "skel_open"))21 if (!ASSERT_OK_PTR(skel, "skel_open"))
22 return;22 return;
23 23
24 skel->bss->target_pid = syscall(SYS_gettid);24 skel->bss->target_pid = sys_gettid();
25 25
26 bpf_program__set_autoload(skel->progs.get_cgroup_id, true);26 bpf_program__set_autoload(skel->progs.get_cgroup_id, true);
27 bpf_program__set_autoload(skel->progs.task_succ, true);27 bpf_program__set_autoload(skel->progs.task_succ, true);
@@ -58,7 +58,7 @@ static void test_rcuptr_acquire(void) · 30 unmodified lines
58 if (!ASSERT_OK_PTR(skel, "skel_open"))58 if (!ASSERT_OK_PTR(skel, "skel_open"))
59 return;59 return;
60 60
61 skel->bss->target_pid = syscall(SYS_gettid);61 skel->bss->target_pid = sys_gettid();
62 62
63 bpf_program__set_autoload(skel->progs.task_acquire, true);63 bpf_program__set_autoload(skel->progs.task_acquire, true);
64 err = rcu_read_lock__load(skel);64 err = rcu_read_lock__load(skel);
tools/testing/selftests/bpf/prog_tests/send_signal.c+111−35
@@ -3,22 +3,32 @@ · 2 unmodified lines
3#include <sys/time.h>3#include <sys/time.h>
4#include <sys/resource.h>4#include <sys/resource.h>
5#include "test_send_signal_kern.skel.h"5#include "test_send_signal_kern.skel.h"
6#include "io_helpers.h"
6 7
7static int sigusr1_received;8static int sigusr1_received;
8 9
9static void sigusr1_handler(int signum)10static void sigusr1_handler(int signum)
10{11{
11 sigusr1_received = 1;12 sigusr1_received = 8;
13}
14
15static void sigusr1_siginfo_handler(int s, siginfo_t *i, void *v)
16{
17 sigusr1_received = (int)(long long)i->si_value.sival_ptr;
12}18}
13 19
14static void test_send_signal_common(struct perf_event_attr *attr,20static void test_send_signal_common(struct perf_event_attr *attr,
15 bool signal_thread)21 bool signal_thread, bool remote)
16{22{
17 struct test_send_signal_kern *skel;23 struct test_send_signal_kern *skel;
24 struct sigaction sa;
18 int pipe_c2p[2], pipe_p2c[2];25 int pipe_c2p[2], pipe_p2c[2];
19 int err = -1, pmu_fd = -1;26 int err = -1, pmu_fd = -1;
27 volatile int j = 0;
28 int retry_count;
20 char buf[256];29 char buf[256];
21 pid_t pid;30 pid_t pid;
31 int old_prio;
22 32
23 if (!ASSERT_OK(pipe(pipe_c2p), "pipe_c2p"))33 if (!ASSERT_OK(pipe(pipe_c2p), "pipe_c2p"))
24 return;34 return;
@@ -39,11 +49,14 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 14 unmodified lines
39 }49 }
40 50
41 if (pid == 0) {51 if (pid == 0) {
42 int old_prio;
43 volatile int j = 0;
44
45 /* install signal handler and notify parent */52 /* install signal handler and notify parent */
46 ASSERT_NEQ(signal(SIGUSR1, sigusr1_handler), SIG_ERR, "signal");53 if (remote) {
54 sa.sa_sigaction = sigusr1_siginfo_handler;
55 sa.sa_flags = SA_RESTART | SA_SIGINFO;
56 ASSERT_NEQ(sigaction(SIGUSR1, &sa, NULL), -1, "sigaction");
57 } else {
58 ASSERT_NEQ(signal(SIGUSR1, sigusr1_handler), SIG_ERR, "signal");
59 }
47 60
48 close(pipe_c2p[0]); /* close read */61 close(pipe_c2p[0]); /* close read */
49 close(pipe_p2c[1]); /* close write */62 close(pipe_p2c[1]); /* close write */
@@ -52,10 +65,12 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 2 unmodified lines
52 * that if an interrupt happens, the underlying task65 * that if an interrupt happens, the underlying task
53 * is this process.66 * is this process.
54 */67 */
55 errno = 0;68 if (!remote) {
56 old_prio = getpriority(PRIO_PROCESS, 0);69 errno = 0;
57 ASSERT_OK(errno, "getpriority");70 old_prio = getpriority(PRIO_PROCESS, 0);
58 ASSERT_OK(setpriority(PRIO_PROCESS, 0, -20), "setpriority");71 ASSERT_OK(errno, "getpriority");
72 ASSERT_OK(setpriority(PRIO_PROCESS, 0, -20), "setpriority");
73 }
59 74
60 /* notify parent signal handler is installed */75 /* notify parent signal handler is installed */
61 ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write");76 ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write");
@@ -66,20 +81,25 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 4 unmodified lines
66 /* wait a little for signal handler */81 /* wait a little for signal handler */
67 for (int i = 0; i < 1000000000 && !sigusr1_received; i++) {82 for (int i = 0; i < 1000000000 && !sigusr1_received; i++) {
68 j /= i + j + 1;83 j /= i + j + 1;
69 if (!attr)84 if (remote)
70 /* trigger the nanosleep tracepoint program. */85 sleep(1);
71 usleep(1);86 else
87 if (!attr)
88 /* trigger the nanosleep tracepoint program. */
89 usleep(1);
72 }90 }
73 91
74 buf[0] = sigusr1_received ? '2' : '0';92 buf[0] = sigusr1_received;
75 ASSERT_EQ(sigusr1_received, 1, "sigusr1_received");93
94 ASSERT_EQ(sigusr1_received, 8, "sigusr1_received");
76 ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write");95 ASSERT_EQ(write(pipe_c2p[1], buf, 1), 1, "pipe_write");
77 96
78 /* wait for parent notification and exit */97 /* wait for parent notification and exit */
79 ASSERT_EQ(read(pipe_p2c[0], buf, 1), 1, "pipe_read");98 ASSERT_EQ(read(pipe_p2c[0], buf, 1), 1, "pipe_read");
80 99
81 /* restore the old priority */100 /* restore the old priority */
82 ASSERT_OK(setpriority(PRIO_PROCESS, 0, old_prio), "setpriority");101 if (!remote)
102 ASSERT_OK(setpriority(PRIO_PROCESS, 0, old_prio), "setpriority");
83 103
84 close(pipe_c2p[1]);104 close(pipe_c2p[1]);
85 close(pipe_p2c[0]);105 close(pipe_p2c[0]);
@@ -93,6 +113,17 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 7 unmodified lines
93 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))113 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
94 goto skel_open_load_failure;114 goto skel_open_load_failure;
95 115
116 /* boost with a high priority so we got a higher chance
117 * that if an interrupt happens, the underlying task
118 * is this process.
119 */
120 if (remote) {
121 errno = 0;
122 old_prio = getpriority(PRIO_PROCESS, 0);
123 ASSERT_OK(errno, "getpriority");
124 ASSERT_OK(setpriority(PRIO_PROCESS, 0, -20), "setpriority");
125 }
126
96 if (!attr) {127 if (!attr) {
97 err = test_send_signal_kern__attach(skel);128 err = test_send_signal_kern__attach(skel);
98 if (!ASSERT_OK(err, "skel_attach")) {129 if (!ASSERT_OK(err, "skel_attach")) {
@@ -100,8 +131,12 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 1 unmodified line
100 goto destroy_skel;131 goto destroy_skel;
101 }132 }
102 } else {133 } else {
103 pmu_fd = syscall(__NR_perf_event_open, attr, pid, -1 /* cpu */,134 if (!remote)
104 -1 /* group id */, 0 /* flags */);135 pmu_fd = syscall(__NR_perf_event_open, attr, pid, -1 /* cpu */,
136 -1 /* group id */, 0 /* flags */);
137 else
138 pmu_fd = syscall(__NR_perf_event_open, attr, getpid(), -1 /* cpu */,
139 -1 /* group id */, 0 /* flags */);
105 if (!ASSERT_GE(pmu_fd, 0, "perf_event_open")) {140 if (!ASSERT_GE(pmu_fd, 0, "perf_event_open")) {
106 err = -1;141 err = -1;
107 goto destroy_skel;142 goto destroy_skel;
@@ -119,13 +154,36 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 11 unmodified lines
119 /* trigger the bpf send_signal */154 /* trigger the bpf send_signal */
120 skel->bss->signal_thread = signal_thread;155 skel->bss->signal_thread = signal_thread;
121 skel->bss->sig = SIGUSR1;156 skel->bss->sig = SIGUSR1;
122 skel->bss->pid = pid;157 if (!remote) {
158 skel->bss->target_pid = 0;
159 skel->bss->pid = pid;
160 } else {
161 skel->bss->target_pid = pid;
162 skel->bss->pid = getpid();
163 }
123 164
124 /* notify child that bpf program can send_signal now */165 /* notify child that bpf program can send_signal now */
125 ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write");166 ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write");
126 167
127 /* wait for result */168 for (retry_count = 0;;) {
128 err = read(pipe_c2p[0], buf, 1);169 /* For the remote test, the BPF program is triggered from this
170 * process but the other process/thread is signaled.
171 */
172 if (remote) {
173 if (!attr) {
174 for (int i = 0; i < 10; i++)
175 usleep(1);
176 } else {
177 for (int i = 0; i < 100000000; i++)
178 j /= i + 1;
179 }
180 }
181 /* wait for result */
182 err = read_with_timeout(pipe_c2p[0], buf, 1, 100);
183 if (err == -EAGAIN && retry_count++ < 10000)
184 continue;
185 break;
186 }
129 if (!ASSERT_GE(err, 0, "reading pipe"))187 if (!ASSERT_GE(err, 0, "reading pipe"))
130 goto disable_pmu;188 goto disable_pmu;
131 if (!ASSERT_GT(err, 0, "reading pipe error: size 0")) {189 if (!ASSERT_GT(err, 0, "reading pipe error: size 0")) {
@@ -133,7 +191,7 @@ static void test_send_signal_common(struct perf_event_attr *attr, · 1 unmodified line
133 goto disable_pmu;191 goto disable_pmu;
134 }192 }
135 193
136 ASSERT_EQ(buf[0], '2', "incorrect result");194 ASSERT_EQ(buf[0], 8, "incorrect result");
137 195
138 /* notify child safe to exit */196 /* notify child safe to exit */
139 ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write");197 ASSERT_EQ(write(pipe_p2c[1], buf, 1), 1, "pipe_write");
@@ -142,18 +200,21 @@ disable_pmu: · 2 unmodified lines
142 close(pmu_fd);200 close(pmu_fd);
143destroy_skel:201destroy_skel:
144 test_send_signal_kern__destroy(skel);202 test_send_signal_kern__destroy(skel);
203 /* restore the old priority */
204 if (remote)
205 ASSERT_OK(setpriority(PRIO_PROCESS, 0, old_prio), "setpriority");
145skel_open_load_failure:206skel_open_load_failure:
146 close(pipe_c2p[0]);207 close(pipe_c2p[0]);
147 close(pipe_p2c[1]);208 close(pipe_p2c[1]);
148 wait(NULL);209 wait(NULL);
149}210}
150 211
151static void test_send_signal_tracepoint(bool signal_thread)212static void test_send_signal_tracepoint(bool signal_thread, bool remote)
152{213{
153 test_send_signal_common(NULL, signal_thread);214 test_send_signal_common(NULL, signal_thread, remote);
154}215}
155 216
156static void test_send_signal_perf(bool signal_thread)217static void test_send_signal_perf(bool signal_thread, bool remote)
157{218{
158 struct perf_event_attr attr = {219 struct perf_event_attr attr = {
159 .freq = 1,220 .freq = 1,
@@ -162,13 +223,14 @@ static void test_send_signal_perf(bool signal_thread) · 2 unmodified lines
162 .config = PERF_COUNT_SW_CPU_CLOCK,223 .config = PERF_COUNT_SW_CPU_CLOCK,
163 };224 };
164 225
165 test_send_signal_common(&attr, signal_thread);226 test_send_signal_common(&attr, signal_thread, remote);
166}227}
167 228
168static void test_send_signal_nmi(bool signal_thread)229static void test_send_signal_nmi(bool signal_thread, bool remote)
169{230{
170 struct perf_event_attr attr = {231 struct perf_event_attr attr = {
171 .sample_period = 1,232 .freq = 1,
233 .sample_freq = 1000,
172 .type = PERF_TYPE_HARDWARE,234 .type = PERF_TYPE_HARDWARE,
173 .config = PERF_COUNT_HW_CPU_CYCLES,235 .config = PERF_COUNT_HW_CPU_CYCLES,
174 };236 };
@@ -191,21 +253,35 @@ static void test_send_signal_nmi(bool signal_thread) · 16 unmodified lines
191 close(pmu_fd);253 close(pmu_fd);
192 }254 }
193 255
194 test_send_signal_common(&attr, signal_thread);256 test_send_signal_common(&attr, signal_thread, remote);
195}257}
196 258
197void test_send_signal(void)259void test_send_signal(void)
198{260{
199 if (test__start_subtest("send_signal_tracepoint"))261 if (test__start_subtest("send_signal_tracepoint"))
200 test_send_signal_tracepoint(false);262 test_send_signal_tracepoint(false, false);
201 if (test__start_subtest("send_signal_perf"))263 if (test__start_subtest("send_signal_perf"))
202 test_send_signal_perf(false);264 test_send_signal_perf(false, false);
203 if (test__start_subtest("send_signal_nmi"))265 if (test__start_subtest("send_signal_nmi"))
204 test_send_signal_nmi(false);266 test_send_signal_nmi(false, false);
205 if (test__start_subtest("send_signal_tracepoint_thread"))267 if (test__start_subtest("send_signal_tracepoint_thread"))
206 test_send_signal_tracepoint(true);268 test_send_signal_tracepoint(true, false);
207 if (test__start_subtest("send_signal_perf_thread"))269 if (test__start_subtest("send_signal_perf_thread"))
208 test_send_signal_perf(true);270 test_send_signal_perf(true, false);
209 if (test__start_subtest("send_signal_nmi_thread"))271 if (test__start_subtest("send_signal_nmi_thread"))
210 test_send_signal_nmi(true);272 test_send_signal_nmi(true, false);
273
274 /* Signal remote thread and thread group */
275 if (test__start_subtest("send_signal_tracepoint_remote"))
276 test_send_signal_tracepoint(false, true);
277 if (test__start_subtest("send_signal_perf_remote"))
278 test_send_signal_perf(false, true);
279 if (test__start_subtest("send_signal_nmi_remote"))
280 test_send_signal_nmi(false, true);
281 if (test__start_subtest("send_signal_tracepoint_thread_remote"))
282 test_send_signal_tracepoint(true, true);
283 if (test__start_subtest("send_signal_perf_thread_remote"))
284 test_send_signal_perf(true, true);
285 if (test__start_subtest("send_signal_nmi_thread_remote"))
286 test_send_signal_nmi(true, true);
211}287}
tools/testing/selftests/bpf/prog_tests/sock_addr.c+0−4
@@ -23,10 +23,6 @@ · 22 unmodified lines
23#include "getpeername_unix_prog.skel.h"23#include "getpeername_unix_prog.skel.h"
24#include "network_helpers.h"24#include "network_helpers.h"
25 25
26#ifndef ENOTSUPP
27# define ENOTSUPP 524
28#endif
29
30#define TEST_NS "sock_addr"26#define TEST_NS "sock_addr"
31#define TEST_IF_PREFIX "test_sock_addr"27#define TEST_IF_PREFIX "test_sock_addr"
32#define TEST_IPV4 "127.0.0.4"28#define TEST_IPV4 "127.0.0.4"
tools/testing/selftests/bpf/prog_tests/sock_create.c+348−0
@@ -0,0 +1,348 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bpf.h>
3#include <test_progs.h>
4#include "cgroup_helpers.h"
5
6static char bpf_log_buf[4096];
7static bool verbose;
8
9enum sock_create_test_error {
10 OK = 0,
11 DENY_CREATE,
12};
13
14static struct sock_create_test {
15 const char *descr;
16 const struct bpf_insn insns[64];
17 enum bpf_attach_type attach_type;
18 enum bpf_attach_type expected_attach_type;
19
20 int domain;
21 int type;
22 int protocol;
23
24 int optname;
25 int optval;
26 enum sock_create_test_error error;
27} tests[] = {
28 {
29 .descr = "AF_INET set priority",
30 .insns = {
31 /* r3 = 123 (priority) */
32 BPF_MOV64_IMM(BPF_REG_3, 123),
33 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
34 offsetof(struct bpf_sock, priority)),
35
36 /* return 1 */
37 BPF_MOV64_IMM(BPF_REG_0, 1),
38 BPF_EXIT_INSN(),
39 },
40 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
41 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
42
43 .domain = AF_INET,
44 .type = SOCK_DGRAM,
45
46 .optname = SO_PRIORITY,
47 .optval = 123,
48 },
49 {
50 .descr = "AF_INET6 set priority",
51 .insns = {
52 /* r3 = 123 (priority) */
53 BPF_MOV64_IMM(BPF_REG_3, 123),
54 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
55 offsetof(struct bpf_sock, priority)),
56
57 /* return 1 */
58 BPF_MOV64_IMM(BPF_REG_0, 1),
59 BPF_EXIT_INSN(),
60 },
61 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
62 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
63
64 .domain = AF_INET6,
65 .type = SOCK_DGRAM,
66
67 .optname = SO_PRIORITY,
68 .optval = 123,
69 },
70 {
71 .descr = "AF_INET set mark",
72 .insns = {
73 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
74
75 /* get uid of process */
76 BPF_EMIT_CALL(BPF_FUNC_get_current_uid_gid),
77 BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffffffff),
78
79 /* if uid is 0, use given mark(666), else use uid as the mark */
80 BPF_MOV64_REG(BPF_REG_3, BPF_REG_0),
81 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
82 BPF_MOV64_IMM(BPF_REG_3, 666),
83
84 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
85 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
86 offsetof(struct bpf_sock, mark)),
87
88 /* return 1 */
89 BPF_MOV64_IMM(BPF_REG_0, 1),
90 BPF_EXIT_INSN(),
91 },
92 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
93 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
94
95 .domain = AF_INET,
96 .type = SOCK_DGRAM,
97
98 .optname = SO_MARK,
99 .optval = 666,
100 },
101 {
102 .descr = "AF_INET6 set mark",
103 .insns = {
104 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
105
106 /* get uid of process */
107 BPF_EMIT_CALL(BPF_FUNC_get_current_uid_gid),
108 BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffffffff),
109
110 /* if uid is 0, use given mark(666), else use uid as the mark */
111 BPF_MOV64_REG(BPF_REG_3, BPF_REG_0),
112 BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
113 BPF_MOV64_IMM(BPF_REG_3, 666),
114
115 BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
116 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
117 offsetof(struct bpf_sock, mark)),
118
119 /* return 1 */
120 BPF_MOV64_IMM(BPF_REG_0, 1),
121 BPF_EXIT_INSN(),
122 },
123 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
124 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
125
126 .domain = AF_INET6,
127 .type = SOCK_DGRAM,
128
129 .optname = SO_MARK,
130 .optval = 666,
131 },
132 {
133 .descr = "AF_INET bound to iface",
134 .insns = {
135 /* r3 = 1 (lo interface) */
136 BPF_MOV64_IMM(BPF_REG_3, 1),
137 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
138 offsetof(struct bpf_sock, bound_dev_if)),
139
140 /* return 1 */
141 BPF_MOV64_IMM(BPF_REG_0, 1),
142 BPF_EXIT_INSN(),
143 },
144 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
145 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
146
147 .domain = AF_INET,
148 .type = SOCK_DGRAM,
149
150 .optname = SO_BINDTOIFINDEX,
151 .optval = 1,
152 },
153 {
154 .descr = "AF_INET6 bound to iface",
155 .insns = {
156 /* r3 = 1 (lo interface) */
157 BPF_MOV64_IMM(BPF_REG_3, 1),
158 BPF_STX_MEM(BPF_W, BPF_REG_1, BPF_REG_3,
159 offsetof(struct bpf_sock, bound_dev_if)),
160
161 /* return 1 */
162 BPF_MOV64_IMM(BPF_REG_0, 1),
163 BPF_EXIT_INSN(),
164 },
165 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
166 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
167
168 .domain = AF_INET6,
169 .type = SOCK_DGRAM,
170
171 .optname = SO_BINDTOIFINDEX,
172 .optval = 1,
173 },
174 {
175 .descr = "block AF_INET, SOCK_DGRAM, IPPROTO_ICMP socket",
176 .insns = {
177 BPF_MOV64_IMM(BPF_REG_0, 1), /* r0 = verdict */
178
179 /* sock->family == AF_INET */
180 BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
181 offsetof(struct bpf_sock, family)),
182 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, AF_INET, 5),
183
184 /* sock->type == SOCK_DGRAM */
185 BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
186 offsetof(struct bpf_sock, type)),
187 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, SOCK_DGRAM, 3),
188
189 /* sock->protocol == IPPROTO_ICMP */
190 BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
191 offsetof(struct bpf_sock, protocol)),
192 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, IPPROTO_ICMP, 1),
193
194 /* return 0 (block) */
195 BPF_MOV64_IMM(BPF_REG_0, 0),
196 BPF_EXIT_INSN(),
197 },
198 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
199 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
200
201 .domain = AF_INET,
202 .type = SOCK_DGRAM,
203 .protocol = IPPROTO_ICMP,
204
205 .error = DENY_CREATE,
206 },
207 {
208 .descr = "block AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6 socket",
209 .insns = {
210 BPF_MOV64_IMM(BPF_REG_0, 1), /* r0 = verdict */
211
212 /* sock->family == AF_INET6 */
213 BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
214 offsetof(struct bpf_sock, family)),
215 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, AF_INET6, 5),
216
217 /* sock->type == SOCK_DGRAM */
218 BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
219 offsetof(struct bpf_sock, type)),
220 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, SOCK_DGRAM, 3),
221
222 /* sock->protocol == IPPROTO_ICMPV6 */
223 BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1,
224 offsetof(struct bpf_sock, protocol)),
225 BPF_JMP_IMM(BPF_JNE, BPF_REG_2, IPPROTO_ICMPV6, 1),
226
227 /* return 0 (block) */
228 BPF_MOV64_IMM(BPF_REG_0, 0),
229 BPF_EXIT_INSN(),
230 },
231 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
232 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
233
234 .domain = AF_INET,
235 .type = SOCK_DGRAM,
236 .protocol = IPPROTO_ICMPV6,
237
238 .error = DENY_CREATE,
239 },
240 {
241 .descr = "load w/o expected_attach_type (compat mode)",
242 .insns = {
243 /* return 1 */
244 BPF_MOV64_IMM(BPF_REG_0, 1),
245 BPF_EXIT_INSN(),
246 },
247 .expected_attach_type = 0,
248 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
249
250 .domain = AF_INET,
251 .type = SOCK_STREAM,
252 },
253};
254
255static int load_prog(const struct bpf_insn *insns,
256 enum bpf_attach_type expected_attach_type)
257{
258 LIBBPF_OPTS(bpf_prog_load_opts, opts,
259 .expected_attach_type = expected_attach_type,
260 .log_level = 2,
261 .log_buf = bpf_log_buf,
262 .log_size = sizeof(bpf_log_buf),
263 );
264 int fd, insns_cnt = 0;
265
266 for (;
267 insns[insns_cnt].code != (BPF_JMP | BPF_EXIT);
268 insns_cnt++) {
269 }
270 insns_cnt++;
271
272 fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns,
273 insns_cnt, &opts);
274 if (verbose && fd < 0)
275 fprintf(stderr, "%s\n", bpf_log_buf);
276
277 return fd;
278}
279
280static int run_test(int cgroup_fd, struct sock_create_test *test)
281{
282 int sock_fd, err, prog_fd, optval, ret = -1;
283 socklen_t optlen = sizeof(optval);
284
285 prog_fd = load_prog(test->insns, test->expected_attach_type);
286 if (prog_fd < 0) {
287 log_err("Failed to load BPF program");
288 return -1;
289 }
290
291 err = bpf_prog_attach(prog_fd, cgroup_fd, test->attach_type, 0);
292 if (err < 0) {
293 log_err("Failed to attach BPF program");
294 goto close_prog_fd;
295 }
296
297 sock_fd = socket(test->domain, test->type, test->protocol);
298 if (sock_fd < 0) {
299 if (test->error == DENY_CREATE)
300 ret = 0;
301 else
302 log_err("Failed to create socket");
303
304 goto detach_prog;
305 }
306
307 if (test->optname) {
308 err = getsockopt(sock_fd, SOL_SOCKET, test->optname, &optval, &optlen);
309 if (err) {
310 log_err("Failed to call getsockopt");
311 goto cleanup;
312 }
313
314 if (optval != test->optval) {
315 errno = 0;
316 log_err("getsockopt returned unexpected optval");
317 goto cleanup;
318 }
319 }
320
321 ret = test->error != OK;
322
323cleanup:
324 close(sock_fd);
325detach_prog:
326 bpf_prog_detach2(prog_fd, cgroup_fd, test->attach_type);
327close_prog_fd:
328 close(prog_fd);
329 return ret;
330}
331
332void test_sock_create(void)
333{
334 int cgroup_fd, i;
335
336 cgroup_fd = test__join_cgroup("/sock_create");
337 if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup"))
338 return;
339
340 for (i = 0; i < ARRAY_SIZE(tests); i++) {
341 if (!test__start_subtest(tests[i].descr))
342 continue;
343
344 ASSERT_OK(run_test(cgroup_fd, &tests[i]), tests[i].descr);
345 }
346
347 close(cgroup_fd);
348}
tools/testing/selftests/bpf/prog_tests/sock_post_bind.crenamed from tools/testing/selftests/bpf/test_sock.c+62−192
@@ -1,132 +1,35 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2018 Facebook2#include <linux/bpf.h>
3 3#include <test_progs.h>
4#include <stdio.h>
5#include <unistd.h>
6
7#include <arpa/inet.h>
8#include <sys/types.h>
9#include <sys/socket.h>
10
11#include <linux/filter.h>
12
13#include <bpf/bpf.h>
14
15#include "cgroup_helpers.h"4#include "cgroup_helpers.h"
16#include <bpf/bpf_endian.h>
17#include "bpf_util.h"
18 5
19#define CG_PATH "/foo"6#define TEST_NS "sock_post_bind"
20#define MAX_INSNS 512
21 7
22char bpf_log_buf[BPF_LOG_BUF_SIZE];8static char bpf_log_buf[4096];
23static bool verbose = false;
24 9
25struct sock_test {10static struct sock_post_bind_test {
26 const char *descr;11 const char *descr;
27 /* BPF prog properties */12 /* BPF prog properties */
28 struct bpf_insn insns[MAX_INSNS];13 const struct bpf_insn insns[64];
29 enum bpf_attach_type expected_attach_type;14 enum bpf_attach_type attach_type;
30 enum bpf_attach_type attach_type;15 enum bpf_attach_type expected_attach_type;
31 /* Socket properties */16 /* Socket properties */
32 int domain;17 int domain;
33 int type;18 int type;
34 /* Endpoint to bind() to */19 /* Endpoint to bind() to */
35 const char *ip;20 const char *ip;
36 unsigned short port;21 unsigned short port;
37 unsigned short port_retry;22 unsigned short port_retry;
23
38 /* Expected test result */24 /* Expected test result */
39 enum {25 enum {
40 LOAD_REJECT,
41 ATTACH_REJECT,26 ATTACH_REJECT,
42 BIND_REJECT,27 BIND_REJECT,
43 SUCCESS,28 SUCCESS,
44 RETRY_SUCCESS,29 RETRY_SUCCESS,
45 RETRY_REJECT30 RETRY_REJECT
46 } result;31 } result;
47};32} tests[] = {
48
49static struct sock_test tests[] = {
50 {
51 .descr = "bind4 load with invalid access: src_ip6",
52 .insns = {
53 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
54 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
55 offsetof(struct bpf_sock, src_ip6[0])),
56 BPF_MOV64_IMM(BPF_REG_0, 1),
57 BPF_EXIT_INSN(),
58 },
59 .expected_attach_type = BPF_CGROUP_INET4_POST_BIND,
60 .attach_type = BPF_CGROUP_INET4_POST_BIND,
61 .result = LOAD_REJECT,
62 },
63 {
64 .descr = "bind4 load with invalid access: mark",
65 .insns = {
66 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
67 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
68 offsetof(struct bpf_sock, mark)),
69 BPF_MOV64_IMM(BPF_REG_0, 1),
70 BPF_EXIT_INSN(),
71 },
72 .expected_attach_type = BPF_CGROUP_INET4_POST_BIND,
73 .attach_type = BPF_CGROUP_INET4_POST_BIND,
74 .result = LOAD_REJECT,
75 },
76 {
77 .descr = "bind6 load with invalid access: src_ip4",
78 .insns = {
79 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
80 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
81 offsetof(struct bpf_sock, src_ip4)),
82 BPF_MOV64_IMM(BPF_REG_0, 1),
83 BPF_EXIT_INSN(),
84 },
85 .expected_attach_type = BPF_CGROUP_INET6_POST_BIND,
86 .attach_type = BPF_CGROUP_INET6_POST_BIND,
87 .result = LOAD_REJECT,
88 },
89 {
90 .descr = "sock_create load with invalid access: src_port",
91 .insns = {
92 BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
93 BPF_LDX_MEM(BPF_W, BPF_REG_7, BPF_REG_6,
94 offsetof(struct bpf_sock, src_port)),
95 BPF_MOV64_IMM(BPF_REG_0, 1),
96 BPF_EXIT_INSN(),
97 },
98 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
99 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
100 .result = LOAD_REJECT,
101 },
102 {
103 .descr = "sock_create load w/o expected_attach_type (compat mode)",
104 .insns = {
105 BPF_MOV64_IMM(BPF_REG_0, 1),
106 BPF_EXIT_INSN(),
107 },
108 .expected_attach_type = 0,
109 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
110 .domain = AF_INET,
111 .type = SOCK_STREAM,
112 .ip = "127.0.0.1",
113 .port = 8097,
114 .result = SUCCESS,
115 },
116 {
117 .descr = "sock_create load w/ expected_attach_type",
118 .insns = {
119 BPF_MOV64_IMM(BPF_REG_0, 1),
120 BPF_EXIT_INSN(),
121 },
122 .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE,
123 .attach_type = BPF_CGROUP_INET_SOCK_CREATE,
124 .domain = AF_INET,
125 .type = SOCK_STREAM,
126 .ip = "127.0.0.1",
127 .port = 8097,
128 .result = SUCCESS,
129 },
130 {33 {
131 .descr = "attach type mismatch bind4 vs bind6",34 .descr = "attach type mismatch bind4 vs bind6",
132 .insns = {35 .insns = {
@@ -374,40 +277,29 @@ static struct sock_test tests[] = { · 241 unmodified lines
374 },277 },
375};278};
376 279
377static size_t probe_prog_length(const struct bpf_insn *fp)280static int load_prog(const struct bpf_insn *insns,
378{281 enum bpf_attach_type expected_attach_type)
379 size_t len;
380
381 for (len = MAX_INSNS - 1; len > 0; --len)
382 if (fp[len].code != 0 || fp[len].imm != 0)
383 break;
384 return len + 1;
385}
386
387static int load_sock_prog(const struct bpf_insn *prog,
388 enum bpf_attach_type attach_type)
389{282{
390 LIBBPF_OPTS(bpf_prog_load_opts, opts);283 LIBBPF_OPTS(bpf_prog_load_opts, opts,
391 int ret, insn_cnt;284 .expected_attach_type = expected_attach_type,
392 285 .log_level = 2,
393 insn_cnt = probe_prog_length(prog);286 .log_buf = bpf_log_buf,
394 287 .log_size = sizeof(bpf_log_buf),
395 opts.expected_attach_type = attach_type;288 );
396 opts.log_buf = bpf_log_buf;289 int fd, insns_cnt = 0;
397 opts.log_size = BPF_LOG_BUF_SIZE;290
398 opts.log_level = 2;291 for (;
292 insns[insns_cnt].code != (BPF_JMP | BPF_EXIT);
293 insns_cnt++) {
294 }
295 insns_cnt++;
399 296
400 ret = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", prog, insn_cnt, &opts);297 fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns,
401 if (verbose && ret < 0)298 insns_cnt, &opts);
299 if (fd < 0)
402 fprintf(stderr, "%s\n", bpf_log_buf);300 fprintf(stderr, "%s\n", bpf_log_buf);
403 301
404 return ret;302 return fd;
405}
406
407static int attach_sock_prog(int cgfd, int progfd,
408 enum bpf_attach_type attach_type)
409{
410 return bpf_prog_attach(progfd, cgfd, attach_type, BPF_F_ALLOW_OVERRIDE);
411}303}
412 304
413static int bind_sock(int domain, int type, const char *ip,305static int bind_sock(int domain, int type, const char *ip,
@@ -477,22 +369,16 @@ out: · 63 unmodified lines
477 return res;369 return res;
478}370}
479 371
480static int run_test_case(int cgfd, const struct sock_test *test)372static int run_test(int cgroup_fd, struct sock_post_bind_test *test)
481{373{
482 int progfd = -1;374 int err, prog_fd, res, ret = 0;
483 int err = 0;
484 int res;
485
486 printf("Test case: %s .. ", test->descr);
487 progfd = load_sock_prog(test->insns, test->expected_attach_type);
488 if (progfd < 0) {
489 if (test->result == LOAD_REJECT)
490 goto out;
491 else
492 goto err;
493 }
494 375
495 if (attach_sock_prog(cgfd, progfd, test->attach_type) < 0) {376 prog_fd = load_prog(test->insns, test->expected_attach_type);
377 if (prog_fd < 0)
378 goto err;
379
380 err = bpf_prog_attach(prog_fd, cgroup_fd, test->attach_type, 0);
381 if (err < 0) {
496 if (test->result == ATTACH_REJECT)382 if (test->result == ATTACH_REJECT)
497 goto out;383 goto out;
498 else384 else
@@ -503,54 +389,38 @@ static int run_test_case(int cgfd, const struct sock_test *test) · 4 unmodified lines
503 test->port_retry);389 test->port_retry);
504 if (res > 0 && test->result == res)390 if (res > 0 && test->result == res)
505 goto out;391 goto out;
506
507err:392err:
508 err = -1;393 ret = -1;
509out:394out:
510 /* Detaching w/o checking return code: best effort attempt. */395 /* Detaching w/o checking return code: best effort attempt. */
511 if (progfd != -1)396 if (prog_fd != -1)
512 bpf_prog_detach(cgfd, test->attach_type);397 bpf_prog_detach(cgroup_fd, test->attach_type);
513 close(progfd);398 close(prog_fd);
514 printf("[%s]\n", err ? "FAIL" : "PASS");399 return ret;
515 return err;
516}400}
517 401
518static int run_tests(int cgfd)402void test_sock_post_bind(void)
519{403{
520 int passes = 0;404 struct netns_obj *ns;
521 int fails = 0;405 int cgroup_fd;
522 int i;406 int i;
523 407
524 for (i = 0; i < ARRAY_SIZE(tests); ++i) {408 cgroup_fd = test__join_cgroup("/post_bind");
525 if (run_test_case(cgfd, &tests[i]))409 if (!ASSERT_OK_FD(cgroup_fd, "join_cgroup"))
526 ++fails;410 return;
527 else
528 ++passes;
529 }
530 printf("Summary: %d PASSED, %d FAILED\n", passes, fails);
531 return fails ? -1 : 0;
532}
533
534int main(int argc, char **argv)
535{
536 int cgfd = -1;
537 int err = 0;
538 411
539 cgfd = cgroup_setup_and_join(CG_PATH);412 ns = netns_new(TEST_NS, true);
540 if (cgfd < 0)413 if (!ASSERT_OK_PTR(ns, "netns_new"))
541 goto err;414 goto cleanup;
542 415
543 /* Use libbpf 1.0 API mode */416 for (i = 0; i < ARRAY_SIZE(tests); i++) {
544 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);417 if (!test__start_subtest(tests[i].descr))
418 continue;
545 419
546 if (run_tests(cgfd))420 ASSERT_OK(run_test(cgroup_fd, &tests[i]), tests[i].descr);
547 goto err;421 }
548 422
549 goto out;423cleanup:
550err:424 netns_free(ns);
551 err = -1;425 close(cgroup_fd);
552out:
553 close(cgfd);
554 cleanup_cgroup_environment();
555 return err;
556}426}
tools/testing/selftests/bpf/prog_tests/struct_ops_private_stack.c+106−0
@@ -0,0 +1,106 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <test_progs.h>
4#include "struct_ops_private_stack.skel.h"
5#include "struct_ops_private_stack_fail.skel.h"
6#include "struct_ops_private_stack_recur.skel.h"
7
8static void test_private_stack(void)
9{
10 struct struct_ops_private_stack *skel;
11 struct bpf_link *link;
12 int err;
13
14 skel = struct_ops_private_stack__open();
15 if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack__open"))
16 return;
17
18 if (skel->data->skip) {
19 test__skip();
20 goto cleanup;
21 }
22
23 err = struct_ops_private_stack__load(skel);
24 if (!ASSERT_OK(err, "struct_ops_private_stack__load"))
25 goto cleanup;
26
27 link = bpf_map__attach_struct_ops(skel->maps.testmod_1);
28 if (!ASSERT_OK_PTR(link, "attach_struct_ops"))
29 goto cleanup;
30
31 ASSERT_OK(trigger_module_test_read(256), "trigger_read");
32
33 ASSERT_EQ(skel->bss->val_i, 3, "val_i");
34 ASSERT_EQ(skel->bss->val_j, 8, "val_j");
35
36 bpf_link__destroy(link);
37
38cleanup:
39 struct_ops_private_stack__destroy(skel);
40}
41
42static void test_private_stack_fail(void)
43{
44 struct struct_ops_private_stack_fail *skel;
45 int err;
46
47 skel = struct_ops_private_stack_fail__open();
48 if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_fail__open"))
49 return;
50
51 if (skel->data->skip) {
52 test__skip();
53 goto cleanup;
54 }
55
56 err = struct_ops_private_stack_fail__load(skel);
57 if (!ASSERT_ERR(err, "struct_ops_private_stack_fail__load"))
58 goto cleanup;
59 return;
60
61cleanup:
62 struct_ops_private_stack_fail__destroy(skel);
63}
64
65static void test_private_stack_recur(void)
66{
67 struct struct_ops_private_stack_recur *skel;
68 struct bpf_link *link;
69 int err;
70
71 skel = struct_ops_private_stack_recur__open();
72 if (!ASSERT_OK_PTR(skel, "struct_ops_private_stack_recur__open"))
73 return;
74
75 if (skel->data->skip) {
76 test__skip();
77 goto cleanup;
78 }
79
80 err = struct_ops_private_stack_recur__load(skel);
81 if (!ASSERT_OK(err, "struct_ops_private_stack_recur__load"))
82 goto cleanup;
83
84 link = bpf_map__attach_struct_ops(skel->maps.testmod_1);
85 if (!ASSERT_OK_PTR(link, "attach_struct_ops"))
86 goto cleanup;
87
88 ASSERT_OK(trigger_module_test_read(256), "trigger_read");
89
90 ASSERT_EQ(skel->bss->val_j, 3, "val_j");
91
92 bpf_link__destroy(link);
93
94cleanup:
95 struct_ops_private_stack_recur__destroy(skel);
96}
97
98void test_struct_ops_private_stack(void)
99{
100 if (test__start_subtest("private_stack"))
101 test_private_stack();
102 if (test__start_subtest("private_stack_fail"))
103 test_private_stack_fail();
104 if (test__start_subtest("private_stack_recur"))
105 test_private_stack_recur();
106}
tools/testing/selftests/bpf/prog_tests/subskeleton.c+75−1
@@ -46,7 +46,8 @@ static int subskeleton_lib_subresult(struct bpf_object *obj) · 45 unmodified lines
46 return result;46 return result;
47}47}
48 48
49void test_subskeleton(void)49/* initialize and load through skeleton, then instantiate subskeleton out of it */
50static void subtest_skel_subskeleton(void)
50{51{
51 int err, result;52 int err, result;
52 struct test_subskeleton *skel;53 struct test_subskeleton *skel;
@@ -76,3 +77,76 @@ void test_subskeleton(void) · 23 unmodified lines
76cleanup:77cleanup:
77 test_subskeleton__destroy(skel);78 test_subskeleton__destroy(skel);
78}79}
80
81/* initialize and load through generic bpf_object API, then instantiate subskeleton out of it */
82static void subtest_obj_subskeleton(void)
83{
84 int err, result;
85 const void *elf_bytes;
86 size_t elf_bytes_sz = 0, rodata_sz = 0, bss_sz = 0;
87 struct bpf_object *obj;
88 const struct bpf_map *map;
89 const struct bpf_program *prog;
90 struct bpf_link *link = NULL;
91 struct test_subskeleton__rodata *rodata;
92 struct test_subskeleton__bss *bss;
93
94 elf_bytes = test_subskeleton__elf_bytes(&elf_bytes_sz);
95 if (!ASSERT_OK_PTR(elf_bytes, "elf_bytes"))
96 return;
97
98 obj = bpf_object__open_mem(elf_bytes, elf_bytes_sz, NULL);
99 if (!ASSERT_OK_PTR(obj, "obj_open_mem"))
100 return;
101
102 map = bpf_object__find_map_by_name(obj, ".rodata");
103 if (!ASSERT_OK_PTR(map, "rodata_map_by_name"))
104 goto cleanup;
105
106 rodata = bpf_map__initial_value(map, &rodata_sz);
107 if (!ASSERT_OK_PTR(rodata, "rodata_get"))
108 goto cleanup;
109
110 rodata->rovar1 = 10;
111 rodata->var1 = 1;
112 subskeleton_lib_setup(obj);
113
114 err = bpf_object__load(obj);
115 if (!ASSERT_OK(err, "obj_load"))
116 goto cleanup;
117
118 prog = bpf_object__find_program_by_name(obj, "handler1");
119 if (!ASSERT_OK_PTR(prog, "prog_by_name"))
120 goto cleanup;
121
122 link = bpf_program__attach(prog);
123 if (!ASSERT_OK_PTR(link, "prog_attach"))
124 goto cleanup;
125
126 /* trigger tracepoint */
127 usleep(1);
128
129 map = bpf_object__find_map_by_name(obj, ".bss");
130 if (!ASSERT_OK_PTR(map, "bss_map_by_name"))
131 goto cleanup;
132
133 bss = bpf_map__initial_value(map, &bss_sz);
134 if (!ASSERT_OK_PTR(rodata, "rodata_get"))
135 goto cleanup;
136
137 result = subskeleton_lib_subresult(obj) * 10;
138 ASSERT_EQ(bss->out1, result, "out1");
139
140cleanup:
141 bpf_link__destroy(link);
142 bpf_object__close(obj);
143}
144
145
146void test_subskeleton(void)
147{
148 if (test__start_subtest("skel_subskel"))
149 subtest_skel_subskeleton();
150 if (test__start_subtest("obj_subskel"))
151 subtest_obj_subskeleton();
152}
tools/testing/selftests/bpf/prog_tests/tailcalls.c+114−14
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include "tailcall_bpf2bpf_hierarchy3.skel.h"7#include "tailcall_bpf2bpf_hierarchy3.skel.h"
8#include "tailcall_freplace.skel.h"8#include "tailcall_freplace.skel.h"
9#include "tc_bpf2bpf.skel.h"9#include "tc_bpf2bpf.skel.h"
10#include "tailcall_fail.skel.h"
10 11
11/* test_tailcall_1 checks basic functionality by patching multiple locations12/* test_tailcall_1 checks basic functionality by patching multiple locations
12 * in a single program for a single tail call slot with nop->jmp, jmp->nop13 * in a single program for a single tail call slot with nop->jmp, jmp->nop
@@ -1496,8 +1497,8 @@ static void test_tailcall_bpf2bpf_hierarchy_3(void) · 1483 unmodified lines
1496 RUN_TESTS(tailcall_bpf2bpf_hierarchy3);1497 RUN_TESTS(tailcall_bpf2bpf_hierarchy3);
1497}1498}
1498 1499
1499/* test_tailcall_freplace checks that the attached freplace prog is OK to1500/* test_tailcall_freplace checks that the freplace prog fails to update the
1500 * update the prog_array map.1501 * prog_array map, no matter whether the freplace prog attaches to its target.
1501 */1502 */
1502static void test_tailcall_freplace(void)1503static void test_tailcall_freplace(void)
1503{1504{
@@ -1505,7 +1506,7 @@ static void test_tailcall_freplace(void) · 1 unmodified line
1505 struct bpf_link *freplace_link = NULL;1506 struct bpf_link *freplace_link = NULL;
1506 struct bpf_program *freplace_prog;1507 struct bpf_program *freplace_prog;
1507 struct tc_bpf2bpf *tc_skel = NULL;1508 struct tc_bpf2bpf *tc_skel = NULL;
1508 int prog_fd, map_fd;1509 int prog_fd, tc_prog_fd, map_fd;
1509 char buff[128] = {};1510 char buff[128] = {};
1510 int err, key;1511 int err, key;
1511 1512
@@ -1523,9 +1524,10 @@ static void test_tailcall_freplace(void) · 11 unmodified lines
1523 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))1524 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))
1524 goto out;1525 goto out;
1525 1526
1526 prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);1527 tc_prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
1527 freplace_prog = freplace_skel->progs.entry_freplace;1528 freplace_prog = freplace_skel->progs.entry_freplace;
1528 err = bpf_program__set_attach_target(freplace_prog, prog_fd, "subprog");1529 err = bpf_program__set_attach_target(freplace_prog, tc_prog_fd,
1530 "subprog_tc");
1529 if (!ASSERT_OK(err, "set_attach_target"))1531 if (!ASSERT_OK(err, "set_attach_target"))
1530 goto out;1532 goto out;
1531 1533
@@ -1533,27 +1535,121 @@ static void test_tailcall_freplace(void) · 1 unmodified line
1533 if (!ASSERT_OK(err, "tailcall_freplace__load"))1535 if (!ASSERT_OK(err, "tailcall_freplace__load"))
1534 goto out;1536 goto out;
1535 1537
1536 freplace_link = bpf_program__attach_freplace(freplace_prog, prog_fd,1538 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);
1537 "subprog");1539 prog_fd = bpf_program__fd(freplace_prog);
1540 key = 0;
1541 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1542 ASSERT_ERR(err, "update jmp_table failure");
1543
1544 freplace_link = bpf_program__attach_freplace(freplace_prog, tc_prog_fd,
1545 "subprog_tc");
1538 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))1546 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1539 goto out;1547 goto out;
1540 1548
1541 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);1549 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1542 prog_fd = bpf_program__fd(freplace_prog);1550 ASSERT_ERR(err, "update jmp_table failure");
1551
1552out:
1553 bpf_link__destroy(freplace_link);
1554 tailcall_freplace__destroy(freplace_skel);
1555 tc_bpf2bpf__destroy(tc_skel);
1556}
1557
1558/* test_tailcall_bpf2bpf_freplace checks the failure that fails to attach a tail
1559 * callee prog with freplace prog or fails to update an extended prog to
1560 * prog_array map.
1561 */
1562static void test_tailcall_bpf2bpf_freplace(void)
1563{
1564 struct tailcall_freplace *freplace_skel = NULL;
1565 struct bpf_link *freplace_link = NULL;
1566 struct tc_bpf2bpf *tc_skel = NULL;
1567 char buff[128] = {};
1568 int prog_fd, map_fd;
1569 int err, key;
1570
1571 LIBBPF_OPTS(bpf_test_run_opts, topts,
1572 .data_in = buff,
1573 .data_size_in = sizeof(buff),
1574 .repeat = 1,
1575 );
1576
1577 tc_skel = tc_bpf2bpf__open_and_load();
1578 if (!ASSERT_OK_PTR(tc_skel, "tc_bpf2bpf__open_and_load"))
1579 goto out;
1580
1581 prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);
1582 freplace_skel = tailcall_freplace__open();
1583 if (!ASSERT_OK_PTR(freplace_skel, "tailcall_freplace__open"))
1584 goto out;
1585
1586 err = bpf_program__set_attach_target(freplace_skel->progs.entry_freplace,
1587 prog_fd, "subprog_tc");
1588 if (!ASSERT_OK(err, "set_attach_target"))
1589 goto out;
1590
1591 err = tailcall_freplace__load(freplace_skel);
1592 if (!ASSERT_OK(err, "tailcall_freplace__load"))
1593 goto out;
1594
1595 /* OK to attach then detach freplace prog. */
1596
1597 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1598 prog_fd, "subprog_tc");
1599 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1600 goto out;
1601
1602 err = bpf_link__destroy(freplace_link);
1603 if (!ASSERT_OK(err, "destroy link"))
1604 goto out;
1605
1606 /* OK to update prog_array map then delete element from the map. */
1607
1543 key = 0;1608 key = 0;
1609 map_fd = bpf_map__fd(freplace_skel->maps.jmp_table);
1544 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);1610 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1545 if (!ASSERT_OK(err, "update jmp_table"))1611 if (!ASSERT_OK(err, "update jmp_table"))
1546 goto out;1612 goto out;
1547 1613
1548 prog_fd = bpf_program__fd(tc_skel->progs.entry_tc);1614 err = bpf_map_delete_elem(map_fd, &key);
1549 err = bpf_prog_test_run_opts(prog_fd, &topts);1615 if (!ASSERT_OK(err, "delete_elem from jmp_table"))
1550 ASSERT_OK(err, "test_run");1616 goto out;
1551 ASSERT_EQ(topts.retval, 34, "test_run retval");1617
1618 /* Fail to attach a tail callee prog with freplace prog. */
1619
1620 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1621 if (!ASSERT_OK(err, "update jmp_table"))
1622 goto out;
1623
1624 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1625 prog_fd, "subprog_tc");
1626 if (!ASSERT_ERR_PTR(freplace_link, "attach_freplace failure"))
1627 goto out;
1628
1629 err = bpf_map_delete_elem(map_fd, &key);
1630 if (!ASSERT_OK(err, "delete_elem from jmp_table"))
1631 goto out;
1632
1633 /* Fail to update an extended prog to prog_array map. */
1634
1635 freplace_link = bpf_program__attach_freplace(freplace_skel->progs.entry_freplace,
1636 prog_fd, "subprog_tc");
1637 if (!ASSERT_OK_PTR(freplace_link, "attach_freplace"))
1638 goto out;
1639
1640 err = bpf_map_update_elem(map_fd, &key, &prog_fd, BPF_ANY);
1641 if (!ASSERT_ERR(err, "update jmp_table failure"))
1642 goto out;
1552 1643
1553out:1644out:
1554 bpf_link__destroy(freplace_link);1645 bpf_link__destroy(freplace_link);
1555 tc_bpf2bpf__destroy(tc_skel);
1556 tailcall_freplace__destroy(freplace_skel);1646 tailcall_freplace__destroy(freplace_skel);
1647 tc_bpf2bpf__destroy(tc_skel);
1648}
1649
1650static void test_tailcall_failure()
1651{
1652 RUN_TESTS(tailcall_fail);
1557}1653}
1558 1654
1559void test_tailcalls(void)1655void test_tailcalls(void)
@@ -1606,4 +1702,8 @@ void test_tailcalls(void) · 46 unmodified lines
1606 test_tailcall_bpf2bpf_hierarchy_3();1702 test_tailcall_bpf2bpf_hierarchy_3();
1607 if (test__start_subtest("tailcall_freplace"))1703 if (test__start_subtest("tailcall_freplace"))
1608 test_tailcall_freplace();1704 test_tailcall_freplace();
1705 if (test__start_subtest("tailcall_bpf2bpf_freplace"))
1706 test_tailcall_bpf2bpf_freplace();
1707 if (test__start_subtest("tailcall_failure"))
1708 test_tailcall_failure();
1609}1709}
tools/testing/selftests/bpf/prog_tests/task_kfunc.c+80−0
@@ -68,6 +68,74 @@ cleanup: · 67 unmodified lines
68 task_kfunc_success__destroy(skel);68 task_kfunc_success__destroy(skel);
69}69}
70 70
71static int run_vpid_test(void *prog_name)
72{
73 struct task_kfunc_success *skel;
74 struct bpf_program *prog;
75 int prog_fd, err = 0;
76
77 if (getpid() != 1)
78 return 1;
79
80 skel = open_load_task_kfunc_skel();
81 if (!skel)
82 return 2;
83
84 if (skel->bss->err) {
85 err = 3;
86 goto cleanup;
87 }
88
89 prog = bpf_object__find_program_by_name(skel->obj, prog_name);
90 if (!prog) {
91 err = 4;
92 goto cleanup;
93 }
94
95 prog_fd = bpf_program__fd(prog);
96 if (prog_fd < 0) {
97 err = 5;
98 goto cleanup;
99 }
100
101 if (bpf_prog_test_run_opts(prog_fd, NULL)) {
102 err = 6;
103 goto cleanup;
104 }
105
106 if (skel->bss->err)
107 err = 7 + skel->bss->err;
108cleanup:
109 task_kfunc_success__destroy(skel);
110 return err;
111}
112
113static void run_vpid_success_test(const char *prog_name)
114{
115 const int stack_size = 1024 * 1024;
116 int child_pid, wstatus;
117 char *stack;
118
119 stack = (char *)malloc(stack_size);
120 if (!ASSERT_OK_PTR(stack, "clone_stack"))
121 return;
122
123 child_pid = clone(run_vpid_test, stack + stack_size,
124 CLONE_NEWPID | SIGCHLD, (void *)prog_name);
125 if (!ASSERT_GT(child_pid, -1, "child_pid"))
126 goto cleanup;
127
128 if (!ASSERT_GT(waitpid(child_pid, &wstatus, 0), -1, "waitpid"))
129 goto cleanup;
130
131 if (WEXITSTATUS(wstatus) > 7)
132 ASSERT_OK(WEXITSTATUS(wstatus) - 7, "vpid_test_failure");
133 else
134 ASSERT_OK(WEXITSTATUS(wstatus), "run_vpid_test_err");
135cleanup:
136 free(stack);
137}
138
71static const char * const success_tests[] = {139static const char * const success_tests[] = {
72 "test_task_acquire_release_argument",140 "test_task_acquire_release_argument",
73 "test_task_acquire_release_current",141 "test_task_acquire_release_current",
@@ -83,6 +151,11 @@ static const char * const success_tests[] = { · 9 unmodified lines
83 "test_task_kfunc_flavor_relo_not_found",151 "test_task_kfunc_flavor_relo_not_found",
84};152};
85 153
154static const char * const vpid_success_tests[] = {
155 "test_task_from_vpid_current",
156 "test_task_from_vpid_invalid",
157};
158
86void test_task_kfunc(void)159void test_task_kfunc(void)
87{160{
88 int i;161 int i;
@@ -94,5 +167,12 @@ void test_task_kfunc(void) · 5 unmodified lines
94 run_success_test(success_tests[i]);167 run_success_test(success_tests[i]);
95 }168 }
96 169
170 for (i = 0; i < ARRAY_SIZE(vpid_success_tests); i++) {
171 if (!test__start_subtest(vpid_success_tests[i]))
172 continue;
173
174 run_vpid_success_test(vpid_success_tests[i]);
175 }
176
97 RUN_TESTS(task_kfunc_failure);177 RUN_TESTS(task_kfunc_failure);
98}178}
tools/testing/selftests/bpf/prog_tests/task_local_storage.c+282−4
@@ -7,12 +7,20 @@ · 6 unmodified lines
7#include <pthread.h>7#include <pthread.h>
8#include <sys/syscall.h> /* For SYS_xxx definitions */8#include <sys/syscall.h> /* For SYS_xxx definitions */
9#include <sys/types.h>9#include <sys/types.h>
10#include <sys/eventfd.h>
11#include <sys/mman.h>
10#include <test_progs.h>12#include <test_progs.h>
13#include <bpf/btf.h>
11#include "task_local_storage_helpers.h"14#include "task_local_storage_helpers.h"
12#include "task_local_storage.skel.h"15#include "task_local_storage.skel.h"
13#include "task_local_storage_exit_creds.skel.h"16#include "task_local_storage_exit_creds.skel.h"
14#include "task_ls_recursion.skel.h"17#include "task_ls_recursion.skel.h"
15#include "task_storage_nodeadlock.skel.h"18#include "task_storage_nodeadlock.skel.h"
19#include "uptr_test_common.h"
20#include "task_ls_uptr.skel.h"
21#include "uptr_update_failure.skel.h"
22#include "uptr_failure.skel.h"
23#include "uptr_map_failure.skel.h"
16 24
17static void test_sys_enter_exit(void)25static void test_sys_enter_exit(void)
18{26{
@@ -23,14 +31,14 @@ static void test_sys_enter_exit(void) · 4 unmodified lines
23 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))31 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
24 return;32 return;
25 33
26 skel->bss->target_pid = syscall(SYS_gettid);34 skel->bss->target_pid = sys_gettid();
27 35
28 err = task_local_storage__attach(skel);36 err = task_local_storage__attach(skel);
29 if (!ASSERT_OK(err, "skel_attach"))37 if (!ASSERT_OK(err, "skel_attach"))
30 goto out;38 goto out;
31 39
32 syscall(SYS_gettid);40 sys_gettid();
33 syscall(SYS_gettid);41 sys_gettid();
34 42
35 /* 3x syscalls: 1x attach and 2x gettid */43 /* 3x syscalls: 1x attach and 2x gettid */
36 ASSERT_EQ(skel->bss->enter_cnt, 3, "enter_cnt");44 ASSERT_EQ(skel->bss->enter_cnt, 3, "enter_cnt");
@@ -99,7 +107,7 @@ static void test_recursion(void) · 62 unmodified lines
99 107
100 /* trigger sys_enter, make sure it does not cause deadlock */108 /* trigger sys_enter, make sure it does not cause deadlock */
101 skel->bss->test_pid = getpid();109 skel->bss->test_pid = getpid();
102 syscall(SYS_gettid);110 sys_gettid();
103 skel->bss->test_pid = 0;111 skel->bss->test_pid = 0;
104 task_ls_recursion__detach(skel);112 task_ls_recursion__detach(skel);
105 113
@@ -227,6 +235,259 @@ done: · 121 unmodified lines
227 sched_setaffinity(getpid(), sizeof(old), &old);235 sched_setaffinity(getpid(), sizeof(old), &old);
228}236}
229 237
238static struct user_data udata __attribute__((aligned(16))) = {
239 .a = 1,
240 .b = 2,
241};
242
243static struct user_data udata2 __attribute__((aligned(16))) = {
244 .a = 3,
245 .b = 4,
246};
247
248static void check_udata2(int expected)
249{
250 udata2.result = udata2.nested_result = 0;
251 usleep(1);
252 ASSERT_EQ(udata2.result, expected, "udata2.result");
253 ASSERT_EQ(udata2.nested_result, expected, "udata2.nested_result");
254}
255
256static void test_uptr_basic(void)
257{
258 int map_fd, parent_task_fd, ev_fd;
259 struct value_type value = {};
260 struct task_ls_uptr *skel;
261 pid_t child_pid, my_tid;
262 __u64 ev_dummy_data = 1;
263 int err;
264
265 my_tid = sys_gettid();
266 parent_task_fd = sys_pidfd_open(my_tid, 0);
267 if (!ASSERT_OK_FD(parent_task_fd, "parent_task_fd"))
268 return;
269
270 ev_fd = eventfd(0, 0);
271 if (!ASSERT_OK_FD(ev_fd, "ev_fd")) {
272 close(parent_task_fd);
273 return;
274 }
275
276 skel = task_ls_uptr__open_and_load();
277 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
278 goto out;
279
280 map_fd = bpf_map__fd(skel->maps.datamap);
281 value.udata = &udata;
282 value.nested.udata = &udata;
283 err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_NOEXIST);
284 if (!ASSERT_OK(err, "update_elem(udata)"))
285 goto out;
286
287 err = task_ls_uptr__attach(skel);
288 if (!ASSERT_OK(err, "skel_attach"))
289 goto out;
290
291 child_pid = fork();
292 if (!ASSERT_NEQ(child_pid, -1, "fork"))
293 goto out;
294
295 /* Call syscall in the child process, but access the map value of
296 * the parent process in the BPF program to check if the user kptr
297 * is translated/mapped correctly.
298 */
299 if (child_pid == 0) {
300 /* child */
301
302 /* Overwrite the user_data in the child process to check if
303 * the BPF program accesses the user_data of the parent.
304 */
305 udata.a = 0;
306 udata.b = 0;
307
308 /* Wait for the parent to set child_pid */
309 read(ev_fd, &ev_dummy_data, sizeof(ev_dummy_data));
310 exit(0);
311 }
312
313 skel->bss->parent_pid = my_tid;
314 skel->bss->target_pid = child_pid;
315
316 write(ev_fd, &ev_dummy_data, sizeof(ev_dummy_data));
317
318 err = waitpid(child_pid, NULL, 0);
319 ASSERT_EQ(err, child_pid, "waitpid");
320 ASSERT_EQ(udata.result, MAGIC_VALUE + udata.a + udata.b, "udata.result");
321 ASSERT_EQ(udata.nested_result, MAGIC_VALUE + udata.a + udata.b, "udata.nested_result");
322
323 skel->bss->target_pid = my_tid;
324
325 /* update_elem: uptr changes from udata1 to udata2 */
326 value.udata = &udata2;
327 value.nested.udata = &udata2;
328 err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_EXIST);
329 if (!ASSERT_OK(err, "update_elem(udata2)"))
330 goto out;
331 check_udata2(MAGIC_VALUE + udata2.a + udata2.b);
332
333 /* update_elem: uptr changes from udata2 uptr to NULL */
334 memset(&value, 0, sizeof(value));
335 err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_EXIST);
336 if (!ASSERT_OK(err, "update_elem(udata2)"))
337 goto out;
338 check_udata2(0);
339
340 /* update_elem: uptr changes from NULL to udata2 */
341 value.udata = &udata2;
342 value.nested.udata = &udata2;
343 err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_EXIST);
344 if (!ASSERT_OK(err, "update_elem(udata2)"))
345 goto out;
346 check_udata2(MAGIC_VALUE + udata2.a + udata2.b);
347
348 /* Check if user programs can access the value of user kptrs
349 * through bpf_map_lookup_elem(). Make sure the kernel value is not
350 * leaked.
351 */
352 err = bpf_map_lookup_elem(map_fd, &parent_task_fd, &value);
353 if (!ASSERT_OK(err, "bpf_map_lookup_elem"))
354 goto out;
355 ASSERT_EQ(value.udata, NULL, "value.udata");
356 ASSERT_EQ(value.nested.udata, NULL, "value.nested.udata");
357
358 /* delete_elem */
359 err = bpf_map_delete_elem(map_fd, &parent_task_fd);
360 ASSERT_OK(err, "delete_elem(udata2)");
361 check_udata2(0);
362
363 /* update_elem: add uptr back to test map_free */
364 value.udata = &udata2;
365 value.nested.udata = &udata2;
366 err = bpf_map_update_elem(map_fd, &parent_task_fd, &value, BPF_NOEXIST);
367 ASSERT_OK(err, "update_elem(udata2)");
368
369out:
370 task_ls_uptr__destroy(skel);
371 close(ev_fd);
372 close(parent_task_fd);
373}
374
375static void test_uptr_across_pages(void)
376{
377 int page_size = getpagesize();
378 struct value_type value = {};
379 struct task_ls_uptr *skel;
380 int err, task_fd, map_fd;
381 void *mem;
382
383 task_fd = sys_pidfd_open(getpid(), 0);
384 if (!ASSERT_OK_FD(task_fd, "task_fd"))
385 return;
386
387 mem = mmap(NULL, page_size * 2, PROT_READ | PROT_WRITE,
388 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
389 if (!ASSERT_OK_PTR(mem, "mmap(page_size * 2)")) {
390 close(task_fd);
391 return;
392 }
393
394 skel = task_ls_uptr__open_and_load();
395 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
396 goto out;
397
398 map_fd = bpf_map__fd(skel->maps.datamap);
399 value.udata = mem + page_size - offsetof(struct user_data, b);
400 err = bpf_map_update_elem(map_fd, &task_fd, &value, 0);
401 if (!ASSERT_ERR(err, "update_elem(udata)"))
402 goto out;
403 ASSERT_EQ(errno, EOPNOTSUPP, "errno");
404
405 value.udata = mem + page_size - sizeof(struct user_data);
406 err = bpf_map_update_elem(map_fd, &task_fd, &value, 0);
407 ASSERT_OK(err, "update_elem(udata)");
408
409out:
410 task_ls_uptr__destroy(skel);
411 close(task_fd);
412 munmap(mem, page_size * 2);
413}
414
415static void test_uptr_update_failure(void)
416{
417 struct value_lock_type value = {};
418 struct uptr_update_failure *skel;
419 int err, task_fd, map_fd;
420
421 task_fd = sys_pidfd_open(getpid(), 0);
422 if (!ASSERT_OK_FD(task_fd, "task_fd"))
423 return;
424
425 skel = uptr_update_failure__open_and_load();
426 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
427 goto out;
428
429 map_fd = bpf_map__fd(skel->maps.datamap);
430
431 value.udata = &udata;
432 err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_F_LOCK);
433 if (!ASSERT_ERR(err, "update_elem(udata, BPF_F_LOCK)"))
434 goto out;
435 ASSERT_EQ(errno, EOPNOTSUPP, "errno");
436
437 err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_EXIST);
438 if (!ASSERT_ERR(err, "update_elem(udata, BPF_EXIST)"))
439 goto out;
440 ASSERT_EQ(errno, ENOENT, "errno");
441
442 err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_NOEXIST);
443 if (!ASSERT_OK(err, "update_elem(udata, BPF_NOEXIST)"))
444 goto out;
445
446 value.udata = &udata2;
447 err = bpf_map_update_elem(map_fd, &task_fd, &value, BPF_NOEXIST);
448 if (!ASSERT_ERR(err, "update_elem(udata2, BPF_NOEXIST)"))
449 goto out;
450 ASSERT_EQ(errno, EEXIST, "errno");
451
452out:
453 uptr_update_failure__destroy(skel);
454 close(task_fd);
455}
456
457static void test_uptr_map_failure(const char *map_name, int expected_errno)
458{
459 LIBBPF_OPTS(bpf_map_create_opts, create_attr);
460 struct uptr_map_failure *skel;
461 struct bpf_map *map;
462 struct btf *btf;
463 int map_fd, err;
464
465 skel = uptr_map_failure__open();
466 if (!ASSERT_OK_PTR(skel, "uptr_map_failure__open"))
467 return;
468
469 map = bpf_object__find_map_by_name(skel->obj, map_name);
470 btf = bpf_object__btf(skel->obj);
471 err = btf__load_into_kernel(btf);
472 if (!ASSERT_OK(err, "btf__load_into_kernel"))
473 goto done;
474
475 create_attr.map_flags = bpf_map__map_flags(map);
476 create_attr.btf_fd = btf__fd(btf);
477 create_attr.btf_key_type_id = bpf_map__btf_key_type_id(map);
478 create_attr.btf_value_type_id = bpf_map__btf_value_type_id(map);
479 map_fd = bpf_map_create(bpf_map__type(map), map_name,
480 bpf_map__key_size(map), bpf_map__value_size(map),
481 0, &create_attr);
482 if (ASSERT_ERR_FD(map_fd, "map_create"))
483 ASSERT_EQ(errno, expected_errno, "errno");
484 else
485 close(map_fd);
486
487done:
488 uptr_map_failure__destroy(skel);
489}
490
230void test_task_local_storage(void)491void test_task_local_storage(void)
231{492{
232 if (test__start_subtest("sys_enter_exit"))493 if (test__start_subtest("sys_enter_exit"))
@@ -237,4 +498,21 @@ void test_task_local_storage(void) · 4 unmodified lines
237 test_recursion();498 test_recursion();
238 if (test__start_subtest("nodeadlock"))499 if (test__start_subtest("nodeadlock"))
239 test_nodeadlock();500 test_nodeadlock();
501 if (test__start_subtest("uptr_basic"))
502 test_uptr_basic();
503 if (test__start_subtest("uptr_across_pages"))
504 test_uptr_across_pages();
505 if (test__start_subtest("uptr_update_failure"))
506 test_uptr_update_failure();
507 if (test__start_subtest("uptr_map_failure_e2big")) {
508 if (getpagesize() == PAGE_SIZE)
509 test_uptr_map_failure("large_uptr_map", E2BIG);
510 else
511 test__skip();
512 }
513 if (test__start_subtest("uptr_map_failure_size0"))
514 test_uptr_map_failure("empty_uptr_map", EINVAL);
515 if (test__start_subtest("uptr_map_failure_kstruct"))
516 test_uptr_map_failure("kstruct_uptr_map", EINVAL);
517 RUN_TESTS(uptr_failure);
240}518}
tools/testing/selftests/bpf/prog_tests/timer_lockup.c+6−0
@@ -5,6 +5,7 @@ · 4 unmodified lines
5#include <test_progs.h>5#include <test_progs.h>
6#include <pthread.h>6#include <pthread.h>
7#include <network_helpers.h>7#include <network_helpers.h>
8#include <sys/sysinfo.h>
8 9
9#include "timer_lockup.skel.h"10#include "timer_lockup.skel.h"
10 11
@@ -52,6 +53,11 @@ void test_timer_lockup(void) · 41 unmodified lines
52 pthread_t thrds[2];53 pthread_t thrds[2];
53 void *ret;54 void *ret;
54 55
56 if (get_nprocs() < 2) {
57 test__skip();
58 return;
59 }
60
55 skel = timer_lockup__open_and_load();61 skel = timer_lockup__open_and_load();
56 if (!ASSERT_OK_PTR(skel, "timer_lockup__open_and_load"))62 if (!ASSERT_OK_PTR(skel, "timer_lockup__open_and_load"))
57 return;63 return;
tools/testing/selftests/bpf/prog_tests/token.c+12−7
@@ -828,8 +828,12 @@ static int userns_obj_priv_btf_success(int mnt_fd, struct token_lsm *lsm_skel) · 827 unmodified lines
828 return validate_struct_ops_load(mnt_fd, true /* should succeed */);828 return validate_struct_ops_load(mnt_fd, true /* should succeed */);
829}829}
830 830
831static const char *token_bpffs_custom_dir()
832{
833 return getenv("BPF_SELFTESTS_BPF_TOKEN_DIR") ?: "/tmp/bpf-token-fs";
834}
835
831#define TOKEN_ENVVAR "LIBBPF_BPF_TOKEN_PATH"836#define TOKEN_ENVVAR "LIBBPF_BPF_TOKEN_PATH"
832#define TOKEN_BPFFS_CUSTOM "/bpf-token-fs"
833 837
834static int userns_obj_priv_implicit_token(int mnt_fd, struct token_lsm *lsm_skel)838static int userns_obj_priv_implicit_token(int mnt_fd, struct token_lsm *lsm_skel)
835{839{
@@ -892,6 +896,7 @@ static int userns_obj_priv_implicit_token(int mnt_fd, struct token_lsm *lsm_skel · 56 unmodified lines
892 896
893static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *lsm_skel)897static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *lsm_skel)
894{898{
899 const char *custom_dir = token_bpffs_custom_dir();
895 LIBBPF_OPTS(bpf_object_open_opts, opts);900 LIBBPF_OPTS(bpf_object_open_opts, opts);
896 struct dummy_st_ops_success *skel;901 struct dummy_st_ops_success *skel;
897 int err;902 int err;
@@ -909,10 +914,10 @@ static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *l · 11 unmodified lines
909 * BPF token implicitly, unless pointed to it through914 * BPF token implicitly, unless pointed to it through
910 * LIBBPF_BPF_TOKEN_PATH envvar915 * LIBBPF_BPF_TOKEN_PATH envvar
911 */916 */
912 rmdir(TOKEN_BPFFS_CUSTOM);917 rmdir(custom_dir);
913 if (!ASSERT_OK(mkdir(TOKEN_BPFFS_CUSTOM, 0777), "mkdir_bpffs_custom"))918 if (!ASSERT_OK(mkdir(custom_dir, 0777), "mkdir_bpffs_custom"))
914 goto err_out;919 goto err_out;
915 err = sys_move_mount(mnt_fd, "", AT_FDCWD, TOKEN_BPFFS_CUSTOM, MOVE_MOUNT_F_EMPTY_PATH);920 err = sys_move_mount(mnt_fd, "", AT_FDCWD, custom_dir, MOVE_MOUNT_F_EMPTY_PATH);
916 if (!ASSERT_OK(err, "move_mount_bpffs"))921 if (!ASSERT_OK(err, "move_mount_bpffs"))
917 goto err_out;922 goto err_out;
918 923
@@ -925,7 +930,7 @@ static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *l · 6 unmodified lines
925 goto err_out;930 goto err_out;
926 }931 }
927 932
928 err = setenv(TOKEN_ENVVAR, TOKEN_BPFFS_CUSTOM, 1 /*overwrite*/);933 err = setenv(TOKEN_ENVVAR, custom_dir, 1 /*overwrite*/);
929 if (!ASSERT_OK(err, "setenv_token_path"))934 if (!ASSERT_OK(err, "setenv_token_path"))
930 goto err_out;935 goto err_out;
931 936
@@ -951,11 +956,11 @@ static int userns_obj_priv_implicit_token_envvar(int mnt_fd, struct token_lsm *l · 19 unmodified lines
951 if (!ASSERT_ERR(err, "obj_empty_token_path_load"))956 if (!ASSERT_ERR(err, "obj_empty_token_path_load"))
952 goto err_out;957 goto err_out;
953 958
954 rmdir(TOKEN_BPFFS_CUSTOM);959 rmdir(custom_dir);
955 unsetenv(TOKEN_ENVVAR);960 unsetenv(TOKEN_ENVVAR);
956 return 0;961 return 0;
957err_out:962err_out:
958 rmdir(TOKEN_BPFFS_CUSTOM);963 rmdir(custom_dir);
959 unsetenv(TOKEN_ENVVAR);964 unsetenv(TOKEN_ENVVAR);
960 return -EINVAL;965 return -EINVAL;
961}966}
tools/testing/selftests/bpf/prog_tests/uprobe_multi_test.c+311−50
@@ -8,6 +8,11 @@ · 7 unmodified lines
8#include "uprobe_multi_usdt.skel.h"8#include "uprobe_multi_usdt.skel.h"
9#include "uprobe_multi_consumers.skel.h"9#include "uprobe_multi_consumers.skel.h"
10#include "uprobe_multi_pid_filter.skel.h"10#include "uprobe_multi_pid_filter.skel.h"
11#include "uprobe_multi_session.skel.h"
12#include "uprobe_multi_session_single.skel.h"
13#include "uprobe_multi_session_cookie.skel.h"
14#include "uprobe_multi_session_recursive.skel.h"
15#include "uprobe_multi_verifier.skel.h"
11#include "bpf/libbpf_internal.h"16#include "bpf/libbpf_internal.h"
12#include "testing_helpers.h"17#include "testing_helpers.h"
13#include "../sdt.h"18#include "../sdt.h"
@@ -34,6 +39,12 @@ noinline void usdt_trigger(void) · 20 unmodified lines
34 STAP_PROBE(test, pid_filter_usdt);39 STAP_PROBE(test, pid_filter_usdt);
35}40}
36 41
42noinline void uprobe_session_recursive(int i)
43{
44 if (i)
45 uprobe_session_recursive(i - 1);
46}
47
37struct child {48struct child {
38 int go[2];49 int go[2];
39 int c2p[2]; /* child -> parent channel */50 int c2p[2]; /* child -> parent channel */
@@ -125,7 +136,7 @@ static void *child_thread(void *ctx) · 85 unmodified lines
125 struct child *child = ctx;136 struct child *child = ctx;
126 int c = 0, err;137 int c = 0, err;
127 138
128 child->tid = syscall(SYS_gettid);139 child->tid = sys_gettid();
129 140
130 /* let parent know we are ready */141 /* let parent know we are ready */
131 err = write(child->c2p[1], &c, 1);142 err = write(child->c2p[1], &c, 1);
@@ -778,7 +789,7 @@ get_link(struct uprobe_multi_consumers *skel, int link) · 646 unmodified lines
778 }789 }
779}790}
780 791
781static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx)792static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx, unsigned long offset)
782{793{
783 struct bpf_program *prog = get_program(skel, idx);794 struct bpf_program *prog = get_program(skel, idx);
784 struct bpf_link **link = get_link(skel, idx);795 struct bpf_link **link = get_link(skel, idx);
@@ -787,15 +798,19 @@ static int uprobe_attach(struct uprobe_multi_consumers *skel, int idx) · 2 unmodified lines
787 if (!prog || !link)798 if (!prog || !link)
788 return -1;799 return -1;
789 800
801 opts.offsets = &offset;
802 opts.cnt = 1;
803
790 /*804 /*
791 * bit/prog: 0,1 uprobe entry805 * bit/prog: 0 uprobe entry
792 * bit/prog: 2,3 uprobe return806 * bit/prog: 1 uprobe return
807 * bit/prog: 2 uprobe session without return
808 * bit/prog: 3 uprobe session with return
793 */809 */
794 opts.retprobe = idx == 2 || idx == 3;810 opts.retprobe = idx == 1;
811 opts.session = idx == 2 || idx == 3;
795 812
796 *link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe",813 *link = bpf_program__attach_uprobe_multi(prog, 0, "/proc/self/exe", NULL, &opts);
797 "uprobe_consumer_test",
798 &opts);
799 if (!ASSERT_OK_PTR(*link, "bpf_program__attach_uprobe_multi"))814 if (!ASSERT_OK_PTR(*link, "bpf_program__attach_uprobe_multi"))
800 return -1;815 return -1;
801 return 0;816 return 0;
@@ -816,7 +831,8 @@ static bool test_bit(int bit, unsigned long val) · 14 unmodified lines
816 831
817noinline int832noinline int
818uprobe_consumer_test(struct uprobe_multi_consumers *skel,833uprobe_consumer_test(struct uprobe_multi_consumers *skel,
819 unsigned long before, unsigned long after)834 unsigned long before, unsigned long after,
835 unsigned long offset)
820{836{
821 int idx;837 int idx;
822 838
@@ -829,89 +845,174 @@ uprobe_consumer_test(struct uprobe_multi_consumers *skel, · 6 unmodified lines
829 /* ... and attach all new programs in 'after' state */845 /* ... and attach all new programs in 'after' state */
830 for (idx = 0; idx < 4; idx++) {846 for (idx = 0; idx < 4; idx++) {
831 if (!test_bit(idx, before) && test_bit(idx, after)) {847 if (!test_bit(idx, before) && test_bit(idx, after)) {
832 if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_after"))848 if (!ASSERT_OK(uprobe_attach(skel, idx, offset), "uprobe_attach_after"))
833 return -1;849 return -1;
834 }850 }
835 }851 }
836 return 0;852 return 0;
837}853}
838 854
839static void consumer_test(struct uprobe_multi_consumers *skel,855/*
840 unsigned long before, unsigned long after)856 * We generate 16 consumer_testX functions that will have uprobe installed on
857 * and will be called in separate threads. All function pointer are stored in
858 * "consumers" section and each thread will pick one function based on index.
859 */
860
861extern const void *__start_consumers;
862
863#define __CONSUMER_TEST(func) \
864noinline int func(struct uprobe_multi_consumers *skel, unsigned long before, \
865 unsigned long after, unsigned long offset) \
866{ \
867 return uprobe_consumer_test(skel, before, after, offset); \
868} \
869void *__ ## func __used __attribute__((section("consumers"))) = (void *) func;
870
871#define CONSUMER_TEST(func) __CONSUMER_TEST(func)
872
873#define C1 CONSUMER_TEST(__PASTE(consumer_test, __COUNTER__))
874#define C4 C1 C1 C1 C1
875#define C16 C4 C4 C4 C4
876
877C16
878
879typedef int (*test_t)(struct uprobe_multi_consumers *, unsigned long,
880 unsigned long, unsigned long);
881
882static int consumer_test(struct uprobe_multi_consumers *skel,
883 unsigned long before, unsigned long after,
884 test_t test, unsigned long offset)
841{885{
842 int err, idx;886 int err, idx, ret = -1;
843 887
844 printf("consumer_test before %lu after %lu\n", before, after);888 printf("consumer_test before %lu after %lu\n", before, after);
845 889
846 /* 'before' is each, we attach uprobe for every set idx */890 /* 'before' is each, we attach uprobe for every set idx */
847 for (idx = 0; idx < 4; idx++) {891 for (idx = 0; idx < 4; idx++) {
848 if (test_bit(idx, before)) {892 if (test_bit(idx, before)) {
849 if (!ASSERT_OK(uprobe_attach(skel, idx), "uprobe_attach_before"))893 if (!ASSERT_OK(uprobe_attach(skel, idx, offset), "uprobe_attach_before"))
850 goto cleanup;894 goto cleanup;
851 }895 }
852 }896 }
853 897
854 err = uprobe_consumer_test(skel, before, after);898 err = test(skel, before, after, offset);
855 if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))899 if (!ASSERT_EQ(err, 0, "uprobe_consumer_test"))
856 goto cleanup;900 goto cleanup;
857 901
858 for (idx = 0; idx < 4; idx++) {902 for (idx = 0; idx < 4; idx++) {
903 bool uret_stays, uret_survives;
859 const char *fmt = "BUG";904 const char *fmt = "BUG";
860 __u64 val = 0;905 __u64 val = 0;
861 906
862 if (idx < 2) {907 switch (idx) {
908 case 0:
863 /*909 /*
864 * uprobe entry910 * uprobe entry
865 * +1 if define in 'before'911 * +1 if define in 'before'
866 */912 */
867 if (test_bit(idx, before))913 if (test_bit(idx, before))
868 val++;914 val++;
869 fmt = "prog 0/1: uprobe";915 fmt = "prog 0: uprobe";
870 } else {916 break;
917 case 1:
871 /*918 /*
872 * uprobe return is tricky ;-)919 * To trigger uretprobe consumer, the uretprobe under test either stayed from
873 *920 * before to after (uret_stays + test_bit) or uretprobe instance survived and
874 * to trigger uretprobe consumer, the uretprobe needs to be installed,921 * we have uretprobe active in after (uret_survives + test_bit)
875 * which means one of the 'return' uprobes was alive when probe was hit:
876 *
877 * idxs: 2/3 uprobe return in 'installed' mask
878 *
879 * in addition if 'after' state removes everything that was installed in
880 * 'before' state, then uprobe kernel object goes away and return uprobe
881 * is not installed and we won't hit it even if it's in 'after' state.
882 */922 */
883 unsigned long had_uretprobes = before & 0b1100; /* is uretprobe installed */923 uret_stays = before & after & 0b0110;
884 unsigned long probe_preserved = before & after; /* did uprobe go away */924 uret_survives = ((before & 0b0110) && (after & 0b0110) && (before & 0b1001));
885 925
886 if (had_uretprobes && probe_preserved && test_bit(idx, after))926 if ((uret_stays || uret_survives) && test_bit(idx, after))
887 val++;927 val++;
888 fmt = "idx 2/3: uretprobe";928 fmt = "prog 1: uretprobe";
929 break;
930 case 2:
931 /*
932 * session with return
933 * +1 if defined in 'before'
934 * +1 if defined in 'after'
935 */
936 if (test_bit(idx, before)) {
937 val++;
938 if (test_bit(idx, after))
939 val++;
940 }
941 fmt = "prog 2: session with return";
942 break;
943 case 3:
944 /*
945 * session without return
946 * +1 if defined in 'before'
947 */
948 if (test_bit(idx, before))
949 val++;
950 fmt = "prog 3: session with NO return";
951 break;
889 }952 }
890 953
891 ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt);954 if (!ASSERT_EQ(skel->bss->uprobe_result[idx], val, fmt))
955 goto cleanup;
892 skel->bss->uprobe_result[idx] = 0;956 skel->bss->uprobe_result[idx] = 0;
893 }957 }
894 958
959 ret = 0;
960
895cleanup:961cleanup:
896 for (idx = 0; idx < 4; idx++)962 for (idx = 0; idx < 4; idx++)
897 uprobe_detach(skel, idx);963 uprobe_detach(skel, idx);
964 return ret;
898}965}
899 966
900static void test_consumers(void)967#define CONSUMER_MAX 16
968
969/*
970 * Each thread runs 1/16 of the load by running test for single
971 * 'before' number (based on thread index) and full scale of
972 * 'after' numbers.
973 */
974static void *consumer_thread(void *arg)
901{975{
976 unsigned long idx = (unsigned long) arg;
902 struct uprobe_multi_consumers *skel;977 struct uprobe_multi_consumers *skel;
903 int before, after;978 unsigned long offset;
979 const void *func;
980 int after;
904 981
905 skel = uprobe_multi_consumers__open_and_load();982 skel = uprobe_multi_consumers__open_and_load();
906 if (!ASSERT_OK_PTR(skel, "uprobe_multi_consumers__open_and_load"))983 if (!ASSERT_OK_PTR(skel, "uprobe_multi_consumers__open_and_load"))
907 return;984 return NULL;
985
986 func = *((&__start_consumers) + idx);
987
988 offset = get_uprobe_offset(func);
989 if (!ASSERT_GE(offset, 0, "uprobe_offset"))
990 goto out;
991
992 for (after = 0; after < CONSUMER_MAX; after++)
993 if (consumer_test(skel, idx, after, func, offset))
994 goto out;
995
996out:
997 uprobe_multi_consumers__destroy(skel);
998 return NULL;
999}
1000
1001
1002static void test_consumers(void)
1003{
1004 pthread_t pt[CONSUMER_MAX];
1005 unsigned long idx;
1006 int err;
908 1007
909 /*1008 /*
910 * The idea of this test is to try all possible combinations of1009 * The idea of this test is to try all possible combinations of
911 * uprobes consumers attached on single function.1010 * uprobes consumers attached on single function.
912 *1011 *
913 * - 2 uprobe entry consumer1012 * - 1 uprobe entry consumer
914 * - 2 uprobe exit consumers1013 * - 1 uprobe exit consumer
1014 * - 1 uprobe session with return
1015 * - 1 uprobe session without return
915 *1016 *
916 * The test uses 4 uprobes attached on single function, but that1017 * The test uses 4 uprobes attached on single function, but that
917 * translates into single uprobe with 4 consumers in kernel.1018 * translates into single uprobe with 4 consumers in kernel.
@@ -919,37 +1020,38 @@ static void test_consumers(void) · 1 unmodified line
919 * The before/after values present the state of attached consumers1020 * The before/after values present the state of attached consumers
920 * before and after the probed function:1021 * before and after the probed function:
921 *1022 *
922 * bit/prog 0,1 : uprobe entry1023 * bit/prog 0 : uprobe entry
923 * bit/prog 2,3 : uprobe return1024 * bit/prog 1 : uprobe return
924 *1025 *
925 * For example for:1026 * For example for:
926 *1027 *
927 * before = 0b01011028 * before = 0b01
928 * after = 0b01101029 * after = 0b10
929 *1030 *
930 * it means that before we call 'uprobe_consumer_test' we attach1031 * it means that before we call 'uprobe_consumer_test' we attach
931 * uprobes defined in 'before' value:1032 * uprobes defined in 'before' value:
932 *1033 *
933 * - bit/prog 0: uprobe entry1034 * - bit/prog 1: uprobe entry
934 * - bit/prog 2: uprobe return
935 *1035 *
936 * uprobe_consumer_test is called and inside it we attach and detach1036 * uprobe_consumer_test is called and inside it we attach and detach
937 * uprobes based on 'after' value:1037 * uprobes based on 'after' value:
938 *1038 *
939 * - bit/prog 0: stays untouched1039 * - bit/prog 0: is detached
940 * - bit/prog 2: uprobe return is detached1040 * - bit/prog 1: is attached
941 *1041 *
942 * uprobe_consumer_test returns and we check counters values increased1042 * uprobe_consumer_test returns and we check counters values increased
943 * by bpf programs on each uprobe to match the expected count based on1043 * by bpf programs on each uprobe to match the expected count based on
944 * before/after bits.1044 * before/after bits.
945 */1045 */
946 1046
947 for (before = 0; before < 16; before++) {1047 for (idx = 0; idx < CONSUMER_MAX; idx++) {
948 for (after = 0; after < 16; after++)1048 err = pthread_create(&pt[idx], NULL, consumer_thread, (void *) idx);
949 consumer_test(skel, before, after);1049 if (!ASSERT_OK(err, "pthread_create"))
1050 break;
950 }1051 }
951 1052
952 uprobe_multi_consumers__destroy(skel);1053 while (idx)
1054 pthread_join(pt[--idx], NULL);
953}1055}
954 1056
955static struct bpf_program *uprobe_multi_program(struct uprobe_multi_pid_filter *skel, int idx)1057static struct bpf_program *uprobe_multi_program(struct uprobe_multi_pid_filter *skel, int idx)
@@ -1016,6 +1118,156 @@ static void test_pid_filter_process(bool clone_vm) · 60 unmodified lines
1016 uprobe_multi_pid_filter__destroy(skel);1118 uprobe_multi_pid_filter__destroy(skel);
1017}1119}
1018 1120
1121static void test_session_skel_api(void)
1122{
1123 struct uprobe_multi_session *skel = NULL;
1124 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
1125 struct bpf_link *link = NULL;
1126 int err;
1127
1128 skel = uprobe_multi_session__open_and_load();
1129 if (!ASSERT_OK_PTR(skel, "uprobe_multi_session__open_and_load"))
1130 goto cleanup;
1131
1132 skel->bss->pid = getpid();
1133 skel->bss->user_ptr = test_data;
1134
1135 err = uprobe_multi_session__attach(skel);
1136 if (!ASSERT_OK(err, "uprobe_multi_session__attach"))
1137 goto cleanup;
1138
1139 /* trigger all probes */
1140 skel->bss->uprobe_multi_func_1_addr = (__u64) uprobe_multi_func_1;
1141 skel->bss->uprobe_multi_func_2_addr = (__u64) uprobe_multi_func_2;
1142 skel->bss->uprobe_multi_func_3_addr = (__u64) uprobe_multi_func_3;
1143
1144 uprobe_multi_func_1();
1145 uprobe_multi_func_2();
1146 uprobe_multi_func_3();
1147
1148 /*
1149 * We expect 2 for uprobe_multi_func_2 because it runs both entry/return probe,
1150 * uprobe_multi_func_[13] run just the entry probe. All expected numbers are
1151 * doubled, because we run extra test for sleepable session.
1152 */
1153 ASSERT_EQ(skel->bss->uprobe_session_result[0], 2, "uprobe_multi_func_1_result");
1154 ASSERT_EQ(skel->bss->uprobe_session_result[1], 4, "uprobe_multi_func_2_result");
1155 ASSERT_EQ(skel->bss->uprobe_session_result[2], 2, "uprobe_multi_func_3_result");
1156
1157 /* We expect increase in 3 entry and 1 return session calls -> 4 */
1158 ASSERT_EQ(skel->bss->uprobe_multi_sleep_result, 4, "uprobe_multi_sleep_result");
1159
1160cleanup:
1161 bpf_link__destroy(link);
1162 uprobe_multi_session__destroy(skel);
1163}
1164
1165static void test_session_single_skel_api(void)
1166{
1167 struct uprobe_multi_session_single *skel = NULL;
1168 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
1169 int err;
1170
1171 skel = uprobe_multi_session_single__open_and_load();
1172 if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_single__open_and_load"))
1173 goto cleanup;
1174
1175 skel->bss->pid = getpid();
1176
1177 err = uprobe_multi_session_single__attach(skel);
1178 if (!ASSERT_OK(err, "uprobe_multi_session_single__attach"))
1179 goto cleanup;
1180
1181 uprobe_multi_func_1();
1182
1183 /*
1184 * We expect consumer 0 and 2 to trigger just entry handler (value 1)
1185 * and consumer 1 to hit both (value 2).
1186 */
1187 ASSERT_EQ(skel->bss->uprobe_session_result[0], 1, "uprobe_session_result_0");
1188 ASSERT_EQ(skel->bss->uprobe_session_result[1], 2, "uprobe_session_result_1");
1189 ASSERT_EQ(skel->bss->uprobe_session_result[2], 1, "uprobe_session_result_2");
1190
1191cleanup:
1192 uprobe_multi_session_single__destroy(skel);
1193}
1194
1195static void test_session_cookie_skel_api(void)
1196{
1197 struct uprobe_multi_session_cookie *skel = NULL;
1198 int err;
1199
1200 skel = uprobe_multi_session_cookie__open_and_load();
1201 if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_cookie__open_and_load"))
1202 goto cleanup;
1203
1204 skel->bss->pid = getpid();
1205
1206 err = uprobe_multi_session_cookie__attach(skel);
1207 if (!ASSERT_OK(err, "uprobe_multi_session_cookie__attach"))
1208 goto cleanup;
1209
1210 /* trigger all probes */
1211 uprobe_multi_func_1();
1212 uprobe_multi_func_2();
1213 uprobe_multi_func_3();
1214
1215 ASSERT_EQ(skel->bss->test_uprobe_1_result, 1, "test_uprobe_1_result");
1216 ASSERT_EQ(skel->bss->test_uprobe_2_result, 2, "test_uprobe_2_result");
1217 ASSERT_EQ(skel->bss->test_uprobe_3_result, 3, "test_uprobe_3_result");
1218
1219cleanup:
1220 uprobe_multi_session_cookie__destroy(skel);
1221}
1222
1223static void test_session_recursive_skel_api(void)
1224{
1225 struct uprobe_multi_session_recursive *skel = NULL;
1226 int i, err;
1227
1228 skel = uprobe_multi_session_recursive__open_and_load();
1229 if (!ASSERT_OK_PTR(skel, "uprobe_multi_session_recursive__open_and_load"))
1230 goto cleanup;
1231
1232 skel->bss->pid = getpid();
1233
1234 err = uprobe_multi_session_recursive__attach(skel);
1235 if (!ASSERT_OK(err, "uprobe_multi_session_recursive__attach"))
1236 goto cleanup;
1237
1238 for (i = 0; i < ARRAY_SIZE(skel->bss->test_uprobe_cookie_entry); i++)
1239 skel->bss->test_uprobe_cookie_entry[i] = i + 1;
1240
1241 uprobe_session_recursive(5);
1242
1243 /*
1244 * entry uprobe:
1245 * uprobe_session_recursive(5) { *cookie = 1, return 0
1246 * uprobe_session_recursive(4) { *cookie = 2, return 1
1247 * uprobe_session_recursive(3) { *cookie = 3, return 0
1248 * uprobe_session_recursive(2) { *cookie = 4, return 1
1249 * uprobe_session_recursive(1) { *cookie = 5, return 0
1250 * uprobe_session_recursive(0) { *cookie = 6, return 1
1251 * return uprobe:
1252 * } i = 0 not executed
1253 * } i = 1 test_uprobe_cookie_return[0] = 5
1254 * } i = 2 not executed
1255 * } i = 3 test_uprobe_cookie_return[1] = 3
1256 * } i = 4 not executed
1257 * } i = 5 test_uprobe_cookie_return[2] = 1
1258 */
1259
1260 ASSERT_EQ(skel->bss->idx_entry, 6, "idx_entry");
1261 ASSERT_EQ(skel->bss->idx_return, 3, "idx_return");
1262
1263 ASSERT_EQ(skel->bss->test_uprobe_cookie_return[0], 5, "test_uprobe_cookie_return[0]");
1264 ASSERT_EQ(skel->bss->test_uprobe_cookie_return[1], 3, "test_uprobe_cookie_return[1]");
1265 ASSERT_EQ(skel->bss->test_uprobe_cookie_return[2], 1, "test_uprobe_cookie_return[2]");
1266
1267cleanup:
1268 uprobe_multi_session_recursive__destroy(skel);
1269}
1270
1019static void test_bench_attach_uprobe(void)1271static void test_bench_attach_uprobe(void)
1020{1272{
1021 long attach_start_ns = 0, attach_end_ns = 0;1273 long attach_start_ns = 0, attach_end_ns = 0;
@@ -1112,4 +1364,13 @@ void test_uprobe_multi_test(void) · 90 unmodified lines
1112 test_pid_filter_process(false);1364 test_pid_filter_process(false);
1113 if (test__start_subtest("filter_clone_vm"))1365 if (test__start_subtest("filter_clone_vm"))
1114 test_pid_filter_process(true);1366 test_pid_filter_process(true);
1367 if (test__start_subtest("session"))
1368 test_session_skel_api();
1369 if (test__start_subtest("session_single"))
1370 test_session_single_skel_api();
1371 if (test__start_subtest("session_cookie"))
1372 test_session_cookie_skel_api();
1373 if (test__start_subtest("session_cookie_recursive"))
1374 test_session_recursive_skel_api();
1375 RUN_TESTS(uprobe_multi_verifier);
1115}1376}
tools/testing/selftests/bpf/prog_tests/verifier.c+2−0
@@ -61,6 +61,7 @@ · 60 unmodified lines
61#include "verifier_or_jmp32_k.skel.h"61#include "verifier_or_jmp32_k.skel.h"
62#include "verifier_precision.skel.h"62#include "verifier_precision.skel.h"
63#include "verifier_prevent_map_lookup.skel.h"63#include "verifier_prevent_map_lookup.skel.h"
64#include "verifier_private_stack.skel.h"
64#include "verifier_raw_stack.skel.h"65#include "verifier_raw_stack.skel.h"
65#include "verifier_raw_tp_writable.skel.h"66#include "verifier_raw_tp_writable.skel.h"
66#include "verifier_reg_equal.skel.h"67#include "verifier_reg_equal.skel.h"
@@ -188,6 +189,7 @@ void test_verifier_bpf_fastcall(void) { RUN(verifier_bpf_fastcall); } · 121 unmodified lines
188void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); }189void test_verifier_or_jmp32_k(void) { RUN(verifier_or_jmp32_k); }
189void test_verifier_precision(void) { RUN(verifier_precision); }190void test_verifier_precision(void) { RUN(verifier_precision); }
190void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); }191void test_verifier_prevent_map_lookup(void) { RUN(verifier_prevent_map_lookup); }
192void test_verifier_private_stack(void) { RUN(verifier_private_stack); }
191void test_verifier_raw_stack(void) { RUN(verifier_raw_stack); }193void test_verifier_raw_stack(void) { RUN(verifier_raw_stack); }
192void test_verifier_raw_tp_writable(void) { RUN(verifier_raw_tp_writable); }194void test_verifier_raw_tp_writable(void) { RUN(verifier_raw_tp_writable); }
193void test_verifier_reg_equal(void) { RUN(verifier_reg_equal); }195void test_verifier_reg_equal(void) { RUN(verifier_reg_equal); }
tools/testing/selftests/bpf/progs/bpf_iter.h+0−167
@@ -1,167 +0,0 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2020 Facebook */
3/* "undefine" structs in vmlinux.h, because we "override" them below */
4#define bpf_iter_meta bpf_iter_meta___not_used
5#define bpf_iter__bpf_map bpf_iter__bpf_map___not_used
6#define bpf_iter__ipv6_route bpf_iter__ipv6_route___not_used
7#define bpf_iter__netlink bpf_iter__netlink___not_used
8#define bpf_iter__task bpf_iter__task___not_used
9#define bpf_iter__task_file bpf_iter__task_file___not_used
10#define bpf_iter__task_vma bpf_iter__task_vma___not_used
11#define bpf_iter__tcp bpf_iter__tcp___not_used
12#define tcp6_sock tcp6_sock___not_used
13#define bpf_iter__udp bpf_iter__udp___not_used
14#define udp6_sock udp6_sock___not_used
15#define bpf_iter__unix bpf_iter__unix___not_used
16#define bpf_iter__bpf_map_elem bpf_iter__bpf_map_elem___not_used
17#define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used
18#define bpf_iter__sockmap bpf_iter__sockmap___not_used
19#define bpf_iter__bpf_link bpf_iter__bpf_link___not_used
20#define bpf_iter__cgroup bpf_iter__cgroup___not_used
21#define btf_ptr btf_ptr___not_used
22#define BTF_F_COMPACT BTF_F_COMPACT___not_used
23#define BTF_F_NONAME BTF_F_NONAME___not_used
24#define BTF_F_PTR_RAW BTF_F_PTR_RAW___not_used
25#define BTF_F_ZERO BTF_F_ZERO___not_used
26#define bpf_iter__ksym bpf_iter__ksym___not_used
27#include "vmlinux.h"
28#undef bpf_iter_meta
29#undef bpf_iter__bpf_map
30#undef bpf_iter__ipv6_route
31#undef bpf_iter__netlink
32#undef bpf_iter__task
33#undef bpf_iter__task_file
34#undef bpf_iter__task_vma
35#undef bpf_iter__tcp
36#undef tcp6_sock
37#undef bpf_iter__udp
38#undef udp6_sock
39#undef bpf_iter__unix
40#undef bpf_iter__bpf_map_elem
41#undef bpf_iter__bpf_sk_storage_map
42#undef bpf_iter__sockmap
43#undef bpf_iter__bpf_link
44#undef bpf_iter__cgroup
45#undef btf_ptr
46#undef BTF_F_COMPACT
47#undef BTF_F_NONAME
48#undef BTF_F_PTR_RAW
49#undef BTF_F_ZERO
50#undef bpf_iter__ksym
51
52struct bpf_iter_meta {
53 struct seq_file *seq;
54 __u64 session_id;
55 __u64 seq_num;
56} __attribute__((preserve_access_index));
57
58struct bpf_iter__ipv6_route {
59 struct bpf_iter_meta *meta;
60 struct fib6_info *rt;
61} __attribute__((preserve_access_index));
62
63struct bpf_iter__netlink {
64 struct bpf_iter_meta *meta;
65 struct netlink_sock *sk;
66} __attribute__((preserve_access_index));
67
68struct bpf_iter__task {
69 struct bpf_iter_meta *meta;
70 struct task_struct *task;
71} __attribute__((preserve_access_index));
72
73struct bpf_iter__task_file {
74 struct bpf_iter_meta *meta;
75 struct task_struct *task;
76 __u32 fd;
77 struct file *file;
78} __attribute__((preserve_access_index));
79
80struct bpf_iter__task_vma {
81 struct bpf_iter_meta *meta;
82 struct task_struct *task;
83 struct vm_area_struct *vma;
84} __attribute__((preserve_access_index));
85
86struct bpf_iter__bpf_map {
87 struct bpf_iter_meta *meta;
88 struct bpf_map *map;
89} __attribute__((preserve_access_index));
90
91struct bpf_iter__tcp {
92 struct bpf_iter_meta *meta;
93 struct sock_common *sk_common;
94 uid_t uid;
95} __attribute__((preserve_access_index));
96
97struct tcp6_sock {
98 struct tcp_sock tcp;
99 struct ipv6_pinfo inet6;
100} __attribute__((preserve_access_index));
101
102struct bpf_iter__udp {
103 struct bpf_iter_meta *meta;
104 struct udp_sock *udp_sk;
105 uid_t uid __attribute__((aligned(8)));
106 int bucket __attribute__((aligned(8)));
107} __attribute__((preserve_access_index));
108
109struct udp6_sock {
110 struct udp_sock udp;
111 struct ipv6_pinfo inet6;
112} __attribute__((preserve_access_index));
113
114struct bpf_iter__unix {
115 struct bpf_iter_meta *meta;
116 struct unix_sock *unix_sk;
117 uid_t uid;
118} __attribute__((preserve_access_index));
119
120struct bpf_iter__bpf_map_elem {
121 struct bpf_iter_meta *meta;
122 struct bpf_map *map;
123 void *key;
124 void *value;
125};
126
127struct bpf_iter__bpf_sk_storage_map {
128 struct bpf_iter_meta *meta;
129 struct bpf_map *map;
130 struct sock *sk;
131 void *value;
132};
133
134struct bpf_iter__sockmap {
135 struct bpf_iter_meta *meta;
136 struct bpf_map *map;
137 void *key;
138 struct sock *sk;
139};
140
141struct bpf_iter__bpf_link {
142 struct bpf_iter_meta *meta;
143 struct bpf_link *link;
144};
145
146struct bpf_iter__cgroup {
147 struct bpf_iter_meta *meta;
148 struct cgroup *cgroup;
149} __attribute__((preserve_access_index));
150
151struct btf_ptr {
152 void *ptr;
153 __u32 type_id;
154 __u32 flags;
155};
156
157enum {
158 BTF_F_COMPACT = (1ULL << 0),
159 BTF_F_NONAME = (1ULL << 1),
160 BTF_F_PTR_RAW = (1ULL << 2),
161 BTF_F_ZERO = (1ULL << 3),
162};
163
164struct bpf_iter__ksym {
165 struct bpf_iter_meta *meta;
166 struct kallsym_iter *ksym;
167};
tools/testing/selftests/bpf/progs/bpf_iter_bpf_array_map.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_bpf_hash_map.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_bpf_link.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2022 Red Hat, Inc. */2/* Copyright (c) 2022 Red Hat, Inc. */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_bpf_map.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_array_map.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_bpf_percpu_hash_map.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_helpers.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Google LLC. */2/* Copyright (c) 2020 Google LLC. */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_bpf_sk_storage_map.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>6#include <bpf/bpf_tracing.h>
tools/testing/selftests/bpf/progs/bpf_iter_ipv6_route.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_ksym.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2022, Oracle and/or its affiliates. */2/* Copyright (c) 2022, Oracle and/or its affiliates. */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_netlink.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_setsockopt.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2021 Facebook */2/* Copyright (c) 2021 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_endian.h>6#include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_setsockopt_unix.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright Amazon.com Inc. or its affiliates. */2/* Copyright Amazon.com Inc. or its affiliates. */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <limits.h>6#include <limits.h>
tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Cloudflare */2/* Copyright (c) 2020 Cloudflare */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>6#include <bpf/bpf_tracing.h>
tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020, Oracle and/or its affiliates. */2/* Copyright (c) 2020, Oracle and/or its affiliates. */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_core_read.h>5#include <bpf/bpf_core_read.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_task_file.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_tasks.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_endian.h>6#include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_endian.h>6#include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_test_kern3.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_test_kern4.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_test_kern5.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
6 6
tools/testing/selftests/bpf/progs/bpf_iter_test_kern6.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_test_kern_common.h+1−1
@@ -1,6 +1,6 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/bpf_iter_udp4.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_endian.h>6#include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_udp6.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */2/* Copyright (c) 2020 Facebook */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_endian.h>6#include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_unix.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright Amazon.com Inc. or its affiliates. */2/* Copyright Amazon.com Inc. or its affiliates. */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include "bpf_tracing_net.h"4#include "bpf_tracing_net.h"
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_endian.h>6#include <bpf/bpf_endian.h>
tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c+1−1
@@ -1,6 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3#include "bpf_iter.h"3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
5 5
6char _license[] SEC("license") = "GPL";6char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/cgroup_iter.c+1−2
@@ -1,7 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2022 Google */2/* Copyright (c) 2022 Google */
3 3#include <vmlinux.h>
4#include "bpf_iter.h"
5#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
7 6
tools/testing/selftests/bpf/progs/cgrp_ls_sleepable.c+1−2
@@ -1,7 +1,6 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */2/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
3 3#include <vmlinux.h>
4#include "bpf_iter.h"
5#include <bpf/bpf_helpers.h>4#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>5#include <bpf/bpf_tracing.h>
7#include "bpf_misc.h"6#include "bpf_misc.h"
tools/testing/selftests/bpf/progs/exceptions_fail.c+2−2
@@ -131,7 +131,7 @@ int reject_subprog_with_lock(void *ctx) · 130 unmodified lines
131}131}
132 132
133SEC("?tc")133SEC("?tc")
134__failure __msg("bpf_rcu_read_unlock is missing")134__failure __msg("BPF_EXIT instruction cannot be used inside bpf_rcu_read_lock-ed region")
135int reject_with_rcu_read_lock(void *ctx)135int reject_with_rcu_read_lock(void *ctx)
136{136{
137 bpf_rcu_read_lock();137 bpf_rcu_read_lock();
@@ -147,7 +147,7 @@ __noinline static int throwing_subprog(struct __sk_buff *ctx) · 9 unmodified lines
147}147}
148 148
149SEC("?tc")149SEC("?tc")
150__failure __msg("bpf_rcu_read_unlock is missing")150__failure __msg("BPF_EXIT instruction cannot be used inside bpf_rcu_read_lock-ed region")
151int reject_subprog_with_rcu_read_lock(void *ctx)151int reject_subprog_with_rcu_read_lock(void *ctx)
152{152{
153 bpf_rcu_read_lock();153 bpf_rcu_read_lock();
tools/testing/selftests/bpf/progs/kmem_cache_iter.c+108−0
@@ -0,0 +1,108 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Google */
3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6#include "bpf_experimental.h"
7
8char _license[] SEC("license") = "GPL";
9
10#define SLAB_NAME_MAX 32
11
12struct kmem_cache_result {
13 char name[SLAB_NAME_MAX];
14 long obj_size;
15};
16
17struct {
18 __uint(type, BPF_MAP_TYPE_HASH);
19 __uint(key_size, sizeof(void *));
20 __uint(value_size, SLAB_NAME_MAX);
21 __uint(max_entries, 1);
22} slab_hash SEC(".maps");
23
24struct {
25 __uint(type, BPF_MAP_TYPE_ARRAY);
26 __uint(key_size, sizeof(int));
27 __uint(value_size, sizeof(struct kmem_cache_result));
28 __uint(max_entries, 1024);
29} slab_result SEC(".maps");
30
31extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym;
32
33/* Result, will be checked by userspace */
34int task_struct_found;
35int kmem_cache_seen;
36int open_coded_seen;
37
38SEC("iter/kmem_cache")
39int slab_info_collector(struct bpf_iter__kmem_cache *ctx)
40{
41 struct seq_file *seq = ctx->meta->seq;
42 struct kmem_cache *s = ctx->s;
43 struct kmem_cache_result *r;
44 int idx;
45
46 if (s) {
47 /* To make sure if the slab_iter implements the seq interface
48 * properly and it's also useful for debugging.
49 */
50 BPF_SEQ_PRINTF(seq, "%s: %u\n", s->name, s->size);
51
52 idx = kmem_cache_seen;
53 r = bpf_map_lookup_elem(&slab_result, &idx);
54 if (r == NULL)
55 return 0;
56
57 kmem_cache_seen++;
58
59 /* Save name and size to match /proc/slabinfo */
60 bpf_probe_read_kernel_str(r->name, sizeof(r->name), s->name);
61 r->obj_size = s->size;
62
63 if (!bpf_strncmp(r->name, 11, "task_struct"))
64 bpf_map_update_elem(&slab_hash, &s, r->name, BPF_NOEXIST);
65 }
66
67 return 0;
68}
69
70SEC("raw_tp/bpf_test_finish")
71int BPF_PROG(check_task_struct)
72{
73 u64 curr = bpf_get_current_task();
74 struct kmem_cache *s;
75 char *name;
76
77 s = bpf_get_kmem_cache(curr);
78 if (s == NULL) {
79 task_struct_found = -1;
80 return 0;
81 }
82 name = bpf_map_lookup_elem(&slab_hash, &s);
83 if (name && !bpf_strncmp(name, 11, "task_struct"))
84 task_struct_found = 1;
85 else
86 task_struct_found = -2;
87 return 0;
88}
89
90SEC("syscall")
91int open_coded_iter(const void *ctx)
92{
93 struct kmem_cache *s;
94
95 bpf_for_each(kmem_cache, s) {
96 struct kmem_cache_result *r;
97
98 r = bpf_map_lookup_elem(&slab_result, &open_coded_seen);
99 if (!r)
100 break;
101
102 if (r->obj_size != s->size)
103 break;
104
105 open_coded_seen++;
106 }
107 return 0;
108}
tools/testing/selftests/bpf/progs/kprobe_multi_verifier.c+31−0
@@ -0,0 +1,31 @@
1// SPDX-License-Identifier: GPL-2.0
2#include "vmlinux.h"
3#include <bpf/bpf_helpers.h>
4#include <bpf/bpf_tracing.h>
5#include <bpf/usdt.bpf.h>
6#include "bpf_misc.h"
7
8char _license[] SEC("license") = "GPL";
9
10
11SEC("kprobe.session")
12__success
13int kprobe_session_return_0(struct pt_regs *ctx)
14{
15 return 0;
16}
17
18SEC("kprobe.session")
19__success
20int kprobe_session_return_1(struct pt_regs *ctx)
21{
22 return 1;
23}
24
25SEC("kprobe.session")
26__failure
27__msg("At program exit the register R0 has smin=2 smax=2 should have been in [0, 1]")
28int kprobe_session_return_2(struct pt_regs *ctx)
29{
30 return 2;
31}
tools/testing/selftests/bpf/progs/linked_funcs1.c+8−0
@@ -63,6 +63,8 @@ extern int set_output_val2(int x); · 62 unmodified lines
63/* here we'll force set_output_ctx2() to be __hidden in the final obj file */63/* here we'll force set_output_ctx2() to be __hidden in the final obj file */
64__hidden extern void set_output_ctx2(__u64 *ctx);64__hidden extern void set_output_ctx2(__u64 *ctx);
65 65
66void *bpf_cast_to_kern_ctx(void *obj) __ksym;
67
66SEC("?raw_tp/sys_enter")68SEC("?raw_tp/sys_enter")
67int BPF_PROG(handler1, struct pt_regs *regs, long id)69int BPF_PROG(handler1, struct pt_regs *regs, long id)
68{70{
@@ -86,4 +88,10 @@ int BPF_PROG(handler1, struct pt_regs *regs, long id) · 17 unmodified lines
86 return 0;88 return 0;
87}89}
88 90
91/* Generate BTF FUNC record and test linking with duplicate extern functions */
92void kfunc_gen1(void)
93{
94 bpf_cast_to_kern_ctx(0);
95}
96
89char LICENSE[] SEC("license") = "GPL";97char LICENSE[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/linked_funcs2.c+8−0
@@ -63,6 +63,8 @@ extern int set_output_val1(int x); · 62 unmodified lines
63/* here we'll force set_output_ctx1() to be __hidden in the final obj file */63/* here we'll force set_output_ctx1() to be __hidden in the final obj file */
64__hidden extern void set_output_ctx1(__u64 *ctx);64__hidden extern void set_output_ctx1(__u64 *ctx);
65 65
66void *bpf_cast_to_kern_ctx(void *obj) __ksym;
67
66SEC("?raw_tp/sys_enter")68SEC("?raw_tp/sys_enter")
67int BPF_PROG(handler2, struct pt_regs *regs, long id)69int BPF_PROG(handler2, struct pt_regs *regs, long id)
68{70{
@@ -86,4 +88,10 @@ int BPF_PROG(handler2, struct pt_regs *regs, long id) · 17 unmodified lines
86 return 0;88 return 0;
87}89}
88 90
91/* Generate BTF FUNC record and test linking with duplicate extern functions */
92void kfunc_gen2(void)
93{
94 bpf_cast_to_kern_ctx(0);
95}
96
89char LICENSE[] SEC("license") = "GPL";97char LICENSE[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/preempt_lock.c+7−7
@@ -6,7 +6,7 @@ · 5 unmodified lines
6#include "bpf_experimental.h"6#include "bpf_experimental.h"
7 7
8SEC("?tc")8SEC("?tc")
9__failure __msg("1 bpf_preempt_enable is missing")9__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
10int preempt_lock_missing_1(struct __sk_buff *ctx)10int preempt_lock_missing_1(struct __sk_buff *ctx)
11{11{
12 bpf_preempt_disable();12 bpf_preempt_disable();
@@ -14,7 +14,7 @@ int preempt_lock_missing_1(struct __sk_buff *ctx) · 1 unmodified line
14}14}
15 15
16SEC("?tc")16SEC("?tc")
17__failure __msg("2 bpf_preempt_enable(s) are missing")17__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
18int preempt_lock_missing_2(struct __sk_buff *ctx)18int preempt_lock_missing_2(struct __sk_buff *ctx)
19{19{
20 bpf_preempt_disable();20 bpf_preempt_disable();
@@ -23,7 +23,7 @@ int preempt_lock_missing_2(struct __sk_buff *ctx) · 2 unmodified lines
23}23}
24 24
25SEC("?tc")25SEC("?tc")
26__failure __msg("3 bpf_preempt_enable(s) are missing")26__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
27int preempt_lock_missing_3(struct __sk_buff *ctx)27int preempt_lock_missing_3(struct __sk_buff *ctx)
28{28{
29 bpf_preempt_disable();29 bpf_preempt_disable();
@@ -33,7 +33,7 @@ int preempt_lock_missing_3(struct __sk_buff *ctx) · 3 unmodified lines
33}33}
34 34
35SEC("?tc")35SEC("?tc")
36__failure __msg("1 bpf_preempt_enable is missing")36__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
37int preempt_lock_missing_3_minus_2(struct __sk_buff *ctx)37int preempt_lock_missing_3_minus_2(struct __sk_buff *ctx)
38{38{
39 bpf_preempt_disable();39 bpf_preempt_disable();
@@ -55,7 +55,7 @@ static __noinline void preempt_enable(void) · 15 unmodified lines
55}55}
56 56
57SEC("?tc")57SEC("?tc")
58__failure __msg("1 bpf_preempt_enable is missing")58__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
59int preempt_lock_missing_1_subprog(struct __sk_buff *ctx)59int preempt_lock_missing_1_subprog(struct __sk_buff *ctx)
60{60{
61 preempt_disable();61 preempt_disable();
@@ -63,7 +63,7 @@ int preempt_lock_missing_1_subprog(struct __sk_buff *ctx) · 1 unmodified line
63}63}
64 64
65SEC("?tc")65SEC("?tc")
66__failure __msg("2 bpf_preempt_enable(s) are missing")66__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
67int preempt_lock_missing_2_subprog(struct __sk_buff *ctx)67int preempt_lock_missing_2_subprog(struct __sk_buff *ctx)
68{68{
69 preempt_disable();69 preempt_disable();
@@ -72,7 +72,7 @@ int preempt_lock_missing_2_subprog(struct __sk_buff *ctx) · 2 unmodified lines
72}72}
73 73
74SEC("?tc")74SEC("?tc")
75__failure __msg("1 bpf_preempt_enable is missing")75__failure __msg("BPF_EXIT instruction cannot be used inside bpf_preempt_disable-ed region")
76int preempt_lock_missing_2_minus_1_subprog(struct __sk_buff *ctx)76int preempt_lock_missing_2_minus_1_subprog(struct __sk_buff *ctx)
77{77{
78 preempt_disable();78 preempt_disable();
tools/testing/selftests/bpf/progs/raw_tp_null.c+32−0
@@ -0,0 +1,32 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#include <vmlinux.h>
5#include <bpf/bpf_tracing.h>
6
7char _license[] SEC("license") = "GPL";
8
9int tid;
10int i;
11
12SEC("tp_btf/bpf_testmod_test_raw_tp_null")
13int BPF_PROG(test_raw_tp_null, struct sk_buff *skb)
14{
15 struct task_struct *task = bpf_get_current_task_btf();
16
17 if (task->pid != tid)
18 return 0;
19
20 i = i + skb->mark + 1;
21 /* The compiler may move the NULL check before this deref, which causes
22 * the load to fail as deref of scalar. Prevent that by using a barrier.
23 */
24 barrier();
25 /* If dead code elimination kicks in, the increment below will
26 * be removed. For raw_tp programs, we mark input arguments as
27 * PTR_MAYBE_NULL, so branch prediction should never kick in.
28 */
29 if (!skb)
30 i += 2;
31 return 0;
32}
tools/testing/selftests/bpf/progs/struct_ops_detach.c+12−0
@@ -6,5 +6,17 @@ · 5 unmodified lines
6 6
7char _license[] SEC("license") = "GPL";7char _license[] SEC("license") = "GPL";
8 8
9/*
10 * This subprogram validates that libbpf handles the situation in which BPF
11 * object has subprograms in .text section, but has no entry BPF programs.
12 * At some point that was causing issues due to legacy logic of treating such
13 * subprogram as entry program (with unknown program type, which would fail).
14 */
15int dangling_subprog(void)
16{
17 /* do nothing, just be here */
18 return 0;
19}
20
9SEC(".struct_ops.link")21SEC(".struct_ops.link")
10struct bpf_testmod_ops testmod_do_detach;22struct bpf_testmod_ops testmod_do_detach;
tools/testing/selftests/bpf/progs/struct_ops_private_stack.c+62−0
@@ -0,0 +1,62 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6#include "../bpf_testmod/bpf_testmod.h"
7
8char _license[] SEC("license") = "GPL";
9
10#if defined(__TARGET_ARCH_x86)
11bool skip __attribute((__section__(".data"))) = false;
12#else
13bool skip = true;
14#endif
15
16void bpf_testmod_ops3_call_test_2(void) __ksym;
17
18int val_i, val_j;
19
20__noinline static int subprog2(int *a, int *b)
21{
22 return val_i + a[10] + b[20];
23}
24
25__noinline static int subprog1(int *a)
26{
27 /* stack size 200 bytes */
28 int b[50] = {};
29
30 b[20] = 2;
31 return subprog2(a, b);
32}
33
34
35SEC("struct_ops")
36int BPF_PROG(test_1)
37{
38 /* stack size 400 bytes */
39 int a[100] = {};
40
41 a[10] = 1;
42 val_i = subprog1(a);
43 bpf_testmod_ops3_call_test_2();
44 return 0;
45}
46
47SEC("struct_ops")
48int BPF_PROG(test_2)
49{
50 /* stack size 200 bytes */
51 int a[50] = {};
52
53 a[10] = 3;
54 val_j = subprog1(a);
55 return 0;
56}
57
58SEC(".struct_ops")
59struct bpf_testmod_ops3 testmod_1 = {
60 .test_1 = (void *)test_1,
61 .test_2 = (void *)test_2,
62};
tools/testing/selftests/bpf/progs/struct_ops_private_stack_fail.c+62−0
@@ -0,0 +1,62 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6#include "../bpf_testmod/bpf_testmod.h"
7
8char _license[] SEC("license") = "GPL";
9
10#if defined(__TARGET_ARCH_x86)
11bool skip __attribute((__section__(".data"))) = false;
12#else
13bool skip = true;
14#endif
15
16void bpf_testmod_ops3_call_test_2(void) __ksym;
17
18int val_i, val_j;
19
20__noinline static int subprog2(int *a, int *b)
21{
22 return val_i + a[10] + b[20];
23}
24
25__noinline static int subprog1(int *a)
26{
27 /* stack size 200 bytes */
28 int b[50] = {};
29
30 b[20] = 2;
31 return subprog2(a, b);
32}
33
34
35SEC("struct_ops")
36int BPF_PROG(test_1)
37{
38 /* stack size 100 bytes */
39 int a[25] = {};
40
41 a[10] = 1;
42 val_i = subprog1(a);
43 bpf_testmod_ops3_call_test_2();
44 return 0;
45}
46
47SEC("struct_ops")
48int BPF_PROG(test_2)
49{
50 /* stack size 400 bytes */
51 int a[100] = {};
52
53 a[10] = 3;
54 val_j = subprog1(a);
55 return 0;
56}
57
58SEC(".struct_ops")
59struct bpf_testmod_ops3 testmod_1 = {
60 .test_1 = (void *)test_1,
61 .test_2 = (void *)test_2,
62};
tools/testing/selftests/bpf/progs/struct_ops_private_stack_recur.c+50−0
@@ -0,0 +1,50 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6#include "../bpf_testmod/bpf_testmod.h"
7
8char _license[] SEC("license") = "GPL";
9
10#if defined(__TARGET_ARCH_x86)
11bool skip __attribute((__section__(".data"))) = false;
12#else
13bool skip = true;
14#endif
15
16void bpf_testmod_ops3_call_test_1(void) __ksym;
17
18int val_i, val_j;
19
20__noinline static int subprog2(int *a, int *b)
21{
22 return val_i + a[1] + b[20];
23}
24
25__noinline static int subprog1(int *a)
26{
27 /* stack size 400 bytes */
28 int b[100] = {};
29
30 b[20] = 2;
31 return subprog2(a, b);
32}
33
34
35SEC("struct_ops")
36int BPF_PROG(test_1)
37{
38 /* stack size 20 bytes */
39 int a[5] = {};
40
41 a[1] = 1;
42 val_j += subprog1(a);
43 bpf_testmod_ops3_call_test_1();
44 return 0;
45}
46
47SEC(".struct_ops")
48struct bpf_testmod_ops3 testmod_1 = {
49 .test_1 = (void *)test_1,
50};
tools/testing/selftests/bpf/progs/tailcall_fail.c+64−0
@@ -0,0 +1,64 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3#include <vmlinux.h>
4#include <bpf/bpf_tracing.h>
5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_core_read.h>
7
8#include "bpf_misc.h"
9#include "bpf_experimental.h"
10
11extern void bpf_rcu_read_lock(void) __ksym;
12extern void bpf_rcu_read_unlock(void) __ksym;
13
14#define private(name) SEC(".bss." #name) __hidden __attribute__((aligned(8)))
15
16private(A) struct bpf_spin_lock lock;
17
18struct {
19 __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
20 __uint(max_entries, 3);
21 __uint(key_size, sizeof(__u32));
22 __uint(value_size, sizeof(__u32));
23} jmp_table SEC(".maps");
24
25SEC("?tc")
26__failure __msg("function calls are not allowed while holding a lock")
27int reject_tail_call_spin_lock(struct __sk_buff *ctx)
28{
29 bpf_spin_lock(&lock);
30 bpf_tail_call_static(ctx, &jmp_table, 0);
31 return 0;
32}
33
34SEC("?tc")
35__failure __msg("tail_call cannot be used inside bpf_rcu_read_lock-ed region")
36int reject_tail_call_rcu_lock(struct __sk_buff *ctx)
37{
38 bpf_rcu_read_lock();
39 bpf_tail_call_static(ctx, &jmp_table, 0);
40 bpf_rcu_read_unlock();
41 return 0;
42}
43
44SEC("?tc")
45__failure __msg("tail_call cannot be used inside bpf_preempt_disable-ed region")
46int reject_tail_call_preempt_lock(struct __sk_buff *ctx)
47{
48 bpf_guard_preempt();
49 bpf_tail_call_static(ctx, &jmp_table, 0);
50 return 0;
51}
52
53SEC("?tc")
54__failure __msg("tail_call would lead to reference leak")
55int reject_tail_call_ref(struct __sk_buff *ctx)
56{
57 struct foo { int i; } *p;
58
59 p = bpf_obj_new(typeof(*p));
60 bpf_tail_call_static(ctx, &jmp_table, 0);
61 return 0;
62}
63
64char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/task_kfunc_common.h+1−0
@@ -23,6 +23,7 @@ struct { · 22 unmodified lines
23struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;23struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
24void bpf_task_release(struct task_struct *p) __ksym;24void bpf_task_release(struct task_struct *p) __ksym;
25struct task_struct *bpf_task_from_pid(s32 pid) __ksym;25struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
26struct task_struct *bpf_task_from_vpid(s32 vpid) __ksym;
26void bpf_rcu_read_lock(void) __ksym;27void bpf_rcu_read_lock(void) __ksym;
27void bpf_rcu_read_unlock(void) __ksym;28void bpf_rcu_read_unlock(void) __ksym;
28 29
tools/testing/selftests/bpf/progs/task_kfunc_failure.c+14−0
@@ -247,6 +247,20 @@ int BPF_PROG(task_kfunc_from_pid_no_null_check, struct task_struct *task, u64 cl · 246 unmodified lines
247 return 0;247 return 0;
248}248}
249 249
250SEC("tp_btf/task_newtask")
251__failure __msg("Possibly NULL pointer passed to trusted arg0")
252int BPF_PROG(task_kfunc_from_vpid_no_null_check, struct task_struct *task, u64 clone_flags)
253{
254 struct task_struct *acquired;
255
256 acquired = bpf_task_from_vpid(task->pid);
257
258 /* Releasing bpf_task_from_vpid() lookup without a NULL check. */
259 bpf_task_release(acquired);
260
261 return 0;
262}
263
250SEC("lsm/task_free")264SEC("lsm/task_free")
251__failure __msg("R1 must be a rcu pointer")265__failure __msg("R1 must be a rcu pointer")
252int BPF_PROG(task_kfunc_from_lsm_task_free, struct task_struct *task)266int BPF_PROG(task_kfunc_from_lsm_task_free, struct task_struct *task)
tools/testing/selftests/bpf/progs/task_kfunc_success.c+51−0
@@ -366,3 +366,54 @@ int BPF_PROG(task_kfunc_acquire_trusted_walked, struct task_struct *task, u64 cl · 365 unmodified lines
366 366
367 return 0;367 return 0;
368}368}
369
370SEC("syscall")
371int test_task_from_vpid_current(const void *ctx)
372{
373 struct task_struct *current, *v_task;
374
375 v_task = bpf_task_from_vpid(1);
376 if (!v_task) {
377 err = 1;
378 return 0;
379 }
380
381 current = bpf_get_current_task_btf();
382
383 /* The current process should be the init process (pid 1) in the new pid namespace. */
384 if (current != v_task)
385 err = 2;
386
387 bpf_task_release(v_task);
388 return 0;
389}
390
391SEC("syscall")
392int test_task_from_vpid_invalid(const void *ctx)
393{
394 struct task_struct *v_task;
395
396 v_task = bpf_task_from_vpid(-1);
397 if (v_task) {
398 err = 1;
399 goto err;
400 }
401
402 /* There should be only one process (current process) in the new pid namespace. */
403 v_task = bpf_task_from_vpid(2);
404 if (v_task) {
405 err = 2;
406 goto err;
407 }
408
409 v_task = bpf_task_from_vpid(9999);
410 if (v_task) {
411 err = 3;
412 goto err;
413 }
414
415 return 0;
416err:
417 bpf_task_release(v_task);
418 return 0;
419}
tools/testing/selftests/bpf/progs/task_ls_uptr.c+63−0
@@ -0,0 +1,63 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#include <vmlinux.h>
5#include <bpf/bpf_helpers.h>
6#include "uptr_test_common.h"
7
8struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
9void bpf_task_release(struct task_struct *p) __ksym;
10void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
11
12struct {
13 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
14 __uint(map_flags, BPF_F_NO_PREALLOC);
15 __type(key, int);
16 __type(value, struct value_type);
17} datamap SEC(".maps");
18
19pid_t target_pid = 0;
20pid_t parent_pid = 0;
21
22SEC("tp_btf/sys_enter")
23int on_enter(__u64 *ctx)
24{
25 struct task_struct *task, *data_task;
26 struct value_type *ptr;
27 struct user_data *udata;
28 struct cgroup *cgrp;
29
30 task = bpf_get_current_task_btf();
31 if (task->pid != target_pid)
32 return 0;
33
34 data_task = bpf_task_from_pid(parent_pid);
35 if (!data_task)
36 return 0;
37
38 ptr = bpf_task_storage_get(&datamap, data_task, 0, 0);
39 bpf_task_release(data_task);
40 if (!ptr)
41 return 0;
42
43 cgrp = bpf_kptr_xchg(&ptr->cgrp, NULL);
44 if (cgrp) {
45 int lvl = cgrp->level;
46
47 bpf_cgroup_release(cgrp);
48 return lvl;
49 }
50
51 udata = ptr->udata;
52 if (!udata || udata->result)
53 return 0;
54 udata->result = MAGIC_VALUE + udata->a + udata->b;
55
56 udata = ptr->nested.udata;
57 if (udata && !udata->nested_result)
58 udata->nested_result = udata->result;
59
60 return 0;
61}
62
63char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/tc_bpf2bpf.c+3−2
@@ -5,10 +5,11 @@ · 4 unmodified lines
5#include "bpf_misc.h"5#include "bpf_misc.h"
6 6
7__noinline7__noinline
8int subprog(struct __sk_buff *skb)8int subprog_tc(struct __sk_buff *skb)
9{9{
10 int ret = 1;10 int ret = 1;
11 11
12 __sink(skb);
12 __sink(ret);13 __sink(ret);
13 return ret;14 return ret;
14}15}
@@ -16,7 +17,7 @@ int subprog(struct __sk_buff *skb) · 1 unmodified line
16SEC("tc")17SEC("tc")
17int entry_tc(struct __sk_buff *skb)18int entry_tc(struct __sk_buff *skb)
18{19{
19 return subprog(skb);20 return subprog_tc(skb);
20}21}
21 22
22char __license[] SEC("license") = "GPL";23char __license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/test_send_signal_kern.c+29−6
@@ -1,27 +1,50 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2019 Facebook2// Copyright (c) 2019 Facebook
3#include <linux/bpf.h>3#include <vmlinux.h>
4#include <linux/version.h>4#include <linux/version.h>
5#include <bpf/bpf_helpers.h>5#include <bpf/bpf_helpers.h>
6 6
7__u32 sig = 0, pid = 0, status = 0, signal_thread = 0;7struct task_struct *bpf_task_from_pid(int pid) __ksym;
8void bpf_task_release(struct task_struct *p) __ksym;
9int bpf_send_signal_task(struct task_struct *task, int sig, enum pid_type type, u64 value) __ksym;
10
11__u32 sig = 0, pid = 0, status = 0, signal_thread = 0, target_pid = 0;
8 12
9static __always_inline int bpf_send_signal_test(void *ctx)13static __always_inline int bpf_send_signal_test(void *ctx)
10{14{
15 struct task_struct *target_task = NULL;
11 int ret;16 int ret;
17 u64 value;
12 18
13 if (status != 0 || pid == 0)19 if (status != 0 || pid == 0)
14 return 0;20 return 0;
15 21
16 if ((bpf_get_current_pid_tgid() >> 32) == pid) {22 if ((bpf_get_current_pid_tgid() >> 32) == pid) {
17 if (signal_thread)23 if (target_pid) {
18 ret = bpf_send_signal_thread(sig);24 target_task = bpf_task_from_pid(target_pid);
19 else25 if (!target_task)
20 ret = bpf_send_signal(sig);26 return 0;
27 value = 8;
28 }
29
30 if (signal_thread) {
31 if (target_pid)
32 ret = bpf_send_signal_task(target_task, sig, PIDTYPE_PID, value);
33 else
34 ret = bpf_send_signal_thread(sig);
35 } else {
36 if (target_pid)
37 ret = bpf_send_signal_task(target_task, sig, PIDTYPE_TGID, value);
38 else
39 ret = bpf_send_signal(sig);
40 }
21 if (ret == 0)41 if (ret == 0)
22 status = 1;42 status = 1;
23 }43 }
24 44
45 if (target_task)
46 bpf_task_release(target_task);
47
25 return 0;48 return 0;
26}49}
27 50
tools/testing/selftests/bpf/progs/test_spin_lock_fail.c+2−2
@@ -28,8 +28,8 @@ struct { · 27 unmodified lines
28 },28 },
29};29};
30 30
31SEC(".data.A") struct bpf_spin_lock lockA;31static struct bpf_spin_lock lockA SEC(".data.A");
32SEC(".data.B") struct bpf_spin_lock lockB;32static struct bpf_spin_lock lockB SEC(".data.B");
33 33
34SEC("?tc")34SEC("?tc")
35int lock_id_kptr_preserve(void *ctx)35int lock_id_kptr_preserve(void *ctx)
tools/testing/selftests/bpf/progs/test_tp_btf_nullable.c+5−1
@@ -7,7 +7,11 @@ · 6 unmodified lines
7#include "bpf_misc.h"7#include "bpf_misc.h"
8 8
9SEC("tp_btf/bpf_testmod_test_nullable_bare")9SEC("tp_btf/bpf_testmod_test_nullable_bare")
10__failure __msg("R1 invalid mem access 'trusted_ptr_or_null_'")10/* This used to be a failure test, but raw_tp nullable arguments can now
11 * directly be dereferenced, whether they have nullable annotation or not,
12 * and don't need to be explicitly checked.
13 */
14__success
11int BPF_PROG(handle_tp_btf_nullable_bare1, struct bpf_testmod_test_read_ctx *nullable_ctx)15int BPF_PROG(handle_tp_btf_nullable_bare1, struct bpf_testmod_test_read_ctx *nullable_ctx)
12{16{
13 return nullable_ctx->len;17 return nullable_ctx->len;
tools/testing/selftests/bpf/progs/update_map_in_htab.c+30−0
@@ -0,0 +1,30 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (C) 2024. Huawei Technologies Co., Ltd */
3#include <linux/bpf.h>
4#include <bpf/bpf_helpers.h>
5
6struct inner_map_type {
7 __uint(type, BPF_MAP_TYPE_ARRAY);
8 __uint(key_size, 4);
9 __uint(value_size, 4);
10 __uint(max_entries, 1);
11} inner_map SEC(".maps");
12
13struct {
14 __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
15 __type(key, int);
16 __type(value, int);
17 __uint(max_entries, 2);
18 __array(values, struct inner_map_type);
19} outer_htab_map SEC(".maps");
20
21struct {
22 __uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
23 __uint(map_flags, BPF_F_NO_PREALLOC);
24 __type(key, int);
25 __type(value, int);
26 __uint(max_entries, 2);
27 __array(values, struct inner_map_type);
28} outer_alloc_htab_map SEC(".maps");
29
30char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/uprobe_multi_consumers.c+3−3
@@ -24,16 +24,16 @@ int uprobe_1(struct pt_regs *ctx) · 23 unmodified lines
24 return 0;24 return 0;
25}25}
26 26
27SEC("uprobe.multi")27SEC("uprobe.session")
28int uprobe_2(struct pt_regs *ctx)28int uprobe_2(struct pt_regs *ctx)
29{29{
30 uprobe_result[2]++;30 uprobe_result[2]++;
31 return 0;31 return 0;
32}32}
33 33
34SEC("uprobe.multi")34SEC("uprobe.session")
35int uprobe_3(struct pt_regs *ctx)35int uprobe_3(struct pt_regs *ctx)
36{36{
37 uprobe_result[3]++;37 uprobe_result[3]++;
38 return 0;38 return 1;
39}39}
tools/testing/selftests/bpf/progs/uprobe_multi_session.c+71−0
@@ -0,0 +1,71 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bpf.h>
3#include <bpf/bpf_helpers.h>
4#include <bpf/bpf_tracing.h>
5#include <stdbool.h>
6#include "bpf_kfuncs.h"
7#include "bpf_misc.h"
8
9char _license[] SEC("license") = "GPL";
10
11__u64 uprobe_multi_func_1_addr = 0;
12__u64 uprobe_multi_func_2_addr = 0;
13__u64 uprobe_multi_func_3_addr = 0;
14
15__u64 uprobe_session_result[3] = {};
16__u64 uprobe_multi_sleep_result = 0;
17
18void *user_ptr = 0;
19int pid = 0;
20
21static int uprobe_multi_check(void *ctx, bool is_return)
22{
23 const __u64 funcs[] = {
24 uprobe_multi_func_1_addr,
25 uprobe_multi_func_2_addr,
26 uprobe_multi_func_3_addr,
27 };
28 unsigned int i;
29 __u64 addr;
30
31 if (bpf_get_current_pid_tgid() >> 32 != pid)
32 return 1;
33
34 addr = bpf_get_func_ip(ctx);
35
36 for (i = 0; i < ARRAY_SIZE(funcs); i++) {
37 if (funcs[i] == addr) {
38 uprobe_session_result[i]++;
39 break;
40 }
41 }
42
43 /* only uprobe_multi_func_2 executes return probe */
44 if ((addr == uprobe_multi_func_1_addr) ||
45 (addr == uprobe_multi_func_3_addr))
46 return 1;
47
48 return 0;
49}
50
51SEC("uprobe.session//proc/self/exe:uprobe_multi_func_*")
52int uprobe(struct pt_regs *ctx)
53{
54 return uprobe_multi_check(ctx, bpf_session_is_return());
55}
56
57static __always_inline bool verify_sleepable_user_copy(void)
58{
59 char data[9];
60
61 bpf_copy_from_user(data, sizeof(data), user_ptr);
62 return bpf_strncmp(data, sizeof(data), "test_data") == 0;
63}
64
65SEC("uprobe.session.s//proc/self/exe:uprobe_multi_func_*")
66int uprobe_sleepable(struct pt_regs *ctx)
67{
68 if (verify_sleepable_user_copy())
69 uprobe_multi_sleep_result++;
70 return uprobe_multi_check(ctx, bpf_session_is_return());
71}
tools/testing/selftests/bpf/progs/uprobe_multi_session_cookie.c+48−0
@@ -0,0 +1,48 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bpf.h>
3#include <bpf/bpf_helpers.h>
4#include <bpf/bpf_tracing.h>
5#include <stdbool.h>
6#include "bpf_kfuncs.h"
7
8char _license[] SEC("license") = "GPL";
9
10int pid = 0;
11
12__u64 test_uprobe_1_result = 0;
13__u64 test_uprobe_2_result = 0;
14__u64 test_uprobe_3_result = 0;
15
16static int check_cookie(__u64 val, __u64 *result)
17{
18 __u64 *cookie;
19
20 if (bpf_get_current_pid_tgid() >> 32 != pid)
21 return 1;
22
23 cookie = bpf_session_cookie();
24
25 if (bpf_session_is_return())
26 *result = *cookie == val ? val : 0;
27 else
28 *cookie = val;
29 return 0;
30}
31
32SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
33int uprobe_1(struct pt_regs *ctx)
34{
35 return check_cookie(1, &test_uprobe_1_result);
36}
37
38SEC("uprobe.session//proc/self/exe:uprobe_multi_func_2")
39int uprobe_2(struct pt_regs *ctx)
40{
41 return check_cookie(2, &test_uprobe_2_result);
42}
43
44SEC("uprobe.session//proc/self/exe:uprobe_multi_func_3")
45int uprobe_3(struct pt_regs *ctx)
46{
47 return check_cookie(3, &test_uprobe_3_result);
48}
tools/testing/selftests/bpf/progs/uprobe_multi_session_recursive.c+44−0
@@ -0,0 +1,44 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bpf.h>
3#include <bpf/bpf_helpers.h>
4#include <bpf/bpf_tracing.h>
5#include <stdbool.h>
6#include "bpf_kfuncs.h"
7#include "bpf_misc.h"
8
9char _license[] SEC("license") = "GPL";
10
11int pid = 0;
12
13int idx_entry = 0;
14int idx_return = 0;
15
16__u64 test_uprobe_cookie_entry[6];
17__u64 test_uprobe_cookie_return[3];
18
19static int check_cookie(void)
20{
21 __u64 *cookie = bpf_session_cookie();
22
23 if (bpf_session_is_return()) {
24 if (idx_return >= ARRAY_SIZE(test_uprobe_cookie_return))
25 return 1;
26 test_uprobe_cookie_return[idx_return++] = *cookie;
27 return 0;
28 }
29
30 if (idx_entry >= ARRAY_SIZE(test_uprobe_cookie_entry))
31 return 1;
32 *cookie = test_uprobe_cookie_entry[idx_entry];
33 return idx_entry++ % 2;
34}
35
36
37SEC("uprobe.session//proc/self/exe:uprobe_session_recursive")
38int uprobe_recursive(struct pt_regs *ctx)
39{
40 if (bpf_get_current_pid_tgid() >> 32 != pid)
41 return 1;
42
43 return check_cookie();
44}
tools/testing/selftests/bpf/progs/uprobe_multi_session_single.c+44−0
@@ -0,0 +1,44 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bpf.h>
3#include <bpf/bpf_helpers.h>
4#include <bpf/bpf_tracing.h>
5#include <stdbool.h>
6#include "bpf_kfuncs.h"
7#include "bpf_misc.h"
8
9char _license[] SEC("license") = "GPL";
10
11__u64 uprobe_session_result[3] = {};
12int pid = 0;
13
14static int uprobe_multi_check(void *ctx, int idx)
15{
16 if (bpf_get_current_pid_tgid() >> 32 != pid)
17 return 1;
18
19 uprobe_session_result[idx]++;
20
21 /* only consumer 1 executes return probe */
22 if (idx == 0 || idx == 2)
23 return 1;
24
25 return 0;
26}
27
28SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
29int uprobe_0(struct pt_regs *ctx)
30{
31 return uprobe_multi_check(ctx, 0);
32}
33
34SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
35int uprobe_1(struct pt_regs *ctx)
36{
37 return uprobe_multi_check(ctx, 1);
38}
39
40SEC("uprobe.session//proc/self/exe:uprobe_multi_func_1")
41int uprobe_2(struct pt_regs *ctx)
42{
43 return uprobe_multi_check(ctx, 2);
44}
tools/testing/selftests/bpf/progs/uprobe_multi_verifier.c+31−0
@@ -0,0 +1,31 @@
1// SPDX-License-Identifier: GPL-2.0
2#include "vmlinux.h"
3#include <bpf/bpf_helpers.h>
4#include <bpf/bpf_tracing.h>
5#include <bpf/usdt.bpf.h>
6#include "bpf_misc.h"
7
8char _license[] SEC("license") = "GPL";
9
10
11SEC("uprobe.session")
12__success
13int uprobe_sesison_return_0(struct pt_regs *ctx)
14{
15 return 0;
16}
17
18SEC("uprobe.session")
19__success
20int uprobe_sesison_return_1(struct pt_regs *ctx)
21{
22 return 1;
23}
24
25SEC("uprobe.session")
26__failure
27__msg("At program exit the register R0 has smin=2 smax=2 should have been in [0, 1]")
28int uprobe_sesison_return_2(struct pt_regs *ctx)
29{
30 return 2;
31}
tools/testing/selftests/bpf/progs/uptr_failure.c+105−0
@@ -0,0 +1,105 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#include <vmlinux.h>
5#include <bpf/bpf_helpers.h>
6#include "bpf_experimental.h"
7#include "bpf_misc.h"
8#include "uptr_test_common.h"
9
10struct {
11 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
12 __uint(map_flags, BPF_F_NO_PREALLOC);
13 __type(key, int);
14 __type(value, struct value_type);
15} datamap SEC(".maps");
16
17SEC("?syscall")
18__failure __msg("store to uptr disallowed")
19int uptr_write(const void *ctx)
20{
21 struct task_struct *task;
22 struct value_type *v;
23
24 task = bpf_get_current_task_btf();
25 v = bpf_task_storage_get(&datamap, task, 0,
26 BPF_LOCAL_STORAGE_GET_F_CREATE);
27 if (!v)
28 return 0;
29
30 v->udata = NULL;
31 return 0;
32}
33
34SEC("?syscall")
35__failure __msg("store to uptr disallowed")
36int uptr_write_nested(const void *ctx)
37{
38 struct task_struct *task;
39 struct value_type *v;
40
41 task = bpf_get_current_task_btf();
42 v = bpf_task_storage_get(&datamap, task, 0,
43 BPF_LOCAL_STORAGE_GET_F_CREATE);
44 if (!v)
45 return 0;
46
47 v->nested.udata = NULL;
48 return 0;
49}
50
51SEC("?syscall")
52__failure __msg("R1 invalid mem access 'mem_or_null'")
53int uptr_no_null_check(const void *ctx)
54{
55 struct task_struct *task;
56 struct value_type *v;
57
58 task = bpf_get_current_task_btf();
59 v = bpf_task_storage_get(&datamap, task, 0,
60 BPF_LOCAL_STORAGE_GET_F_CREATE);
61 if (!v)
62 return 0;
63
64 v->udata->result = 0;
65
66 return 0;
67}
68
69SEC("?syscall")
70__failure __msg("doesn't point to kptr")
71int uptr_kptr_xchg(const void *ctx)
72{
73 struct task_struct *task;
74 struct value_type *v;
75
76 task = bpf_get_current_task_btf();
77 v = bpf_task_storage_get(&datamap, task, 0,
78 BPF_LOCAL_STORAGE_GET_F_CREATE);
79 if (!v)
80 return 0;
81
82 bpf_kptr_xchg(&v->udata, NULL);
83
84 return 0;
85}
86
87SEC("?syscall")
88__failure __msg("invalid mem access 'scalar'")
89int uptr_obj_new(const void *ctx)
90{
91 struct value_type *v;
92
93 v = bpf_obj_new(typeof(*v));
94 if (!v)
95 return 0;
96
97 if (v->udata)
98 v->udata->result = 0;
99
100 bpf_obj_drop(v);
101
102 return 0;
103}
104
105char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/uptr_map_failure.c+27−0
@@ -0,0 +1,27 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#include <vmlinux.h>
5#include <bpf/bpf_helpers.h>
6#include "uptr_test_common.h"
7
8struct {
9 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
10 __uint(map_flags, BPF_F_NO_PREALLOC);
11 __type(key, int);
12 __type(value, struct large_uptr);
13} large_uptr_map SEC(".maps");
14
15struct {
16 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
17 __uint(map_flags, BPF_F_NO_PREALLOC);
18 __type(key, int);
19 __type(value, struct empty_uptr);
20} empty_uptr_map SEC(".maps");
21
22struct {
23 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
24 __uint(map_flags, BPF_F_NO_PREALLOC);
25 __type(key, int);
26 __type(value, struct kstruct_uptr);
27} kstruct_uptr_map SEC(".maps");
tools/testing/selftests/bpf/progs/uptr_update_failure.c+42−0
@@ -0,0 +1,42 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#include <vmlinux.h>
5#include <bpf/bpf_helpers.h>
6#include "uptr_test_common.h"
7
8struct {
9 __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
10 __uint(map_flags, BPF_F_NO_PREALLOC);
11 __type(key, int);
12 __type(value, struct value_lock_type);
13} datamap SEC(".maps");
14
15/* load test only. not used */
16SEC("syscall")
17int not_used(void *ctx)
18{
19 struct value_lock_type *ptr;
20 struct task_struct *task;
21 struct user_data *udata;
22
23 task = bpf_get_current_task_btf();
24 ptr = bpf_task_storage_get(&datamap, task, 0, 0);
25 if (!ptr)
26 return 0;
27
28 bpf_spin_lock(&ptr->lock);
29
30 udata = ptr->udata;
31 if (!udata) {
32 bpf_spin_unlock(&ptr->lock);
33 return 0;
34 }
35 udata->result = MAGIC_VALUE + udata->a + udata->b;
36
37 bpf_spin_unlock(&ptr->lock);
38
39 return 0;
40}
41
42char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_arena_large.c+108−2
@@ -29,12 +29,12 @@ int big_alloc1(void *ctx) · 28 unmodified lines
29 if (!page1)29 if (!page1)
30 return 1;30 return 1;
31 *page1 = 1;31 *page1 = 1;
32 page2 = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE,32 page2 = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE * 2,
33 1, NUMA_NO_NODE, 0);33 1, NUMA_NO_NODE, 0);
34 if (!page2)34 if (!page2)
35 return 2;35 return 2;
36 *page2 = 2;36 *page2 = 2;
37 no_page = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE,37 no_page = bpf_arena_alloc_pages(&arena, base + ARENA_SIZE - PAGE_SIZE,
38 1, NUMA_NO_NODE, 0);38 1, NUMA_NO_NODE, 0);
39 if (no_page)39 if (no_page)
40 return 3;40 return 3;
@@ -66,4 +66,110 @@ int big_alloc1(void *ctx) · 25 unmodified lines
66#endif66#endif
67 return 0;67 return 0;
68}68}
69
70#if defined(__BPF_FEATURE_ADDR_SPACE_CAST)
71#define PAGE_CNT 100
72__u8 __arena * __arena page[PAGE_CNT]; /* occupies the first page */
73__u8 __arena *base;
74
75/*
76 * Check that arena's range_tree algorithm allocates pages sequentially
77 * on the first pass and then fills in all gaps on the second pass.
78 */
79__noinline int alloc_pages(int page_cnt, int pages_atonce, bool first_pass,
80 int max_idx, int step)
81{
82 __u8 __arena *pg;
83 int i, pg_idx;
84
85 for (i = 0; i < page_cnt; i++) {
86 pg = bpf_arena_alloc_pages(&arena, NULL, pages_atonce,
87 NUMA_NO_NODE, 0);
88 if (!pg)
89 return step;
90 pg_idx = (unsigned long) (pg - base) / PAGE_SIZE;
91 if (first_pass) {
92 /* Pages must be allocated sequentially */
93 if (pg_idx != i)
94 return step + 100;
95 } else {
96 /* Allocator must fill into gaps */
97 if (pg_idx >= max_idx || (pg_idx & 1))
98 return step + 200;
99 }
100 *pg = pg_idx;
101 page[pg_idx] = pg;
102 cond_break;
103 }
104 return 0;
105}
106
107SEC("syscall")
108__success __retval(0)
109int big_alloc2(void *ctx)
110{
111 __u8 __arena *pg;
112 int i, err;
113
114 base = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
115 if (!base)
116 return 1;
117 bpf_arena_free_pages(&arena, (void __arena *)base, 1);
118
119 err = alloc_pages(PAGE_CNT, 1, true, PAGE_CNT, 2);
120 if (err)
121 return err;
122
123 /* Clear all even pages */
124 for (i = 0; i < PAGE_CNT; i += 2) {
125 pg = page[i];
126 if (*pg != i)
127 return 3;
128 bpf_arena_free_pages(&arena, (void __arena *)pg, 1);
129 page[i] = NULL;
130 cond_break;
131 }
132
133 /* Allocate into freed gaps */
134 err = alloc_pages(PAGE_CNT / 2, 1, false, PAGE_CNT, 4);
135 if (err)
136 return err;
137
138 /* Free pairs of pages */
139 for (i = 0; i < PAGE_CNT; i += 4) {
140 pg = page[i];
141 if (*pg != i)
142 return 5;
143 bpf_arena_free_pages(&arena, (void __arena *)pg, 2);
144 page[i] = NULL;
145 page[i + 1] = NULL;
146 cond_break;
147 }
148
149 /* Allocate 2 pages at a time into freed gaps */
150 err = alloc_pages(PAGE_CNT / 4, 2, false, PAGE_CNT, 6);
151 if (err)
152 return err;
153
154 /* Check pages without freeing */
155 for (i = 0; i < PAGE_CNT; i += 2) {
156 pg = page[i];
157 if (*pg != i)
158 return 7;
159 cond_break;
160 }
161
162 pg = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0);
163
164 if (!pg)
165 return 8;
166 /*
167 * The first PAGE_CNT pages are occupied. The new page
168 * must be above.
169 */
170 if ((pg - base) / PAGE_SIZE < PAGE_CNT)
171 return 9;
172 return 0;
173}
174#endif
69char _license[] SEC("license") = "GPL";175char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_private_stack.c+272−0
@@ -0,0 +1,272 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <vmlinux.h>
4#include <bpf/bpf_helpers.h>
5#include "bpf_misc.h"
6#include "bpf_experimental.h"
7
8/* From include/linux/filter.h */
9#define MAX_BPF_STACK 512
10
11#if defined(__TARGET_ARCH_x86)
12
13struct elem {
14 struct bpf_timer t;
15 char pad[256];
16};
17
18struct {
19 __uint(type, BPF_MAP_TYPE_ARRAY);
20 __uint(max_entries, 1);
21 __type(key, int);
22 __type(value, struct elem);
23} array SEC(".maps");
24
25SEC("kprobe")
26__description("Private stack, single prog")
27__success
28__arch_x86_64
29__jited(" movabsq $0x{{.*}}, %r9")
30__jited(" addq %gs:0x{{.*}}, %r9")
31__jited(" movl $0x2a, %edi")
32__jited(" movq %rdi, -0x100(%r9)")
33__naked void private_stack_single_prog(void)
34{
35 asm volatile (" \
36 r1 = 42; \
37 *(u64 *)(r10 - 256) = r1; \
38 r0 = 0; \
39 exit; \
40" ::: __clobber_all);
41}
42
43SEC("raw_tp")
44__description("No private stack")
45__success
46__arch_x86_64
47__jited(" subq $0x8, %rsp")
48__naked void no_private_stack_nested(void)
49{
50 asm volatile (" \
51 r1 = 42; \
52 *(u64 *)(r10 - 8) = r1; \
53 r0 = 0; \
54 exit; \
55" ::: __clobber_all);
56}
57
58__used
59__naked static void cumulative_stack_depth_subprog(void)
60{
61 asm volatile (" \
62 r1 = 41; \
63 *(u64 *)(r10 - 32) = r1; \
64 call %[bpf_get_smp_processor_id]; \
65 exit; \
66" :
67 : __imm(bpf_get_smp_processor_id)
68 : __clobber_all);
69}
70
71SEC("kprobe")
72__description("Private stack, subtree > MAX_BPF_STACK")
73__success
74__arch_x86_64
75/* private stack fp for the main prog */
76__jited(" movabsq $0x{{.*}}, %r9")
77__jited(" addq %gs:0x{{.*}}, %r9")
78__jited(" movl $0x2a, %edi")
79__jited(" movq %rdi, -0x200(%r9)")
80__jited(" pushq %r9")
81__jited(" callq 0x{{.*}}")
82__jited(" popq %r9")
83__jited(" xorl %eax, %eax")
84__naked void private_stack_nested_1(void)
85{
86 asm volatile (" \
87 r1 = 42; \
88 *(u64 *)(r10 - %[max_bpf_stack]) = r1; \
89 call cumulative_stack_depth_subprog; \
90 r0 = 0; \
91 exit; \
92" :
93 : __imm_const(max_bpf_stack, MAX_BPF_STACK)
94 : __clobber_all);
95}
96
97__naked __noinline __used
98static unsigned long loop_callback(void)
99{
100 asm volatile (" \
101 call %[bpf_get_prandom_u32]; \
102 r1 = 42; \
103 *(u64 *)(r10 - 512) = r1; \
104 call cumulative_stack_depth_subprog; \
105 r0 = 0; \
106 exit; \
107" :
108 : __imm(bpf_get_prandom_u32)
109 : __clobber_common);
110}
111
112SEC("raw_tp")
113__description("Private stack, callback")
114__success
115__arch_x86_64
116/* for func loop_callback */
117__jited("func #1")
118__jited(" endbr64")
119__jited(" nopl (%rax,%rax)")
120__jited(" nopl (%rax)")
121__jited(" pushq %rbp")
122__jited(" movq %rsp, %rbp")
123__jited(" endbr64")
124__jited(" movabsq $0x{{.*}}, %r9")
125__jited(" addq %gs:0x{{.*}}, %r9")
126__jited(" pushq %r9")
127__jited(" callq")
128__jited(" popq %r9")
129__jited(" movl $0x2a, %edi")
130__jited(" movq %rdi, -0x200(%r9)")
131__jited(" pushq %r9")
132__jited(" callq")
133__jited(" popq %r9")
134__naked void private_stack_callback(void)
135{
136 asm volatile (" \
137 r1 = 1; \
138 r2 = %[loop_callback]; \
139 r3 = 0; \
140 r4 = 0; \
141 call %[bpf_loop]; \
142 r0 = 0; \
143 exit; \
144" :
145 : __imm_ptr(loop_callback),
146 __imm(bpf_loop)
147 : __clobber_common);
148}
149
150SEC("fentry/bpf_fentry_test9")
151__description("Private stack, exception in main prog")
152__success __retval(0)
153__arch_x86_64
154__jited(" pushq %r9")
155__jited(" callq")
156__jited(" popq %r9")
157int private_stack_exception_main_prog(void)
158{
159 asm volatile (" \
160 r1 = 42; \
161 *(u64 *)(r10 - 512) = r1; \
162" ::: __clobber_common);
163
164 bpf_throw(0);
165 return 0;
166}
167
168__used static int subprog_exception(void)
169{
170 bpf_throw(0);
171 return 0;
172}
173
174SEC("fentry/bpf_fentry_test9")
175__description("Private stack, exception in subprog")
176__success __retval(0)
177__arch_x86_64
178__jited(" movq %rdi, -0x200(%r9)")
179__jited(" pushq %r9")
180__jited(" callq")
181__jited(" popq %r9")
182int private_stack_exception_sub_prog(void)
183{
184 asm volatile (" \
185 r1 = 42; \
186 *(u64 *)(r10 - 512) = r1; \
187 call subprog_exception; \
188" ::: __clobber_common);
189
190 return 0;
191}
192
193int glob;
194__noinline static void subprog2(int *val)
195{
196 glob += val[0] * 2;
197}
198
199__noinline static void subprog1(int *val)
200{
201 int tmp[64] = {};
202
203 tmp[0] = *val;
204 subprog2(tmp);
205}
206
207__noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer)
208{
209 subprog1(key);
210 return 0;
211}
212
213__noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer)
214{
215 return 0;
216}
217
218SEC("fentry/bpf_fentry_test9")
219__description("Private stack, async callback, not nested")
220__success __retval(0)
221__arch_x86_64
222__jited(" movabsq $0x{{.*}}, %r9")
223int private_stack_async_callback_1(void)
224{
225 struct bpf_timer *arr_timer;
226 int array_key = 0;
227
228 arr_timer = bpf_map_lookup_elem(&array, &array_key);
229 if (!arr_timer)
230 return 0;
231
232 bpf_timer_init(arr_timer, &array, 1);
233 bpf_timer_set_callback(arr_timer, timer_cb2);
234 bpf_timer_start(arr_timer, 0, 0);
235 subprog1(&array_key);
236 return 0;
237}
238
239SEC("fentry/bpf_fentry_test9")
240__description("Private stack, async callback, potential nesting")
241__success __retval(0)
242__arch_x86_64
243__jited(" subq $0x100, %rsp")
244int private_stack_async_callback_2(void)
245{
246 struct bpf_timer *arr_timer;
247 int array_key = 0;
248
249 arr_timer = bpf_map_lookup_elem(&array, &array_key);
250 if (!arr_timer)
251 return 0;
252
253 bpf_timer_init(arr_timer, &array, 1);
254 bpf_timer_set_callback(arr_timer, timer_cb1);
255 bpf_timer_start(arr_timer, 0, 0);
256 subprog1(&array_key);
257 return 0;
258}
259
260#else
261
262SEC("kprobe")
263__description("private stack is not supported, use a dummy test")
264__success
265int dummy_test(void)
266{
267 return 0;
268}
269
270#endif
271
272char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_ref_tracking.c+2−2
@@ -791,7 +791,7 @@ l0_%=: r0 = *(u8*)skb[0]; \ · 790 unmodified lines
791 791
792SEC("tc")792SEC("tc")
793__description("reference tracking: forbid LD_ABS while holding reference")793__description("reference tracking: forbid LD_ABS while holding reference")
794__failure __msg("BPF_LD_[ABS|IND] cannot be mixed with socket references")794__failure __msg("BPF_LD_[ABS|IND] would lead to reference leak")
795__naked void ld_abs_while_holding_reference(void)795__naked void ld_abs_while_holding_reference(void)
796{796{
797 asm volatile (" \797 asm volatile (" \
@@ -836,7 +836,7 @@ l0_%=: r7 = 1; \ · 38 unmodified lines
836 836
837SEC("tc")837SEC("tc")
838__description("reference tracking: forbid LD_IND while holding reference")838__description("reference tracking: forbid LD_IND while holding reference")
839__failure __msg("BPF_LD_[ABS|IND] cannot be mixed with socket references")839__failure __msg("BPF_LD_[ABS|IND] would lead to reference leak")
840__naked void ld_ind_while_holding_reference(void)840__naked void ld_ind_while_holding_reference(void)
841{841{
842 asm volatile (" \842 asm volatile (" \
tools/testing/selftests/bpf/progs/verifier_sock.c+60−0
@@ -977,4 +977,64 @@ l1_%=: r0 = *(u8*)(r7 + 0); \ · 976 unmodified lines
977 : __clobber_all);977 : __clobber_all);
978}978}
979 979
980SEC("cgroup/post_bind4")
981__description("sk->src_ip6[0] [load 1st byte]")
982__failure __msg("invalid bpf_context access off=28 size=2")
983__naked void post_bind4_read_src_ip6(void)
984{
985 asm volatile (" \
986 r6 = r1; \
987 r7 = *(u16*)(r6 + %[bpf_sock_src_ip6_0]); \
988 r0 = 1; \
989 exit; \
990" :
991 : __imm_const(bpf_sock_src_ip6_0, offsetof(struct bpf_sock, src_ip6[0]))
992 : __clobber_all);
993}
994
995SEC("cgroup/post_bind4")
996__description("sk->mark [load mark]")
997__failure __msg("invalid bpf_context access off=16 size=2")
998__naked void post_bind4_read_mark(void)
999{
1000 asm volatile (" \
1001 r6 = r1; \
1002 r7 = *(u16*)(r6 + %[bpf_sock_mark]); \
1003 r0 = 1; \
1004 exit; \
1005" :
1006 : __imm_const(bpf_sock_mark, offsetof(struct bpf_sock, mark))
1007 : __clobber_all);
1008}
1009
1010SEC("cgroup/post_bind6")
1011__description("sk->src_ip4 [load src_ip4]")
1012__failure __msg("invalid bpf_context access off=24 size=2")
1013__naked void post_bind6_read_src_ip4(void)
1014{
1015 asm volatile (" \
1016 r6 = r1; \
1017 r7 = *(u16*)(r6 + %[bpf_sock_src_ip4]); \
1018 r0 = 1; \
1019 exit; \
1020" :
1021 : __imm_const(bpf_sock_src_ip4, offsetof(struct bpf_sock, src_ip4))
1022 : __clobber_all);
1023}
1024
1025SEC("cgroup/sock_create")
1026__description("sk->src_port [word load]")
1027__failure __msg("invalid bpf_context access off=44 size=2")
1028__naked void sock_create_read_src_port(void)
1029{
1030 asm volatile (" \
1031 r6 = r1; \
1032 r7 = *(u16*)(r6 + %[bpf_sock_src_port]); \
1033 r0 = 1; \
1034 exit; \
1035" :
1036 : __imm_const(bpf_sock_src_port, offsetof(struct bpf_sock, src_port))
1037 : __clobber_all);
1038}
1039
980char _license[] SEC("license") = "GPL";1040char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/verifier_spin_lock.c+1−1
@@ -187,7 +187,7 @@ l0_%=: r6 = r0; \ · 186 unmodified lines
187 187
188SEC("cgroup/skb")188SEC("cgroup/skb")
189__description("spin_lock: test6 missing unlock")189__description("spin_lock: test6 missing unlock")
190__failure __msg("unlock is missing")190__failure __msg("BPF_EXIT instruction cannot be used inside bpf_spin_lock-ed region")
191__failure_unpriv __msg_unpriv("")191__failure_unpriv __msg_unpriv("")
192__naked void spin_lock_test6_missing_unlock(void)192__naked void spin_lock_test6_missing_unlock(void)
193{193{
tools/testing/selftests/bpf/test_maps.c+0−4
@@ -26,10 +26,6 @@ · 25 unmodified lines
26#include "test_maps.h"26#include "test_maps.h"
27#include "testing_helpers.h"27#include "testing_helpers.h"
28 28
29#ifndef ENOTSUPP
30#define ENOTSUPP 524
31#endif
32
33int skips;29int skips;
34 30
35static struct bpf_map_create_opts map_opts = { .sz = sizeof(map_opts) };31static struct bpf_map_create_opts map_opts = { .sz = sizeof(map_opts) };
tools/testing/selftests/bpf/test_progs.c+111−3
@@ -16,15 +16,18 @@ · 15 unmodified lines
16#include <sys/socket.h>16#include <sys/socket.h>
17#include <sys/un.h>17#include <sys/un.h>
18#include <bpf/btf.h>18#include <bpf/btf.h>
19#include <time.h>
19#include "json_writer.h"20#include "json_writer.h"
20 21
21#include "network_helpers.h"22#include "network_helpers.h"
22 23
24/* backtrace() and backtrace_symbols_fd() are glibc specific,
25 * use header file when glibc is available and provide stub
26 * implementations when another libc implementation is used.
27 */
23#ifdef __GLIBC__28#ifdef __GLIBC__
24#include <execinfo.h> /* backtrace */29#include <execinfo.h> /* backtrace */
25#endif30#else
26
27/* Default backtrace funcs if missing at link */
28__weak int backtrace(void **buffer, int size)31__weak int backtrace(void **buffer, int size)
29{32{
30 return 0;33 return 0;
@@ -34,6 +37,7 @@ __weak void backtrace_symbols_fd(void *const *buffer, int size, int fd) · 3 unmodified lines
34{37{
35 dprintf(fd, "<backtrace not supported>\n");38 dprintf(fd, "<backtrace not supported>\n");
36}39}
40#endif /*__GLIBC__ */
37 41
38int env_verbosity = 0;42int env_verbosity = 0;
39 43
@@ -176,6 +180,88 @@ int usleep(useconds_t usec) · 136 unmodified lines
176 return syscall(__NR_nanosleep, &ts, NULL);180 return syscall(__NR_nanosleep, &ts, NULL);
177}181}
178 182
183/* Watchdog timer is started by watchdog_start() and stopped by watchdog_stop().
184 * If timer is active for longer than env.secs_till_notify,
185 * it prints the name of the current test to the stderr.
186 * If timer is active for longer than env.secs_till_kill,
187 * it kills the thread executing the test by sending a SIGSEGV signal to it.
188 */
189static void watchdog_timer_func(union sigval sigval)
190{
191 struct itimerspec timeout = {};
192 char test_name[256];
193 int err;
194
195 if (env.subtest_state)
196 snprintf(test_name, sizeof(test_name), "%s/%s",
197 env.test->test_name, env.subtest_state->name);
198 else
199 snprintf(test_name, sizeof(test_name), "%s",
200 env.test->test_name);
201
202 switch (env.watchdog_state) {
203 case WD_NOTIFY:
204 fprintf(env.stderr_saved, "WATCHDOG: test case %s executes for %d seconds...\n",
205 test_name, env.secs_till_notify);
206 timeout.it_value.tv_sec = env.secs_till_kill - env.secs_till_notify;
207 env.watchdog_state = WD_KILL;
208 err = timer_settime(env.watchdog, 0, &timeout, NULL);
209 if (err)
210 fprintf(env.stderr_saved, "Failed to arm watchdog timer\n");
211 break;
212 case WD_KILL:
213 fprintf(env.stderr_saved,
214 "WATCHDOG: test case %s executes for %d seconds, terminating with SIGSEGV\n",
215 test_name, env.secs_till_kill);
216 pthread_kill(env.main_thread, SIGSEGV);
217 break;
218 }
219}
220
221static void watchdog_start(void)
222{
223 struct itimerspec timeout = {};
224 int err;
225
226 if (env.secs_till_kill == 0)
227 return;
228 if (env.secs_till_notify > 0) {
229 env.watchdog_state = WD_NOTIFY;
230 timeout.it_value.tv_sec = env.secs_till_notify;
231 } else {
232 env.watchdog_state = WD_KILL;
233 timeout.it_value.tv_sec = env.secs_till_kill;
234 }
235 err = timer_settime(env.watchdog, 0, &timeout, NULL);
236 if (err)
237 fprintf(env.stderr_saved, "Failed to start watchdog timer\n");
238}
239
240static void watchdog_stop(void)
241{
242 struct itimerspec timeout = {};
243 int err;
244
245 env.watchdog_state = WD_NOTIFY;
246 err = timer_settime(env.watchdog, 0, &timeout, NULL);
247 if (err)
248 fprintf(env.stderr_saved, "Failed to stop watchdog timer\n");
249}
250
251static void watchdog_init(void)
252{
253 struct sigevent watchdog_sev = {
254 .sigev_notify = SIGEV_THREAD,
255 .sigev_notify_function = watchdog_timer_func,
256 };
257 int err;
258
259 env.main_thread = pthread_self();
260 err = timer_create(CLOCK_MONOTONIC, &watchdog_sev, &env.watchdog);
261 if (err)
262 fprintf(stderr, "Failed to initialize watchdog timer\n");
263}
264
179static bool should_run(struct test_selector *sel, int num, const char *name)265static bool should_run(struct test_selector *sel, int num, const char *name)
180{266{
181 int i;267 int i;
@@ -512,6 +598,7 @@ bool test__start_subtest(const char *subtest_name) · 330 unmodified lines
512 598
513 env.subtest_state = subtest_state;599 env.subtest_state = subtest_state;
514 stdio_hijack_init(&subtest_state->log_buf, &subtest_state->log_cnt);600 stdio_hijack_init(&subtest_state->log_buf, &subtest_state->log_cnt);
601 watchdog_start();
515 602
516 return true;603 return true;
517}604}
@@ -777,6 +864,7 @@ enum ARG_KEYS { · 259 unmodified lines
777 ARG_DEBUG = -1,864 ARG_DEBUG = -1,
778 ARG_JSON_SUMMARY = 'J',865 ARG_JSON_SUMMARY = 'J',
779 ARG_TRAFFIC_MONITOR = 'm',866 ARG_TRAFFIC_MONITOR = 'm',
867 ARG_WATCHDOG_TIMEOUT = 'w',
780};868};
781 869
782static const struct argp_option opts[] = {870static const struct argp_option opts[] = {
@@ -807,6 +895,8 @@ static const struct argp_option opts[] = { · 24 unmodified lines
807 { "traffic-monitor", ARG_TRAFFIC_MONITOR, "NAMES", 0,895 { "traffic-monitor", ARG_TRAFFIC_MONITOR, "NAMES", 0,
808 "Monitor network traffic of tests with name matching the pattern (supports '*' wildcard)." },896 "Monitor network traffic of tests with name matching the pattern (supports '*' wildcard)." },
809#endif897#endif
898 { "watchdog-timeout", ARG_WATCHDOG_TIMEOUT, "SECONDS", 0,
899 "Kill the process if tests are not making progress for specified number of seconds." },
810 {},900 {},
811};901};
812 902
@@ -868,6 +958,7 @@ static int libbpf_print_fn(enum libbpf_print_level level, · 55 unmodified lines
868 958
869 va_copy(args2, args);959 va_copy(args2, args);
870 vfprintf(libbpf_capture_stream, format, args2);960 vfprintf(libbpf_capture_stream, format, args2);
961 va_end(args2);
871 }962 }
872 963
873 if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)964 if (env.verbosity < VERBOSE_VERY && level == LIBBPF_DEBUG)
@@ -1031,6 +1122,16 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) · 157 unmodified lines
1031 true);1122 true);
1032 break;1123 break;
1033#endif1124#endif
1125 case ARG_WATCHDOG_TIMEOUT:
1126 env->secs_till_kill = atoi(arg);
1127 if (env->secs_till_kill < 0) {
1128 fprintf(stderr, "Invalid watchdog timeout: %s.\n", arg);
1129 return -EINVAL;
1130 }
1131 if (env->secs_till_kill < env->secs_till_notify) {
1132 env->secs_till_notify = 0;
1133 }
1134 break;
1034 default:1135 default:
1035 return ARGP_ERR_UNKNOWN;1136 return ARGP_ERR_UNKNOWN;
1036 }1137 }
@@ -1259,10 +1360,12 @@ static void run_one_test(int test_num) · 222 unmodified lines
1259 1360
1260 stdio_hijack(&state->log_buf, &state->log_cnt);1361 stdio_hijack(&state->log_buf, &state->log_cnt);
1261 1362
1363 watchdog_start();
1262 if (test->run_test)1364 if (test->run_test)
1263 test->run_test();1365 test->run_test();
1264 else if (test->run_serial_test)1366 else if (test->run_serial_test)
1265 test->run_serial_test();1367 test->run_serial_test();
1368 watchdog_stop();
1266 1369
1267 /* ensure last sub-test is finalized properly */1370 /* ensure last sub-test is finalized properly */
1268 if (env.subtest_state)1371 if (env.subtest_state)
@@ -1703,6 +1806,7 @@ out: · 434 unmodified lines
1703static int worker_main(int sock)1806static int worker_main(int sock)
1704{1807{
1705 save_netns();1808 save_netns();
1809 watchdog_init();
1706 1810
1707 while (true) {1811 while (true) {
1708 /* receive command */1812 /* receive command */
@@ -1812,6 +1916,8 @@ int main(int argc, char **argv) · 103 unmodified lines
1812 1916
1813 sigaction(SIGSEGV, &sigact, NULL);1917 sigaction(SIGSEGV, &sigact, NULL);
1814 1918
1919 env.secs_till_notify = 10;
1920 env.secs_till_kill = 120;
1815 err = argp_parse(&argp, argc, argv, 0, NULL, &env);1921 err = argp_parse(&argp, argc, argv, 0, NULL, &env);
1816 if (err)1922 if (err)
1817 return err;1923 return err;
@@ -1820,6 +1926,8 @@ int main(int argc, char **argv) · 2 unmodified lines
1820 if (err)1926 if (err)
1821 return err;1927 return err;
1822 1928
1929 watchdog_init();
1930
1823 /* Use libbpf 1.0 API mode */1931 /* Use libbpf 1.0 API mode */
1824 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);1932 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
1825 libbpf_set_print(libbpf_print_fn);1933 libbpf_set_print(libbpf_print_fn);
tools/testing/selftests/bpf/test_progs.h+14−0
@@ -131,6 +131,12 @@ struct test_env { · 130 unmodified lines
131 pid_t *worker_pids; /* array of worker pids */131 pid_t *worker_pids; /* array of worker pids */
132 int *worker_socks; /* array of worker socks */132 int *worker_socks; /* array of worker socks */
133 int *worker_current_test; /* array of current running test for each worker */133 int *worker_current_test; /* array of current running test for each worker */
134
135 pthread_t main_thread;
136 int secs_till_notify;
137 int secs_till_kill;
138 timer_t watchdog; /* watch for stalled tests/subtests */
139 enum { WD_NOTIFY, WD_KILL } watchdog_state;
134};140};
135 141
136#define MAX_LOG_TRUNK_SIZE 8192142#define MAX_LOG_TRUNK_SIZE 8192
@@ -390,6 +396,14 @@ int test__join_cgroup(const char *path); · 253 unmodified lines
390 ___ok; \396 ___ok; \
391})397})
392 398
399#define ASSERT_ERR_FD(fd, name) ({ \
400 static int duration = 0; \
401 int ___fd = (fd); \
402 bool ___ok = ___fd < 0; \
403 CHECK(!___ok, (name), "unexpected fd: %d\n", ___fd); \
404 ___ok; \
405})
406
393#define SYS(goto_label, fmt, ...) \407#define SYS(goto_label, fmt, ...) \
394 ({ \408 ({ \
395 char cmd[1024]; \409 char cmd[1024]; \
tools/testing/selftests/bpf/test_verifier.c+0−4
@@ -42,10 +42,6 @@ · 41 unmodified lines
42#include "../../../include/linux/filter.h"42#include "../../../include/linux/filter.h"
43#include "testing_helpers.h"43#include "testing_helpers.h"
44 44
45#ifndef ENOTSUPP
46#define ENOTSUPP 524
47#endif
48
49#define MAX_INSNS BPF_MAXINSNS45#define MAX_INSNS BPF_MAXINSNS
50#define MAX_EXPECTED_INSNS 3246#define MAX_EXPECTED_INSNS 32
51#define MAX_UNEXPECTED_INSNS 3247#define MAX_UNEXPECTED_INSNS 32
tools/testing/selftests/bpf/uprobe_multi.c+4−0
@@ -12,6 +12,10 @@ · 11 unmodified lines
12#define MADV_POPULATE_READ 2212#define MADV_POPULATE_READ 22
13#endif13#endif
14 14
15#ifndef MADV_PAGEOUT
16#define MADV_PAGEOUT 21
17#endif
18
15int __attribute__((weak)) uprobe(void)19int __attribute__((weak)) uprobe(void)
16{20{
17 return 0;21 return 0;
tools/testing/selftests/bpf/uptr_test_common.h+63−0
@@ -0,0 +1,63 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3
4#ifndef _UPTR_TEST_COMMON_H
5#define _UPTR_TEST_COMMON_H
6
7#define MAGIC_VALUE 0xabcd1234
8#define PAGE_SIZE 4096
9
10#ifdef __BPF__
11/* Avoid fwd btf type being generated for the following struct */
12struct large_data *dummy_large_data;
13struct empty_data *dummy_empty_data;
14struct user_data *dummy_data;
15struct cgroup *dummy_cgrp;
16#else
17#define __uptr
18#define __kptr
19#endif
20
21struct user_data {
22 int a;
23 int b;
24 int result;
25 int nested_result;
26};
27
28struct nested_udata {
29 struct user_data __uptr *udata;
30};
31
32struct value_type {
33 struct user_data __uptr *udata;
34 struct cgroup __kptr *cgrp;
35 struct nested_udata nested;
36};
37
38struct value_lock_type {
39 struct user_data __uptr *udata;
40 struct bpf_spin_lock lock;
41};
42
43struct large_data {
44 __u8 one_page[PAGE_SIZE];
45 int a;
46};
47
48struct large_uptr {
49 struct large_data __uptr *udata;
50};
51
52struct empty_data {
53};
54
55struct empty_uptr {
56 struct empty_data __uptr *udata;
57};
58
59struct kstruct_uptr {
60 struct cgroup __uptr *cgrp;
61};
62
63#endif
tools/testing/selftests/bpf/veristat.c+159−2
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include <sys/stat.h>16#include <sys/stat.h>
17#include <bpf/libbpf.h>17#include <bpf/libbpf.h>
18#include <bpf/btf.h>18#include <bpf/btf.h>
19#include <bpf/bpf.h>
19#include <libelf.h>20#include <libelf.h>
20#include <gelf.h>21#include <gelf.h>
21#include <float.h>22#include <float.h>
@@ -179,6 +180,7 @@ static struct env { · 157 unmodified lines
179 int files_skipped;180 int files_skipped;
180 int progs_processed;181 int progs_processed;
181 int progs_skipped;182 int progs_skipped;
183 int top_src_lines;
182} env;184} env;
183 185
184static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)186static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
@@ -228,6 +230,7 @@ static const struct argp_option opts[] = { · 43 unmodified lines
228 "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" },230 "Force frequent BPF verifier state checkpointing (set BPF_F_TEST_STATE_FREQ program flag)" },
229 { "test-reg-invariants", 'r', NULL, 0,231 { "test-reg-invariants", 'r', NULL, 0,
230 "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },232 "Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },
233 { "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" },
231 {},234 {},
232};235};
233 236
@@ -327,6 +330,14 @@ static error_t parse_arg(int key, char *arg, struct argp_state *state) · 93 unmodified lines
327 return err;330 return err;
328 }331 }
329 break;332 break;
333 case 'S':
334 errno = 0;
335 env.top_src_lines = strtol(arg, NULL, 10);
336 if (errno) {
337 fprintf(stderr, "invalid top lines N specifier: %s\n", arg);
338 argp_usage(state);
339 }
340 break;
330 case ARGP_KEY_ARG:341 case ARGP_KEY_ARG:
331 tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));342 tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
332 if (!tmp)343 if (!tmp)
@@ -854,6 +865,118 @@ static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats * · 521 unmodified lines
854 return 0;865 return 0;
855}866}
856 867
868struct line_cnt {
869 char *line;
870 int cnt;
871};
872
873static int str_cmp(const void *a, const void *b)
874{
875 const char **str1 = (const char **)a;
876 const char **str2 = (const char **)b;
877
878 return strcmp(*str1, *str2);
879}
880
881static int line_cnt_cmp(const void *a, const void *b)
882{
883 const struct line_cnt *a_cnt = (const struct line_cnt *)a;
884 const struct line_cnt *b_cnt = (const struct line_cnt *)b;
885
886 if (a_cnt->cnt != b_cnt->cnt)
887 return a_cnt->cnt < b_cnt->cnt ? -1 : 1;
888 return strcmp(a_cnt->line, b_cnt->line);
889}
890
891static int print_top_src_lines(char * const buf, size_t buf_sz, const char *prog_name)
892{
893 int lines_cap = 0;
894 int lines_size = 0;
895 char **lines = NULL;
896 char *line = NULL;
897 char *state;
898 struct line_cnt *freq = NULL;
899 struct line_cnt *cur;
900 int unique_lines;
901 int err = 0;
902 int i;
903
904 while ((line = strtok_r(line ? NULL : buf, "\n", &state))) {
905 if (strncmp(line, "; ", 2) != 0)
906 continue;
907 line += 2;
908
909 if (lines_size == lines_cap) {
910 char **tmp;
911
912 lines_cap = max(16, lines_cap * 2);
913 tmp = realloc(lines, lines_cap * sizeof(*tmp));
914 if (!tmp) {
915 err = -ENOMEM;
916 goto cleanup;
917 }
918 lines = tmp;
919 }
920 lines[lines_size] = line;
921 lines_size++;
922 }
923
924 if (lines_size == 0)
925 goto cleanup;
926
927 qsort(lines, lines_size, sizeof(*lines), str_cmp);
928
929 freq = calloc(lines_size, sizeof(*freq));
930 if (!freq) {
931 err = -ENOMEM;
932 goto cleanup;
933 }
934
935 cur = freq;
936 cur->line = lines[0];
937 cur->cnt = 1;
938 for (i = 1; i < lines_size; ++i) {
939 if (strcmp(lines[i], cur->line) != 0) {
940 cur++;
941 cur->line = lines[i];
942 cur->cnt = 0;
943 }
944 cur->cnt++;
945 }
946 unique_lines = cur - freq + 1;
947
948 qsort(freq, unique_lines, sizeof(struct line_cnt), line_cnt_cmp);
949
950 printf("Top source lines (%s):\n", prog_name);
951 for (i = 0; i < min(unique_lines, env.top_src_lines); ++i) {
952 const char *src_code = freq[i].line;
953 const char *src_line = NULL;
954 char *split = strrchr(freq[i].line, '@');
955
956 if (split) {
957 src_line = split + 1;
958
959 while (*src_line && isspace(*src_line))
960 src_line++;
961
962 while (split > src_code && isspace(*split))
963 split--;
964 *split = '\0';
965 }
966
967 if (src_line)
968 printf("%5d: (%s)\t%s\n", freq[i].cnt, src_line, src_code);
969 else
970 printf("%5d: %s\n", freq[i].cnt, src_code);
971 }
972 printf("\n");
973
974cleanup:
975 free(freq);
976 free(lines);
977 return err;
978}
979
857static int guess_prog_type_by_ctx_name(const char *ctx_name,980static int guess_prog_type_by_ctx_name(const char *ctx_name,
858 enum bpf_prog_type *prog_type,981 enum bpf_prog_type *prog_type,
859 enum bpf_attach_type *attach_type)982 enum bpf_attach_type *attach_type)
@@ -987,6 +1110,35 @@ skip_freplace_fixup: · 127 unmodified lines
987 return;1110 return;
988}1111}
989 1112
1113static int max_verifier_log_size(void)
1114{
1115 const int SMALL_LOG_SIZE = UINT_MAX >> 8;
1116 const int BIG_LOG_SIZE = UINT_MAX >> 2;
1117 struct bpf_insn insns[] = {
1118 { .code = BPF_ALU | BPF_MOV | BPF_X, .dst_reg = BPF_REG_0, },
1119 { .code = BPF_JMP | BPF_EXIT, },
1120 };
1121 LIBBPF_OPTS(bpf_prog_load_opts, opts,
1122 .log_size = BIG_LOG_SIZE,
1123 .log_buf = (void *)-1,
1124 .log_level = 4
1125 );
1126 int ret, insn_cnt = ARRAY_SIZE(insns);
1127 static int log_size;
1128
1129 if (log_size != 0)
1130 return log_size;
1131
1132 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
1133
1134 if (ret == -EFAULT)
1135 log_size = BIG_LOG_SIZE;
1136 else /* ret == -EINVAL, big log size is not supported by the verifier */
1137 log_size = SMALL_LOG_SIZE;
1138
1139 return log_size;
1140}
1141
990static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog)1142static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog)
991{1143{
992 const char *base_filename = basename(strdupa(filename));1144 const char *base_filename = basename(strdupa(filename));
@@ -1009,13 +1161,16 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf · 16 unmodified lines
1009 stats = &env.prog_stats[env.prog_stat_cnt++];1161 stats = &env.prog_stats[env.prog_stat_cnt++];
1010 memset(stats, 0, sizeof(*stats));1162 memset(stats, 0, sizeof(*stats));
1011 1163
1012 if (env.verbose) {1164 if (env.verbose || env.top_src_lines > 0) {
1013 buf_sz = env.log_size ? env.log_size : 16 * 1024 * 1024;1165 buf_sz = env.log_size ? env.log_size : max_verifier_log_size();
1014 buf = malloc(buf_sz);1166 buf = malloc(buf_sz);
1015 if (!buf)1167 if (!buf)
1016 return -ENOMEM;1168 return -ENOMEM;
1017 /* ensure we always request stats */1169 /* ensure we always request stats */
1018 log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0);1170 log_level = env.log_level | 4 | (env.log_fixed ? 8 : 0);
1171 /* --top-src-lines needs verifier log */
1172 if (env.top_src_lines > 0 && env.log_level == 0)
1173 log_level |= 2;
1019 } else {1174 } else {
1020 buf = verif_log_buf;1175 buf = verif_log_buf;
1021 buf_sz = sizeof(verif_log_buf);1176 buf_sz = sizeof(verif_log_buf);
@@ -1048,6 +1203,8 @@ static int process_prog(const char *filename, struct bpf_object *obj, struct bpf · 26 unmodified lines
1048 filename, prog_name, stats->stats[DURATION],1203 filename, prog_name, stats->stats[DURATION],
1049 err ? "failure" : "success", buf);1204 err ? "failure" : "success", buf);
1050 }1205 }
1206 if (env.top_src_lines > 0)
1207 print_top_src_lines(buf, buf_sz, stats->prog_name);
1051 1208
1052 if (verif_log_buf != buf)1209 if (verif_log_buf != buf)
1053 free(buf);1210 free(buf);