History

Merge https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2021-09-17 We've added 63 non-merge commits during the last 12 day(s) which contain a total of 65 files changed, 2653 insertions(+), 751 deletions(-). The main changes are: 1) Streamline internal BPF program sections handling and bpf_program__set_attach_target() in libbpf, from Andrii. 2) Add support for new btf kind BTF_KIND_TAG, from Yonghong. 3) Introduce bpf_get_branch_snapshot() to capture LBR, from Song. 4) IMUL optimization for x86-64 JIT, from Jie. 5) xsk selftest improvements, from Magnus. 6) Introduce legacy kprobe events support in libbpf, from Rafael. 7) Access hw timestamp through BPF's __sk_buff, from Vadim. * https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (63 commits) selftests/bpf: Fix a few compiler warnings libbpf: Constify all high-level program attach APIs libbpf: Schedule open_opts.attach_prog_fd deprecation since v0.7 selftests/bpf: Switch fexit_bpf2bpf selftest to set_attach_target() API libbpf: Allow skipping attach_func_name in bpf_program__set_attach_target() libbpf: Deprecated bpf_object_open_opts.relaxed_core_relocs selftests/bpf: Stop using relaxed_core_relocs which has no effect libbpf: Use pre-setup sec_def in libbpf_find_attach_btf_id() bpf: Update bpf_get_smp_processor_id() documentation libbpf: Add sphinx code documentation comments selftests/bpf: Skip btf_tag test if btf_tag attribute not supported docs/bpf: Add documentation for BTF_KIND_TAG selftests/bpf: Add a test with a bpf program with btf_tag attributes selftests/bpf: Test BTF_KIND_TAG for deduplication selftests/bpf: Add BTF_KIND_TAG unit tests selftests/bpf: Change NAME_NTH/IS_NAME_NTH for BTF_KIND_TAG format selftests/bpf: Test libbpf API function btf__add_tag() bpftool: Add support for BTF_KIND_TAG libbpf: Add support for BTF_KIND_TAG libbpf: Rename btf_{hash,equal}_int to btf_{hash,equal}_int_tag ... ====================

Jakub Kicinski committed 5 years ago

Commit af54faa2 parents f68d08c ca21a3e65 files · +2653 −751Browse files

Unified · Split

65 files
  1. Documentation/bpf/btf.rst +28 −1
  2. arch/x86/events/intel/core.c +61 −6
  3. arch/x86/events/intel/ds.c +1 −1
  4. arch/x86/events/intel/lbr.c +5 −15
  5. arch/x86/events/perf_event.h +19 −0
  6. arch/x86/net/bpf_jit_comp.c +23 −30
  7. include/linux/perf_event.h +23 −0
  8. include/uapi/linux/bpf.h +25 −1
  9. include/uapi/linux/btf.h +35 −20
  10. kernel/bpf/btf.c +128 −0
  11. kernel/bpf/trampoline.c +2 −1
  12. kernel/events/core.c +2 −0
  13. kernel/trace/bpf_trace.c +30 −0
  14. lib/test_bpf.c +1 −0
  15. net/bpf/test_run.c +11 −5
  16. net/core/filter.c +21 −0
  17. tools/bpf/bpftool/Makefile +3 −0
  18. tools/bpf/bpftool/btf.c +12 −0
  19. tools/bpf/bpftool/gen.c +19 −12
  20. tools/bpf/resolve_btfids/Makefile +3 −2
  21. tools/include/uapi/linux/bpf.h +25 −1
  22. tools/include/uapi/linux/btf.h +35 −20
  23. tools/lib/bpf/.gitignore +0 −1
  24. tools/lib/bpf/Makefile +29 −10
  25. tools/lib/bpf/btf.c +76 −8
  26. tools/lib/bpf/btf.h +87 −0
  27. tools/lib/bpf/btf_dump.c +3 −0
  28. tools/lib/bpf/libbpf.c +288 −143
  29. tools/lib/bpf/libbpf.h +22 −19
  30. tools/lib/bpf/libbpf.map +5 −0
  31. tools/lib/bpf/libbpf_common.h +24 −0
  32. tools/lib/bpf/libbpf_internal.h +21 −6
  33. tools/lib/bpf/libbpf_version.h +9 −0
  34. tools/lib/bpf/xsk.c +2 −2
  35. tools/testing/selftests/bpf/.gitignore +3 −2
  36. tools/testing/selftests/bpf/Makefile +2 −2
  37. tools/testing/selftests/bpf/README.rst +14 −0
  38. tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c +17 −2
  39. tools/testing/selftests/bpf/btf_helpers.c +6 −1
  40. tools/testing/selftests/bpf/prog_tests/bpf_iter.c +2 −4
  41. tools/testing/selftests/bpf/prog_tests/btf.c +422 −19
  42. tools/testing/selftests/bpf/prog_tests/btf_tag.c +20 −0
  43. tools/testing/selftests/bpf/prog_tests/btf_write.c +21 −0
  44. tools/testing/selftests/bpf/prog_tests/core_reloc.c +4 −13
  45. tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +26 −17
  46. tools/testing/selftests/bpf/prog_tests/get_branch_snapshot.c +100 −0
  47. tools/testing/selftests/bpf/prog_tests/module_attach.c +0 −39
  48. tools/testing/selftests/bpf/prog_tests/skb_ctx.c +6 −0
  49. tools/testing/selftests/bpf/prog_tests/skeleton.c +6 −0
  50. tools/testing/selftests/bpf/prog_tests/tailcalls.c +20 −5
  51. tools/testing/selftests/bpf/prog_tests/tc_redirect.c +1 −1
  52. tools/testing/selftests/bpf/progs/bpf_cubic.c +3 −9
  53. tools/testing/selftests/bpf/progs/get_branch_snapshot.c +40 −0
  54. tools/testing/selftests/bpf/progs/tag.c +49 −0
  55. tools/testing/selftests/bpf/progs/tailcall6.c +34 −0
  56. tools/testing/selftests/bpf/progs/test_skb_ctx.c +6 −0
  57. tools/testing/selftests/bpf/test_btf.h +3 −0
  58. tools/testing/selftests/bpf/test_progs.c +39 −0
  59. tools/testing/selftests/bpf/test_progs.h +2 −0
  60. tools/testing/selftests/bpf/trace_helpers.c +37 −0
  61. tools/testing/selftests/bpf/trace_helpers.h +5 −0
  62. tools/testing/selftests/bpf/verifier/ctx_skb.c +60 −0
  63. tools/testing/selftests/bpf/verifier/jit.c +19 −3
  64. tools/testing/selftests/bpf/xdpxceiver.c +570 −302
  65. tools/testing/selftests/bpf/xdpxceiver.h +38 −28
Documentation/bpf/btf.rst+28−1
@@ -85,6 +85,7 @@ sequentially and type id is assigned to each recognized type starting from id · 84 unmodified lines
85 #define BTF_KIND_VAR 14 /* Variable */85 #define BTF_KIND_VAR 14 /* Variable */
86 #define BTF_KIND_DATASEC 15 /* Section */86 #define BTF_KIND_DATASEC 15 /* Section */
87 #define BTF_KIND_FLOAT 16 /* Floating point */87 #define BTF_KIND_FLOAT 16 /* Floating point */
88 #define BTF_KIND_TAG 17 /* Tag */
88 89
89Note that the type section encodes debug info, not just pure types.90Note that the type section encodes debug info, not just pure types.
90``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram.91``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram.
@@ -106,7 +107,7 @@ Each type contains the following common data:: · 15 unmodified lines
106 * "size" tells the size of the type it is describing.107 * "size" tells the size of the type it is describing.
107 *108 *
108 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,109 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
109 * FUNC and FUNC_PROTO.110 * FUNC, FUNC_PROTO and TAG.
110 * "type" is a type_id referring to another type.111 * "type" is a type_id referring to another type.
111 */112 */
112 union {113 union {
@@ -465,6 +466,32 @@ map definition. · 352 unmodified lines
465 466
466No additional type data follow ``btf_type``.467No additional type data follow ``btf_type``.
467 468
4692.2.17 BTF_KIND_TAG
470~~~~~~~~~~~~~~~~~~~
471
472``struct btf_type`` encoding requirement:
473 * ``name_off``: offset to a non-empty string
474 * ``info.kind_flag``: 0
475 * ``info.kind``: BTF_KIND_TAG
476 * ``info.vlen``: 0
477 * ``type``: ``struct``, ``union``, ``func`` or ``var``
478
479``btf_type`` is followed by ``struct btf_tag``.::
480
481 struct btf_tag {
482 __u32 component_idx;
483 };
484
485The ``name_off`` encodes btf_tag attribute string.
486The ``type`` should be ``struct``, ``union``, ``func`` or ``var``.
487For ``var`` type, ``btf_tag.component_idx`` must be ``-1``.
488For the other three types, if the btf_tag attribute is
489applied to the ``struct``, ``union`` or ``func`` itself,
490``btf_tag.component_idx`` must be ``-1``. Otherwise,
491the attribute is applied to a ``struct``/``union`` member or
492a ``func`` argument, and ``btf_tag.component_idx`` should be a
493valid index (starting from 0) pointing to a member or an argument.
494
4683. BTF Kernel API4953. BTF Kernel API
469*****************496*****************
470 497
arch/x86/events/intel/core.c+61−6
@@ -2143,19 +2143,19 @@ static __initconst const u64 knl_hw_cache_extra_regs · 2142 unmodified lines
2143 * However, there are some cases which may change PEBS status, e.g. PMI2143 * However, there are some cases which may change PEBS status, e.g. PMI
2144 * throttle. The PEBS_ENABLE should be updated where the status changes.2144 * throttle. The PEBS_ENABLE should be updated where the status changes.
2145 */2145 */
2146static void __intel_pmu_disable_all(void)2146static __always_inline void __intel_pmu_disable_all(bool bts)
2147{2147{
2148 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);2148 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2149 2149
2150 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);2150 wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
2151 2151
2152 if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))2152 if (bts && test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
2153 intel_pmu_disable_bts();2153 intel_pmu_disable_bts();
2154}2154}
2155 2155
2156static void intel_pmu_disable_all(void)2156static __always_inline void intel_pmu_disable_all(void)
2157{2157{
2158 __intel_pmu_disable_all();2158 __intel_pmu_disable_all(true);
2159 intel_pmu_pebs_disable_all();2159 intel_pmu_pebs_disable_all();
2160 intel_pmu_lbr_disable_all();2160 intel_pmu_lbr_disable_all();
2161}2161}
@@ -2186,6 +2186,49 @@ static void intel_pmu_enable_all(int added) · 24 unmodified lines
2186 __intel_pmu_enable_all(added, false);2186 __intel_pmu_enable_all(added, false);
2187}2187}
2188 2188
2189static noinline int
2190__intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries,
2191 unsigned int cnt, unsigned long flags)
2192{
2193 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2194
2195 intel_pmu_lbr_read();
2196 cnt = min_t(unsigned int, cnt, x86_pmu.lbr_nr);
2197
2198 memcpy(entries, cpuc->lbr_entries, sizeof(struct perf_branch_entry) * cnt);
2199 intel_pmu_enable_all(0);
2200 local_irq_restore(flags);
2201 return cnt;
2202}
2203
2204static int
2205intel_pmu_snapshot_branch_stack(struct perf_branch_entry *entries, unsigned int cnt)
2206{
2207 unsigned long flags;
2208
2209 /* must not have branches... */
2210 local_irq_save(flags);
2211 __intel_pmu_disable_all(false); /* we don't care about BTS */
2212 __intel_pmu_pebs_disable_all();
2213 __intel_pmu_lbr_disable();
2214 /* ... until here */
2215 return __intel_pmu_snapshot_branch_stack(entries, cnt, flags);
2216}
2217
2218static int
2219intel_pmu_snapshot_arch_branch_stack(struct perf_branch_entry *entries, unsigned int cnt)
2220{
2221 unsigned long flags;
2222
2223 /* must not have branches... */
2224 local_irq_save(flags);
2225 __intel_pmu_disable_all(false); /* we don't care about BTS */
2226 __intel_pmu_pebs_disable_all();
2227 __intel_pmu_arch_lbr_disable();
2228 /* ... until here */
2229 return __intel_pmu_snapshot_branch_stack(entries, cnt, flags);
2230}
2231
2189/*2232/*
2190 * Workaround for:2233 * Workaround for:
2191 * Intel Errata AAK100 (model 26)2234 * Intel Errata AAK100 (model 26)
@@ -2929,7 +2972,7 @@ static int intel_pmu_handle_irq(struct pt_regs *regs) · 737 unmodified lines
2929 apic_write(APIC_LVTPC, APIC_DM_NMI);2972 apic_write(APIC_LVTPC, APIC_DM_NMI);
2930 intel_bts_disable_local();2973 intel_bts_disable_local();
2931 cpuc->enabled = 0;2974 cpuc->enabled = 0;
2932 __intel_pmu_disable_all();2975 __intel_pmu_disable_all(true);
2933 handled = intel_pmu_drain_bts_buffer();2976 handled = intel_pmu_drain_bts_buffer();
2934 handled += intel_bts_interrupt();2977 handled += intel_bts_interrupt();
2935 status = intel_pmu_get_status();2978 status = intel_pmu_get_status();
@@ -6283,9 +6326,21 @@ __init int intel_pmu_init(void) · 3347 unmodified lines
6283 x86_pmu.lbr_nr = 0;6326 x86_pmu.lbr_nr = 0;
6284 }6327 }
6285 6328
6286 if (x86_pmu.lbr_nr)6329 if (x86_pmu.lbr_nr) {
6287 pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);6330 pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);
6288 6331
6332 /* only support branch_stack snapshot for perfmon >= v2 */
6333 if (x86_pmu.disable_all == intel_pmu_disable_all) {
6334 if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) {
6335 static_call_update(perf_snapshot_branch_stack,
6336 intel_pmu_snapshot_arch_branch_stack);
6337 } else {
6338 static_call_update(perf_snapshot_branch_stack,
6339 intel_pmu_snapshot_branch_stack);
6340 }
6341 }
6342 }
6343
6289 intel_pmu_check_extra_regs(x86_pmu.extra_regs);6344 intel_pmu_check_extra_regs(x86_pmu.extra_regs);
6290 6345
6291 /* Support full width counters using alternative MSR range */6346 /* Support full width counters using alternative MSR range */
arch/x86/events/intel/ds.c+1−1
@@ -1301,7 +1301,7 @@ void intel_pmu_pebs_disable_all(void) · 1300 unmodified lines
1301 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);1301 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1302 1302
1303 if (cpuc->pebs_enabled)1303 if (cpuc->pebs_enabled)
1304 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);1304 __intel_pmu_pebs_disable_all();
1305}1305}
1306 1306
1307static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)1307static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
arch/x86/events/intel/lbr.c+5−15
@@ -228,20 +228,6 @@ static void __intel_pmu_lbr_enable(bool pmi) · 227 unmodified lines
228 wrmsrl(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN);228 wrmsrl(MSR_ARCH_LBR_CTL, lbr_select | ARCH_LBR_CTL_LBREN);
229}229}
230 230
231static void __intel_pmu_lbr_disable(void)
232{
233 u64 debugctl;
234
235 if (static_cpu_has(X86_FEATURE_ARCH_LBR)) {
236 wrmsrl(MSR_ARCH_LBR_CTL, 0);
237 return;
238 }
239
240 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
241 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
242 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
243}
244
245void intel_pmu_lbr_reset_32(void)231void intel_pmu_lbr_reset_32(void)
246{232{
247 int i;233 int i;
@@ -779,8 +765,12 @@ void intel_pmu_lbr_disable_all(void) · 531 unmodified lines
779{765{
780 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);766 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
781 767
782 if (cpuc->lbr_users && !vlbr_exclude_host())768 if (cpuc->lbr_users && !vlbr_exclude_host()) {
769 if (static_cpu_has(X86_FEATURE_ARCH_LBR))
770 return __intel_pmu_arch_lbr_disable();
771
783 __intel_pmu_lbr_disable();772 __intel_pmu_lbr_disable();
773 }
784}774}
785 775
786void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)776void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
arch/x86/events/perf_event.h+19−0
@@ -1240,6 +1240,25 @@ static inline bool intel_pmu_has_bts(struct perf_event *event) · 1239 unmodified lines
1240 return intel_pmu_has_bts_period(event, hwc->sample_period);1240 return intel_pmu_has_bts_period(event, hwc->sample_period);
1241}1241}
1242 1242
1243static __always_inline void __intel_pmu_pebs_disable_all(void)
1244{
1245 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
1246}
1247
1248static __always_inline void __intel_pmu_arch_lbr_disable(void)
1249{
1250 wrmsrl(MSR_ARCH_LBR_CTL, 0);
1251}
1252
1253static __always_inline void __intel_pmu_lbr_disable(void)
1254{
1255 u64 debugctl;
1256
1257 rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1258 debugctl &= ~(DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI);
1259 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
1260}
1261
1243int intel_pmu_save_and_restart(struct perf_event *event);1262int intel_pmu_save_and_restart(struct perf_event *event);
1244 1263
1245struct event_constraint *1264struct event_constraint *
arch/x86/net/bpf_jit_comp.c+23−30
@@ -1070,41 +1070,34 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, · 1069 unmodified lines
1070 break;1070 break;
1071 1071
1072 case BPF_ALU | BPF_MUL | BPF_K:1072 case BPF_ALU | BPF_MUL | BPF_K:
1073 case BPF_ALU | BPF_MUL | BPF_X:
1074 case BPF_ALU64 | BPF_MUL | BPF_K:1073 case BPF_ALU64 | BPF_MUL | BPF_K:
1075 case BPF_ALU64 | BPF_MUL | BPF_X:1074 if (BPF_CLASS(insn->code) == BPF_ALU64)
1076 {1075 EMIT1(add_2mod(0x48, dst_reg, dst_reg));
1077 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;1076 else if (is_ereg(dst_reg))
1077 EMIT1(add_2mod(0x40, dst_reg, dst_reg));
1078 1078
1079 if (dst_reg != BPF_REG_0)1079 if (is_imm8(imm32))
1080 EMIT1(0x50); /* push rax */1080 /* imul dst_reg, dst_reg, imm8 */
1081 if (dst_reg != BPF_REG_3)1081 EMIT3(0x6B, add_2reg(0xC0, dst_reg, dst_reg),
1082 EMIT1(0x52); /* push rdx */1082 imm32);
1083 else
1084 /* imul dst_reg, dst_reg, imm32 */
1085 EMIT2_off32(0x69,
1086 add_2reg(0xC0, dst_reg, dst_reg),
1087 imm32);
1088 break;
1083 1089
1084 /* mov r11, dst_reg */1090 case BPF_ALU | BPF_MUL | BPF_X:
1085 EMIT_mov(AUX_REG, dst_reg);1091 case BPF_ALU64 | BPF_MUL | BPF_X:
1092 if (BPF_CLASS(insn->code) == BPF_ALU64)
1093 EMIT1(add_2mod(0x48, src_reg, dst_reg));
1094 else if (is_ereg(dst_reg) || is_ereg(src_reg))
1095 EMIT1(add_2mod(0x40, src_reg, dst_reg));
1086 1096
1087 if (BPF_SRC(insn->code) == BPF_X)1097 /* imul dst_reg, src_reg */
1088 emit_mov_reg(&prog, is64, BPF_REG_0, src_reg);1098 EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg));
1089 else
1090 emit_mov_imm32(&prog, is64, BPF_REG_0, imm32);
1091
1092 if (is64)
1093 EMIT1(add_1mod(0x48, AUX_REG));
1094 else if (is_ereg(AUX_REG))
1095 EMIT1(add_1mod(0x40, AUX_REG));
1096 /* mul(q) r11 */
1097 EMIT2(0xF7, add_1reg(0xE0, AUX_REG));
1098
1099 if (dst_reg != BPF_REG_3)
1100 EMIT1(0x5A); /* pop rdx */
1101 if (dst_reg != BPF_REG_0) {
1102 /* mov dst_reg, rax */
1103 EMIT_mov(dst_reg, BPF_REG_0);
1104 EMIT1(0x58); /* pop rax */
1105 }
1106 break;1099 break;
1107 }1100
1108 /* Shifts */1101 /* Shifts */
1109 case BPF_ALU | BPF_LSH | BPF_K:1102 case BPF_ALU | BPF_LSH | BPF_K:
1110 case BPF_ALU | BPF_RSH | BPF_K:1103 case BPF_ALU | BPF_RSH | BPF_K:
include/linux/perf_event.h+23−0
@@ -57,6 +57,7 @@ struct perf_guest_info_callbacks { · 56 unmodified lines
57#include <linux/cgroup.h>57#include <linux/cgroup.h>
58#include <linux/refcount.h>58#include <linux/refcount.h>
59#include <linux/security.h>59#include <linux/security.h>
60#include <linux/static_call.h>
60#include <asm/local.h>61#include <asm/local.h>
61 62
62struct perf_callchain_entry {63struct perf_callchain_entry {
@@ -1612,4 +1613,26 @@ extern void __weak arch_perf_update_userpage(struct perf_event *event, · 1549 unmodified lines
1612extern __weak u64 arch_perf_get_page_size(struct mm_struct *mm, unsigned long addr);1613extern __weak u64 arch_perf_get_page_size(struct mm_struct *mm, unsigned long addr);
1613#endif1614#endif
1614 1615
1616/*
1617 * Snapshot branch stack on software events.
1618 *
1619 * Branch stack can be very useful in understanding software events. For
1620 * example, when a long function, e.g. sys_perf_event_open, returns an
1621 * errno, it is not obvious why the function failed. Branch stack could
1622 * provide very helpful information in this type of scenarios.
1623 *
1624 * On software event, it is necessary to stop the hardware branch recorder
1625 * fast. Otherwise, the hardware register/buffer will be flushed with
1626 * entries of the triggering event. Therefore, static call is used to
1627 * stop the hardware recorder.
1628 */
1629
1630/*
1631 * cnt is the number of entries allocated for entries.
1632 * Return number of entries copied to .
1633 */
1634typedef int (perf_snapshot_branch_stack_t)(struct perf_branch_entry *entries,
1635 unsigned int cnt);
1636DECLARE_STATIC_CALL(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);
1637
1615#endif /* _LINUX_PERF_EVENT_H */1638#endif /* _LINUX_PERF_EVENT_H */
include/uapi/linux/bpf.h+25−1
@@ -1629,7 +1629,7 @@ union bpf_attr { · 1628 unmodified lines
1629 * u32 bpf_get_smp_processor_id(void)1629 * u32 bpf_get_smp_processor_id(void)
1630 * Description1630 * Description
1631 * Get the SMP (symmetric multiprocessing) processor id. Note that1631 * Get the SMP (symmetric multiprocessing) processor id. Note that
1632 * all programs run with preemption disabled, which means that the1632 * all programs run with migration disabled, which means that the
1633 * SMP processor id is stable during all the execution of the1633 * SMP processor id is stable during all the execution of the
1634 * program.1634 * program.
1635 * Return1635 * Return
@@ -4877,6 +4877,27 @@ union bpf_attr { · 3241 unmodified lines
4877 * Get the struct pt_regs associated with **task**.4877 * Get the struct pt_regs associated with **task**.
4878 * Return4878 * Return
4879 * A pointer to struct pt_regs.4879 * A pointer to struct pt_regs.
4880 *
4881 * long bpf_get_branch_snapshot(void *entries, u32 size, u64 flags)
4882 * Description
4883 * Get branch trace from hardware engines like Intel LBR. The
4884 * hardware engine is stopped shortly after the helper is
4885 * called. Therefore, the user need to filter branch entries
4886 * based on the actual use case. To capture branch trace
4887 * before the trigger point of the BPF program, the helper
4888 * should be called at the beginning of the BPF program.
4889 *
4890 * The data is stored as struct perf_branch_entry into output
4891 * buffer *entries*. *size* is the size of *entries* in bytes.
4892 * *flags* is reserved for now and must be zero.
4893 *
4894 * Return
4895 * On success, number of bytes written to *buf*. On error, a
4896 * negative value.
4897 *
4898 * **-EINVAL** if *flags* is not zero.
4899 *
4900 * **-ENOENT** if architecture does not support branch records.
4880 */4901 */
4881#define __BPF_FUNC_MAPPER(FN) \4902#define __BPF_FUNC_MAPPER(FN) \
4882 FN(unspec), \4903 FN(unspec), \
@@ -5055,6 +5076,7 @@ union bpf_attr { · 172 unmodified lines
5055 FN(get_func_ip), \5076 FN(get_func_ip), \
5056 FN(get_attach_cookie), \5077 FN(get_attach_cookie), \
5057 FN(task_pt_regs), \5078 FN(task_pt_regs), \
5079 FN(get_branch_snapshot), \
5058 /* */5080 /* */
5059 5081
5060/* integer value in 'imm' field of BPF_CALL instruction selects which helper5082/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -5284,6 +5306,8 @@ struct __sk_buff { · 223 unmodified lines
5284 __u32 gso_segs;5306 __u32 gso_segs;
5285 __bpf_md_ptr(struct bpf_sock *, sk);5307 __bpf_md_ptr(struct bpf_sock *, sk);
5286 __u32 gso_size;5308 __u32 gso_size;
5309 __u32 :32; /* Padding, future use. */
5310 __u64 hwtstamp;
5287};5311};
5288 5312
5289struct bpf_tunnel_key {5313struct bpf_tunnel_key {
include/uapi/linux/btf.h+35−20
@@ -43,7 +43,7 @@ struct btf_type { · 42 unmodified lines
43 * "size" tells the size of the type it is describing.43 * "size" tells the size of the type it is describing.
44 *44 *
45 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,45 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
46 * FUNC, FUNC_PROTO and VAR.46 * FUNC, FUNC_PROTO, VAR and TAG.
47 * "type" is a type_id referring to another type.47 * "type" is a type_id referring to another type.
48 */48 */
49 union {49 union {
@@ -56,25 +56,29 @@ struct btf_type { · 6 unmodified lines
56#define BTF_INFO_VLEN(info) ((info) & 0xffff)56#define BTF_INFO_VLEN(info) ((info) & 0xffff)
57#define BTF_INFO_KFLAG(info) ((info) >> 31)57#define BTF_INFO_KFLAG(info) ((info) >> 31)
58 58
59#define BTF_KIND_UNKN 0 /* Unknown */59enum {
60#define BTF_KIND_INT 1 /* Integer */60 BTF_KIND_UNKN = 0, /* Unknown */
61#define BTF_KIND_PTR 2 /* Pointer */61 BTF_KIND_INT = 1, /* Integer */
62#define BTF_KIND_ARRAY 3 /* Array */62 BTF_KIND_PTR = 2, /* Pointer */
63#define BTF_KIND_STRUCT 4 /* Struct */63 BTF_KIND_ARRAY = 3, /* Array */
64#define BTF_KIND_UNION 5 /* Union */64 BTF_KIND_STRUCT = 4, /* Struct */
65#define BTF_KIND_ENUM 6 /* Enumeration */65 BTF_KIND_UNION = 5, /* Union */
66#define BTF_KIND_FWD 7 /* Forward */66 BTF_KIND_ENUM = 6, /* Enumeration */
67#define BTF_KIND_TYPEDEF 8 /* Typedef */67 BTF_KIND_FWD = 7, /* Forward */
68#define BTF_KIND_VOLATILE 9 /* Volatile */68 BTF_KIND_TYPEDEF = 8, /* Typedef */
69#define BTF_KIND_CONST 10 /* Const */69 BTF_KIND_VOLATILE = 9, /* Volatile */
70#define BTF_KIND_RESTRICT 11 /* Restrict */70 BTF_KIND_CONST = 10, /* Const */
71#define BTF_KIND_FUNC 12 /* Function */71 BTF_KIND_RESTRICT = 11, /* Restrict */
72#define BTF_KIND_FUNC_PROTO 13 /* Function Proto */72 BTF_KIND_FUNC = 12, /* Function */
73#define BTF_KIND_VAR 14 /* Variable */73 BTF_KIND_FUNC_PROTO = 13, /* Function Proto */
74#define BTF_KIND_DATASEC 15 /* Section */74 BTF_KIND_VAR = 14, /* Variable */
75#define BTF_KIND_FLOAT 16 /* Floating point */75 BTF_KIND_DATASEC = 15, /* Section */
76#define BTF_KIND_MAX BTF_KIND_FLOAT76 BTF_KIND_FLOAT = 16, /* Floating point */
77#define NR_BTF_KINDS (BTF_KIND_MAX + 1)77 BTF_KIND_TAG = 17, /* Tag */
78
79 NR_BTF_KINDS,
80 BTF_KIND_MAX = NR_BTF_KINDS - 1,
81};
78 82
79/* For some specific BTF_KIND, "struct btf_type" is immediately83/* For some specific BTF_KIND, "struct btf_type" is immediately
80 * followed by extra data.84 * followed by extra data.
@@ -170,4 +174,15 @@ struct btf_var_secinfo { · 89 unmodified lines
170 __u32 size;174 __u32 size;
171};175};
172 176
177/* BTF_KIND_TAG is followed by a single "struct btf_tag" to describe
178 * additional information related to the tag applied location.
179 * If component_idx == -1, the tag is applied to a struct, union,
180 * variable or function. Otherwise, it is applied to a struct/union
181 * member or a func argument, and component_idx indicates which member
182 * or argument (0 ... vlen-1).
183 */
184struct btf_tag {
185 __s32 component_idx;
186};
187
173#endif /* _UAPI__LINUX_BTF_H__ */188#endif /* _UAPI__LINUX_BTF_H__ */
kernel/bpf/btf.c+128−0
@@ -281,6 +281,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 280 unmodified lines
281 [BTF_KIND_VAR] = "VAR",281 [BTF_KIND_VAR] = "VAR",
282 [BTF_KIND_DATASEC] = "DATASEC",282 [BTF_KIND_DATASEC] = "DATASEC",
283 [BTF_KIND_FLOAT] = "FLOAT",283 [BTF_KIND_FLOAT] = "FLOAT",
284 [BTF_KIND_TAG] = "TAG",
284};285};
285 286
286const char *btf_type_str(const struct btf_type *t)287const char *btf_type_str(const struct btf_type *t)
@@ -459,6 +460,17 @@ static bool btf_type_is_datasec(const struct btf_type *t) · 172 unmodified lines
459 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;460 return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
460}461}
461 462
463static bool btf_type_is_tag(const struct btf_type *t)
464{
465 return BTF_INFO_KIND(t->info) == BTF_KIND_TAG;
466}
467
468static bool btf_type_is_tag_target(const struct btf_type *t)
469{
470 return btf_type_is_func(t) || btf_type_is_struct(t) ||
471 btf_type_is_var(t);
472}
473
462u32 btf_nr_types(const struct btf *btf)474u32 btf_nr_types(const struct btf *btf)
463{475{
464 u32 total = 0;476 u32 total = 0;
@@ -537,6 +549,7 @@ const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf, · 72 unmodified lines
537static bool btf_type_is_resolve_source_only(const struct btf_type *t)549static bool btf_type_is_resolve_source_only(const struct btf_type *t)
538{550{
539 return btf_type_is_var(t) ||551 return btf_type_is_var(t) ||
552 btf_type_is_tag(t) ||
540 btf_type_is_datasec(t);553 btf_type_is_datasec(t);
541}554}
542 555
@@ -563,6 +576,7 @@ static bool btf_type_needs_resolve(const struct btf_type *t) · 20 unmodified lines
563 btf_type_is_struct(t) ||576 btf_type_is_struct(t) ||
564 btf_type_is_array(t) ||577 btf_type_is_array(t) ||
565 btf_type_is_var(t) ||578 btf_type_is_var(t) ||
579 btf_type_is_tag(t) ||
566 btf_type_is_datasec(t);580 btf_type_is_datasec(t);
567}581}
568 582
@@ -616,6 +630,11 @@ static const struct btf_var *btf_type_var(const struct btf_type *t) · 47 unmodified lines
616 return (const struct btf_var *)(t + 1);630 return (const struct btf_var *)(t + 1);
617}631}
618 632
633static const struct btf_tag *btf_type_tag(const struct btf_type *t)
634{
635 return (const struct btf_tag *)(t + 1);
636}
637
619static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)638static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
620{639{
621 return kind_ops[BTF_INFO_KIND(t->info)];640 return kind_ops[BTF_INFO_KIND(t->info)];
@@ -3801,6 +3820,110 @@ static const struct btf_kind_operations float_ops = { · 3179 unmodified lines
3801 .show = btf_df_show,3820 .show = btf_df_show,
3802};3821};
3803 3822
3823static s32 btf_tag_check_meta(struct btf_verifier_env *env,
3824 const struct btf_type *t,
3825 u32 meta_left)
3826{
3827 const struct btf_tag *tag;
3828 u32 meta_needed = sizeof(*tag);
3829 s32 component_idx;
3830 const char *value;
3831
3832 if (meta_left < meta_needed) {
3833 btf_verifier_log_basic(env, t,
3834 "meta_left:%u meta_needed:%u",
3835 meta_left, meta_needed);
3836 return -EINVAL;
3837 }
3838
3839 value = btf_name_by_offset(env->btf, t->name_off);
3840 if (!value || !value[0]) {
3841 btf_verifier_log_type(env, t, "Invalid value");
3842 return -EINVAL;
3843 }
3844
3845 if (btf_type_vlen(t)) {
3846 btf_verifier_log_type(env, t, "vlen != 0");
3847 return -EINVAL;
3848 }
3849
3850 if (btf_type_kflag(t)) {
3851 btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
3852 return -EINVAL;
3853 }
3854
3855 component_idx = btf_type_tag(t)->component_idx;
3856 if (component_idx < -1) {
3857 btf_verifier_log_type(env, t, "Invalid component_idx");
3858 return -EINVAL;
3859 }
3860
3861 btf_verifier_log_type(env, t, NULL);
3862
3863 return meta_needed;
3864}
3865
3866static int btf_tag_resolve(struct btf_verifier_env *env,
3867 const struct resolve_vertex *v)
3868{
3869 const struct btf_type *next_type;
3870 const struct btf_type *t = v->t;
3871 u32 next_type_id = t->type;
3872 struct btf *btf = env->btf;
3873 s32 component_idx;
3874 u32 vlen;
3875
3876 next_type = btf_type_by_id(btf, next_type_id);
3877 if (!next_type || !btf_type_is_tag_target(next_type)) {
3878 btf_verifier_log_type(env, v->t, "Invalid type_id");
3879 return -EINVAL;
3880 }
3881
3882 if (!env_type_is_resolve_sink(env, next_type) &&
3883 !env_type_is_resolved(env, next_type_id))
3884 return env_stack_push(env, next_type, next_type_id);
3885
3886 component_idx = btf_type_tag(t)->component_idx;
3887 if (component_idx != -1) {
3888 if (btf_type_is_var(next_type)) {
3889 btf_verifier_log_type(env, v->t, "Invalid component_idx");
3890 return -EINVAL;
3891 }
3892
3893 if (btf_type_is_struct(next_type)) {
3894 vlen = btf_type_vlen(next_type);
3895 } else {
3896 /* next_type should be a function */
3897 next_type = btf_type_by_id(btf, next_type->type);
3898 vlen = btf_type_vlen(next_type);
3899 }
3900
3901 if ((u32)component_idx >= vlen) {
3902 btf_verifier_log_type(env, v->t, "Invalid component_idx");
3903 return -EINVAL;
3904 }
3905 }
3906
3907 env_stack_pop_resolved(env, next_type_id, 0);
3908
3909 return 0;
3910}
3911
3912static void btf_tag_log(struct btf_verifier_env *env, const struct btf_type *t)
3913{
3914 btf_verifier_log(env, "type=%u component_idx=%d", t->type,
3915 btf_type_tag(t)->component_idx);
3916}
3917
3918static const struct btf_kind_operations tag_ops = {
3919 .check_meta = btf_tag_check_meta,
3920 .resolve = btf_tag_resolve,
3921 .check_member = btf_df_check_member,
3922 .check_kflag_member = btf_df_check_kflag_member,
3923 .log_details = btf_tag_log,
3924 .show = btf_df_show,
3925};
3926
3804static int btf_func_proto_check(struct btf_verifier_env *env,3927static int btf_func_proto_check(struct btf_verifier_env *env,
3805 const struct btf_type *t)3928 const struct btf_type *t)
3806{3929{
@@ -3935,6 +4058,7 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { · 128 unmodified lines
3935 [BTF_KIND_VAR] = &var_ops,4058 [BTF_KIND_VAR] = &var_ops,
3936 [BTF_KIND_DATASEC] = &datasec_ops,4059 [BTF_KIND_DATASEC] = &datasec_ops,
3937 [BTF_KIND_FLOAT] = &float_ops,4060 [BTF_KIND_FLOAT] = &float_ops,
4061 [BTF_KIND_TAG] = &tag_ops,
3938};4062};
3939 4063
3940static s32 btf_check_meta(struct btf_verifier_env *env,4064static s32 btf_check_meta(struct btf_verifier_env *env,
@@ -4019,6 +4143,10 @@ static bool btf_resolve_valid(struct btf_verifier_env *env, · 78 unmodified lines
4019 return !btf_resolved_type_id(btf, type_id) &&4143 return !btf_resolved_type_id(btf, type_id) &&
4020 !btf_resolved_type_size(btf, type_id);4144 !btf_resolved_type_size(btf, type_id);
4021 4145
4146 if (btf_type_is_tag(t))
4147 return btf_resolved_type_id(btf, type_id) &&
4148 !btf_resolved_type_size(btf, type_id);
4149
4022 if (btf_type_is_modifier(t) || btf_type_is_ptr(t) ||4150 if (btf_type_is_modifier(t) || btf_type_is_ptr(t) ||
4023 btf_type_is_var(t)) {4151 btf_type_is_var(t)) {
4024 t = btf_type_id_resolve(btf, &type_id);4152 t = btf_type_id_resolve(btf, &type_id);
kernel/bpf/trampoline.c+2−1
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#include <linux/rcupdate_trace.h>10#include <linux/rcupdate_trace.h>
11#include <linux/rcupdate_wait.h>11#include <linux/rcupdate_wait.h>
12#include <linux/module.h>12#include <linux/module.h>
13#include <linux/static_call.h>
13 14
14/* dummy _ops. The verifier will operate on target program's ops. */15/* dummy _ops. The verifier will operate on target program's ops. */
15const struct bpf_verifier_ops bpf_extension_verifier_ops = {16const struct bpf_verifier_ops bpf_extension_verifier_ops = {
@@ -526,7 +527,7 @@ out: · 510 unmodified lines
526}527}
527 528
528#define NO_START_TIME 1529#define NO_START_TIME 1
529static u64 notrace bpf_prog_start_time(void)530static __always_inline u64 notrace bpf_prog_start_time(void)
530{531{
531 u64 start = NO_START_TIME;532 u64 start = NO_START_TIME;
532 533
kernel/events/core.c+2−0
@@ -13435,3 +13435,5 @@ struct cgroup_subsys perf_event_cgrp_subsys = { · 13434 unmodified lines
13435 .threaded = true,13435 .threaded = true,
13436};13436};
13437#endif /* CONFIG_CGROUP_PERF */13437#endif /* CONFIG_CGROUP_PERF */
13438
13439DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);
kernel/trace/bpf_trace.c+30−0
@@ -1017,6 +1017,34 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto_pe = { · 1016 unmodified lines
1017 .arg1_type = ARG_PTR_TO_CTX,1017 .arg1_type = ARG_PTR_TO_CTX,
1018};1018};
1019 1019
1020BPF_CALL_3(bpf_get_branch_snapshot, void *, buf, u32, size, u64, flags)
1021{
1022#ifndef CONFIG_X86
1023 return -ENOENT;
1024#else
1025 static const u32 br_entry_size = sizeof(struct perf_branch_entry);
1026 u32 entry_cnt = size / br_entry_size;
1027
1028 entry_cnt = static_call(perf_snapshot_branch_stack)(buf, entry_cnt);
1029
1030 if (unlikely(flags))
1031 return -EINVAL;
1032
1033 if (!entry_cnt)
1034 return -ENOENT;
1035
1036 return entry_cnt * br_entry_size;
1037#endif
1038}
1039
1040static const struct bpf_func_proto bpf_get_branch_snapshot_proto = {
1041 .func = bpf_get_branch_snapshot,
1042 .gpl_only = true,
1043 .ret_type = RET_INTEGER,
1044 .arg1_type = ARG_PTR_TO_UNINIT_MEM,
1045 .arg2_type = ARG_CONST_SIZE_OR_ZERO,
1046};
1047
1020static const struct bpf_func_proto *1048static const struct bpf_func_proto *
1021bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)1049bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1022{1050{
@@ -1132,6 +1160,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 109 unmodified lines
1132 return &bpf_snprintf_proto;1160 return &bpf_snprintf_proto;
1133 case BPF_FUNC_get_func_ip:1161 case BPF_FUNC_get_func_ip:
1134 return &bpf_get_func_ip_proto_tracing;1162 return &bpf_get_func_ip_proto_tracing;
1163 case BPF_FUNC_get_branch_snapshot:
1164 return &bpf_get_branch_snapshot_proto;
1135 default:1165 default:
1136 return bpf_base_func_proto(func_id);1166 return bpf_base_func_proto(func_id);
1137 }1167 }
lib/test_bpf.c+1−0
@@ -8800,6 +8800,7 @@ static __init struct sk_buff *build_test_skb(void) · 8799 unmodified lines
8800 skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY;8800 skb_shinfo(skb[0])->gso_type |= SKB_GSO_DODGY;
8801 skb_shinfo(skb[0])->gso_segs = 0;8801 skb_shinfo(skb[0])->gso_segs = 0;
8802 skb_shinfo(skb[0])->frag_list = skb[1];8802 skb_shinfo(skb[0])->frag_list = skb[1];
8803 skb_shinfo(skb[0])->hwtstamps.hwtstamp = 1000;
8803 8804
8804 /* adjust skb[0]'s len */8805 /* adjust skb[0]'s len */
8805 skb[0]->len += skb[1]->len;8806 skb[0]->len += skb[1]->len;
net/bpf/test_run.c+11−5
@@ -483,11 +483,7 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) · 482 unmodified lines
483 return -EINVAL;483 return -EINVAL;
484 484
485 /* priority is allowed */485 /* priority is allowed */
486 486 /* ingress_ifindex is allowed */
487 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, priority),
488 offsetof(struct __sk_buff, ifindex)))
489 return -EINVAL;
490
491 /* ifindex is allowed */487 /* ifindex is allowed */
492 488
493 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),489 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, ifindex),
@@ -511,11 +507,18 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) · 17 unmodified lines
511 /* gso_size is allowed */507 /* gso_size is allowed */
512 508
513 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),509 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, gso_size),
510 offsetof(struct __sk_buff, hwtstamp)))
511 return -EINVAL;
512
513 /* hwtstamp is allowed */
514
515 if (!range_is_zero(__skb, offsetofend(struct __sk_buff, hwtstamp),
514 sizeof(struct __sk_buff)))516 sizeof(struct __sk_buff)))
515 return -EINVAL;517 return -EINVAL;
516 518
517 skb->mark = __skb->mark;519 skb->mark = __skb->mark;
518 skb->priority = __skb->priority;520 skb->priority = __skb->priority;
521 skb->skb_iif = __skb->ingress_ifindex;
519 skb->tstamp = __skb->tstamp;522 skb->tstamp = __skb->tstamp;
520 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);523 memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
521 524
@@ -532,6 +535,7 @@ static int convert___skb_to_skb(struct sk_buff *skb, struct __sk_buff *__skb) · 10 unmodified lines
532 return -EINVAL;535 return -EINVAL;
533 skb_shinfo(skb)->gso_segs = __skb->gso_segs;536 skb_shinfo(skb)->gso_segs = __skb->gso_segs;
534 skb_shinfo(skb)->gso_size = __skb->gso_size;537 skb_shinfo(skb)->gso_size = __skb->gso_size;
538 skb_shinfo(skb)->hwtstamps.hwtstamp = __skb->hwtstamp;
535 539
536 return 0;540 return 0;
537}541}
@@ -545,11 +549,13 @@ static void convert_skb_to___skb(struct sk_buff *skb, struct __sk_buff *__skb) · 7 unmodified lines
545 549
546 __skb->mark = skb->mark;550 __skb->mark = skb->mark;
547 __skb->priority = skb->priority;551 __skb->priority = skb->priority;
552 __skb->ingress_ifindex = skb->skb_iif;
548 __skb->ifindex = skb->dev->ifindex;553 __skb->ifindex = skb->dev->ifindex;
549 __skb->tstamp = skb->tstamp;554 __skb->tstamp = skb->tstamp;
550 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);555 memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
551 __skb->wire_len = cb->pkt_len;556 __skb->wire_len = cb->pkt_len;
552 __skb->gso_segs = skb_shinfo(skb)->gso_segs;557 __skb->gso_segs = skb_shinfo(skb)->gso_segs;
558 __skb->hwtstamp = skb_shinfo(skb)->hwtstamps.hwtstamp;
553}559}
554 560
555int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,561int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
net/core/filter.c+21−0
@@ -7765,6 +7765,10 @@ static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type · 7764 unmodified lines
7765 break;7765 break;
7766 case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):7766 case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):
7767 return false;7767 return false;
7768 case bpf_ctx_range(struct __sk_buff, hwtstamp):
7769 if (type == BPF_WRITE || size != sizeof(__u64))
7770 return false;
7771 break;
7768 case bpf_ctx_range(struct __sk_buff, tstamp):7772 case bpf_ctx_range(struct __sk_buff, tstamp):
7769 if (size != sizeof(__u64))7773 if (size != sizeof(__u64))
7770 return false;7774 return false;
@@ -7774,6 +7778,9 @@ static bool bpf_skb_is_valid_access(int off, int size, enum bpf_access_type type · 3 unmodified lines
7774 return false;7778 return false;
7775 info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;7779 info->reg_type = PTR_TO_SOCK_COMMON_OR_NULL;
7776 break;7780 break;
7781 case offsetofend(struct __sk_buff, gso_size) ... offsetof(struct __sk_buff, hwtstamp) - 1:
7782 /* Explicitly prohibit access to padding in __sk_buff. */
7783 return false;
7777 default:7784 default:
7778 /* Only narrow read access allowed for now. */7785 /* Only narrow read access allowed for now. */
7779 if (type == BPF_WRITE) {7786 if (type == BPF_WRITE) {
@@ -7802,6 +7809,7 @@ static bool sk_filter_is_valid_access(int off, int size, · 22 unmodified lines
7802 case bpf_ctx_range_till(struct __sk_buff, family, local_port):7809 case bpf_ctx_range_till(struct __sk_buff, family, local_port):
7803 case bpf_ctx_range(struct __sk_buff, tstamp):7810 case bpf_ctx_range(struct __sk_buff, tstamp):
7804 case bpf_ctx_range(struct __sk_buff, wire_len):7811 case bpf_ctx_range(struct __sk_buff, wire_len):
7812 case bpf_ctx_range(struct __sk_buff, hwtstamp):
7805 return false;7813 return false;
7806 }7814 }
7807 7815
@@ -7872,6 +7880,7 @@ static bool lwt_is_valid_access(int off, int size, · 64 unmodified lines
7872 case bpf_ctx_range(struct __sk_buff, data_meta):7880 case bpf_ctx_range(struct __sk_buff, data_meta):
7873 case bpf_ctx_range(struct __sk_buff, tstamp):7881 case bpf_ctx_range(struct __sk_buff, tstamp):
7874 case bpf_ctx_range(struct __sk_buff, wire_len):7882 case bpf_ctx_range(struct __sk_buff, wire_len):
7883 case bpf_ctx_range(struct __sk_buff, hwtstamp):
7875 return false;7884 return false;
7876 }7885 }
7877 7886
@@ -8373,6 +8382,7 @@ static bool sk_skb_is_valid_access(int off, int size, · 495 unmodified lines
8373 case bpf_ctx_range(struct __sk_buff, data_meta):8382 case bpf_ctx_range(struct __sk_buff, data_meta):
8374 case bpf_ctx_range(struct __sk_buff, tstamp):8383 case bpf_ctx_range(struct __sk_buff, tstamp):
8375 case bpf_ctx_range(struct __sk_buff, wire_len):8384 case bpf_ctx_range(struct __sk_buff, wire_len):
8385 case bpf_ctx_range(struct __sk_buff, hwtstamp):
8376 return false;8386 return false;
8377 }8387 }
8378 8388
@@ -8884,6 +8894,17 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, · 505 unmodified lines
8884 si->dst_reg, si->src_reg,8894 si->dst_reg, si->src_reg,
8885 offsetof(struct sk_buff, sk));8895 offsetof(struct sk_buff, sk));
8886 break;8896 break;
8897 case offsetof(struct __sk_buff, hwtstamp):
8898 BUILD_BUG_ON(sizeof_field(struct skb_shared_hwtstamps, hwtstamp) != 8);
8899 BUILD_BUG_ON(offsetof(struct skb_shared_hwtstamps, hwtstamp) != 0);
8900
8901 insn = bpf_convert_shinfo_access(si, insn);
8902 *insn++ = BPF_LDX_MEM(BPF_DW,
8903 si->dst_reg, si->dst_reg,
8904 bpf_target_off(struct skb_shared_info,
8905 hwtstamps, 8,
8906 target_size));
8907 break;
8887 }8908 }
8888 8909
8889 return insn - insn_buf;8910 return insn - insn_buf;
tools/bpf/bpftool/Makefile+3−0
@@ -137,7 +137,10 @@ endif · 136 unmodified lines
137BPFTOOL_BOOTSTRAP := $(BOOTSTRAP_OUTPUT)bpftool137BPFTOOL_BOOTSTRAP := $(BOOTSTRAP_OUTPUT)bpftool
138 138
139BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o xlated_dumper.o btf_dumper.o disasm.o)139BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o xlated_dumper.o btf_dumper.o disasm.o)
140$(BOOTSTRAP_OBJS): $(LIBBPF_BOOTSTRAP)
141
140OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o142OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o
143$(OBJS): $(LIBBPF)
141 144
142VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \145VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \
143 $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \146 $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \
tools/bpf/bpftool/btf.c+12−0
@@ -37,6 +37,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 36 unmodified lines
37 [BTF_KIND_VAR] = "VAR",37 [BTF_KIND_VAR] = "VAR",
38 [BTF_KIND_DATASEC] = "DATASEC",38 [BTF_KIND_DATASEC] = "DATASEC",
39 [BTF_KIND_FLOAT] = "FLOAT",39 [BTF_KIND_FLOAT] = "FLOAT",
40 [BTF_KIND_TAG] = "TAG",
40};41};
41 42
42struct btf_attach_table {43struct btf_attach_table {
@@ -347,6 +348,17 @@ static int dump_btf_type(const struct btf *btf, __u32 id, · 304 unmodified lines
347 printf(" size=%u", t->size);348 printf(" size=%u", t->size);
348 break;349 break;
349 }350 }
351 case BTF_KIND_TAG: {
352 const struct btf_tag *tag = (const void *)(t + 1);
353
354 if (json_output) {
355 jsonw_uint_field(w, "type_id", t->type);
356 jsonw_int_field(w, "component_idx", tag->component_idx);
357 } else {
358 printf(" type_id=%u component_idx=%d", t->type, tag->component_idx);
359 }
360 break;
361 }
350 default:362 default:
351 break;363 break;
352 }364 }
tools/bpf/bpftool/gen.c+19−12
@@ -238,8 +238,8 @@ static void codegen(const char *template, ...) · 237 unmodified lines
238 } else if (c == '\n') {238 } else if (c == '\n') {
239 break;239 break;
240 } else {240 } else {
241 p_err("unrecognized character at pos %td in template '%s'",241 p_err("unrecognized character at pos %td in template '%s': '%c'",
242 src - template - 1, template);242 src - template - 1, template, c);
243 free(s);243 free(s);
244 exit(-1);244 exit(-1);
245 }245 }
@@ -406,7 +406,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) · 160 unmodified lines
406 }406 }
407 407
408 bpf_object__for_each_map(map, obj) {408 bpf_object__for_each_map(map, obj) {
409 const char * ident;409 const char *ident;
410 410
411 ident = get_map_ident(map);411 ident = get_map_ident(map);
412 if (!ident)412 if (!ident)
@@ -862,6 +862,8 @@ static int do_skeleton(int argc, char **argv) · 449 unmodified lines
862 codegen("\862 codegen("\
863 \n\863 \n\
864 \n\864 \n\
865 static inline const void *%1$s__elf_bytes(size_t *sz); \n\
866 \n\
865 static inline int \n\867 static inline int \n\
866 %1$s__create_skeleton(struct %1$s *obj) \n\868 %1$s__create_skeleton(struct %1$s *obj) \n\
867 { \n\869 { \n\
@@ -943,10 +945,20 @@ static int do_skeleton(int argc, char **argv) · 75 unmodified lines
943 codegen("\945 codegen("\
944 \n\946 \n\
945 \n\947 \n\
946 s->data_sz = %d; \n\948 s->data = (void *)%2$s__elf_bytes(&s->data_sz); \n\
947 s->data = (void *)\"\\ \n\949 \n\
948 ",950 return 0; \n\
949 file_sz);951 err: \n\
952 bpf_object__destroy_skeleton(s); \n\
953 return -ENOMEM; \n\
954 } \n\
955 \n\
956 static inline const void *%2$s__elf_bytes(size_t *sz) \n\
957 { \n\
958 *sz = %1$d; \n\
959 return (const void *)\"\\ \n\
960 "
961 , file_sz, obj_name);
950 962
951 /* embed contents of BPF object file */963 /* embed contents of BPF object file */
952 print_hex(obj_data, file_sz);964 print_hex(obj_data, file_sz);
@@ -954,11 +966,6 @@ static int do_skeleton(int argc, char **argv) · 1 unmodified line
954 codegen("\966 codegen("\
955 \n\967 \n\
956 \"; \n\968 \"; \n\
957 \n\
958 return 0; \n\
959 err: \n\
960 bpf_object__destroy_skeleton(s); \n\
961 return -ENOMEM; \n\
962 } \n\969 } \n\
963 \n\970 \n\
964 #endif /* %s */ \n\971 #endif /* %s */ \n\
tools/bpf/resolve_btfids/Makefile+3−2
@@ -26,6 +26,7 @@ LIBBPF_SRC := $(srctree)/tools/lib/bpf/ · 25 unmodified lines
26SUBCMD_SRC := $(srctree)/tools/lib/subcmd/26SUBCMD_SRC := $(srctree)/tools/lib/subcmd/
27 27
28BPFOBJ := $(OUTPUT)/libbpf/libbpf.a28BPFOBJ := $(OUTPUT)/libbpf/libbpf.a
29LIBBPF_OUT := $(abspath $(dir $(BPFOBJ)))/
29SUBCMDOBJ := $(OUTPUT)/libsubcmd/libsubcmd.a30SUBCMDOBJ := $(OUTPUT)/libsubcmd/libsubcmd.a
30 31
31BINARY := $(OUTPUT)/resolve_btfids32BINARY := $(OUTPUT)/resolve_btfids
@@ -41,7 +42,7 @@ $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd · 9 unmodified lines
41 $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)42 $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
42 43
43$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf44$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)/libbpf
44 $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)45 $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT) $(abspath $@)
45 46
46CFLAGS := -g \47CFLAGS := -g \
47 -I$(srctree)/tools/include \48 -I$(srctree)/tools/include \
@@ -54,7 +55,7 @@ LIBS = -lelf -lz · 6 unmodified lines
54export srctree OUTPUT CFLAGS Q55export srctree OUTPUT CFLAGS Q
55include $(srctree)/tools/build/Makefile.include56include $(srctree)/tools/build/Makefile.include
56 57
57$(BINARY_IN): fixdep FORCE | $(OUTPUT)58$(BINARY_IN): $(BPFOBJ) fixdep FORCE | $(OUTPUT)
58 $(Q)$(MAKE) $(build)=resolve_btfids59 $(Q)$(MAKE) $(build)=resolve_btfids
59 60
60$(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)61$(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
tools/include/uapi/linux/bpf.h+25−1
@@ -1629,7 +1629,7 @@ union bpf_attr { · 1628 unmodified lines
1629 * u32 bpf_get_smp_processor_id(void)1629 * u32 bpf_get_smp_processor_id(void)
1630 * Description1630 * Description
1631 * Get the SMP (symmetric multiprocessing) processor id. Note that1631 * Get the SMP (symmetric multiprocessing) processor id. Note that
1632 * all programs run with preemption disabled, which means that the1632 * all programs run with migration disabled, which means that the
1633 * SMP processor id is stable during all the execution of the1633 * SMP processor id is stable during all the execution of the
1634 * program.1634 * program.
1635 * Return1635 * Return
@@ -4877,6 +4877,27 @@ union bpf_attr { · 3241 unmodified lines
4877 * Get the struct pt_regs associated with **task**.4877 * Get the struct pt_regs associated with **task**.
4878 * Return4878 * Return
4879 * A pointer to struct pt_regs.4879 * A pointer to struct pt_regs.
4880 *
4881 * long bpf_get_branch_snapshot(void *entries, u32 size, u64 flags)
4882 * Description
4883 * Get branch trace from hardware engines like Intel LBR. The
4884 * hardware engine is stopped shortly after the helper is
4885 * called. Therefore, the user need to filter branch entries
4886 * based on the actual use case. To capture branch trace
4887 * before the trigger point of the BPF program, the helper
4888 * should be called at the beginning of the BPF program.
4889 *
4890 * The data is stored as struct perf_branch_entry into output
4891 * buffer *entries*. *size* is the size of *entries* in bytes.
4892 * *flags* is reserved for now and must be zero.
4893 *
4894 * Return
4895 * On success, number of bytes written to *buf*. On error, a
4896 * negative value.
4897 *
4898 * **-EINVAL** if *flags* is not zero.
4899 *
4900 * **-ENOENT** if architecture does not support branch records.
4880 */4901 */
4881#define __BPF_FUNC_MAPPER(FN) \4902#define __BPF_FUNC_MAPPER(FN) \
4882 FN(unspec), \4903 FN(unspec), \
@@ -5055,6 +5076,7 @@ union bpf_attr { · 172 unmodified lines
5055 FN(get_func_ip), \5076 FN(get_func_ip), \
5056 FN(get_attach_cookie), \5077 FN(get_attach_cookie), \
5057 FN(task_pt_regs), \5078 FN(task_pt_regs), \
5079 FN(get_branch_snapshot), \
5058 /* */5080 /* */
5059 5081
5060/* integer value in 'imm' field of BPF_CALL instruction selects which helper5082/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -5284,6 +5306,8 @@ struct __sk_buff { · 223 unmodified lines
5284 __u32 gso_segs;5306 __u32 gso_segs;
5285 __bpf_md_ptr(struct bpf_sock *, sk);5307 __bpf_md_ptr(struct bpf_sock *, sk);
5286 __u32 gso_size;5308 __u32 gso_size;
5309 __u32 :32; /* Padding, future use. */
5310 __u64 hwtstamp;
5287};5311};
5288 5312
5289struct bpf_tunnel_key {5313struct bpf_tunnel_key {
tools/include/uapi/linux/btf.h+35−20
@@ -43,7 +43,7 @@ struct btf_type { · 42 unmodified lines
43 * "size" tells the size of the type it is describing.43 * "size" tells the size of the type it is describing.
44 *44 *
45 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,45 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
46 * FUNC, FUNC_PROTO and VAR.46 * FUNC, FUNC_PROTO, VAR and TAG.
47 * "type" is a type_id referring to another type.47 * "type" is a type_id referring to another type.
48 */48 */
49 union {49 union {
@@ -56,25 +56,29 @@ struct btf_type { · 6 unmodified lines
56#define BTF_INFO_VLEN(info) ((info) & 0xffff)56#define BTF_INFO_VLEN(info) ((info) & 0xffff)
57#define BTF_INFO_KFLAG(info) ((info) >> 31)57#define BTF_INFO_KFLAG(info) ((info) >> 31)
58 58
59#define BTF_KIND_UNKN 0 /* Unknown */59enum {
60#define BTF_KIND_INT 1 /* Integer */60 BTF_KIND_UNKN = 0, /* Unknown */
61#define BTF_KIND_PTR 2 /* Pointer */61 BTF_KIND_INT = 1, /* Integer */
62#define BTF_KIND_ARRAY 3 /* Array */62 BTF_KIND_PTR = 2, /* Pointer */
63#define BTF_KIND_STRUCT 4 /* Struct */63 BTF_KIND_ARRAY = 3, /* Array */
64#define BTF_KIND_UNION 5 /* Union */64 BTF_KIND_STRUCT = 4, /* Struct */
65#define BTF_KIND_ENUM 6 /* Enumeration */65 BTF_KIND_UNION = 5, /* Union */
66#define BTF_KIND_FWD 7 /* Forward */66 BTF_KIND_ENUM = 6, /* Enumeration */
67#define BTF_KIND_TYPEDEF 8 /* Typedef */67 BTF_KIND_FWD = 7, /* Forward */
68#define BTF_KIND_VOLATILE 9 /* Volatile */68 BTF_KIND_TYPEDEF = 8, /* Typedef */
69#define BTF_KIND_CONST 10 /* Const */69 BTF_KIND_VOLATILE = 9, /* Volatile */
70#define BTF_KIND_RESTRICT 11 /* Restrict */70 BTF_KIND_CONST = 10, /* Const */
71#define BTF_KIND_FUNC 12 /* Function */71 BTF_KIND_RESTRICT = 11, /* Restrict */
72#define BTF_KIND_FUNC_PROTO 13 /* Function Proto */72 BTF_KIND_FUNC = 12, /* Function */
73#define BTF_KIND_VAR 14 /* Variable */73 BTF_KIND_FUNC_PROTO = 13, /* Function Proto */
74#define BTF_KIND_DATASEC 15 /* Section */74 BTF_KIND_VAR = 14, /* Variable */
75#define BTF_KIND_FLOAT 16 /* Floating point */75 BTF_KIND_DATASEC = 15, /* Section */
76#define BTF_KIND_MAX BTF_KIND_FLOAT76 BTF_KIND_FLOAT = 16, /* Floating point */
77#define NR_BTF_KINDS (BTF_KIND_MAX + 1)77 BTF_KIND_TAG = 17, /* Tag */
78
79 NR_BTF_KINDS,
80 BTF_KIND_MAX = NR_BTF_KINDS - 1,
81};
78 82
79/* For some specific BTF_KIND, "struct btf_type" is immediately83/* For some specific BTF_KIND, "struct btf_type" is immediately
80 * followed by extra data.84 * followed by extra data.
@@ -170,4 +174,15 @@ struct btf_var_secinfo { · 89 unmodified lines
170 __u32 size;174 __u32 size;
171};175};
172 176
177/* BTF_KIND_TAG is followed by a single "struct btf_tag" to describe
178 * additional information related to the tag applied location.
179 * If component_idx == -1, the tag is applied to a struct, union,
180 * variable or function. Otherwise, it is applied to a struct/union
181 * member or a func argument, and component_idx indicates which member
182 * or argument (0 ... vlen-1).
183 */
184struct btf_tag {
185 __s32 component_idx;
186};
187
173#endif /* _UAPI__LINUX_BTF_H__ */188#endif /* _UAPI__LINUX_BTF_H__ */
tools/lib/bpf/.gitignore+0−1
@@ -1,5 +1,4 @@
1# SPDX-License-Identifier: GPL-2.0-only1# SPDX-License-Identifier: GPL-2.0-only
2libbpf_version.h
3libbpf.pc2libbpf.pc
4libbpf.so.*3libbpf.so.*
5TAGS4TAGS
tools/lib/bpf/Makefile+29−10
@@ -8,7 +8,8 @@ VERSION_SCRIPT := libbpf.map · 7 unmodified lines
8LIBBPF_VERSION := $(shell \8LIBBPF_VERSION := $(shell \
9 grep -oE '^LIBBPF_([0-9.]+)' $(VERSION_SCRIPT) | \9 grep -oE '^LIBBPF_([0-9.]+)' $(VERSION_SCRIPT) | \
10 sort -rV | head -n1 | cut -d'_' -f2)10 sort -rV | head -n1 | cut -d'_' -f2)
11LIBBPF_MAJOR_VERSION := $(firstword $(subst ., ,$(LIBBPF_VERSION)))11LIBBPF_MAJOR_VERSION := $(word 1,$(subst ., ,$(LIBBPF_VERSION)))
12LIBBPF_MINOR_VERSION := $(word 2,$(subst ., ,$(LIBBPF_VERSION)))
12 13
13MAKEFLAGS += --no-print-directory14MAKEFLAGS += --no-print-directory
14 15
@@ -59,7 +60,8 @@ ifndef VERBOSE · 44 unmodified lines
59 VERBOSE = 060 VERBOSE = 0
60endif61endif
61 62
62INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi63INCLUDES = -I$(if $(OUTPUT),$(OUTPUT),.) \
64 -I$(srctree)/tools/include -I$(srctree)/tools/include/uapi
63 65
64export prefix libdir src obj66export prefix libdir src obj
65 67
@@ -112,6 +114,7 @@ STATIC_OBJDIR := $(OUTPUT)staticobjs/ · 46 unmodified lines
112BPF_IN_SHARED := $(SHARED_OBJDIR)libbpf-in.o114BPF_IN_SHARED := $(SHARED_OBJDIR)libbpf-in.o
113BPF_IN_STATIC := $(STATIC_OBJDIR)libbpf-in.o115BPF_IN_STATIC := $(STATIC_OBJDIR)libbpf-in.o
114BPF_HELPER_DEFS := $(OUTPUT)bpf_helper_defs.h116BPF_HELPER_DEFS := $(OUTPUT)bpf_helper_defs.h
117BPF_GENERATED := $(BPF_HELPER_DEFS)
115 118
116LIB_TARGET := $(addprefix $(OUTPUT),$(LIB_TARGET))119LIB_TARGET := $(addprefix $(OUTPUT),$(LIB_TARGET))
117LIB_FILE := $(addprefix $(OUTPUT),$(LIB_FILE))120LIB_FILE := $(addprefix $(OUTPUT),$(LIB_FILE))
@@ -136,7 +139,7 @@ all: fixdep · 18 unmodified lines
136 139
137all_cmd: $(CMD_TARGETS) check140all_cmd: $(CMD_TARGETS) check
138 141
139$(BPF_IN_SHARED): force $(BPF_HELPER_DEFS)142$(BPF_IN_SHARED): force $(BPF_GENERATED)
140 @(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \143 @(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \
141 (diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \144 (diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \
142 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true145 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true
@@ -154,7 +157,7 @@ $(BPF_IN_SHARED): force $(BPF_HELPER_DEFS) · 11 unmodified lines
154 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true157 echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true
155 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(SHARED_OBJDIR) CFLAGS="$(CFLAGS) $(SHLIB_FLAGS)"158 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(SHARED_OBJDIR) CFLAGS="$(CFLAGS) $(SHLIB_FLAGS)"
156 159
157$(BPF_IN_STATIC): force $(BPF_HELPER_DEFS)160$(BPF_IN_STATIC): force $(BPF_GENERATED)
158 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR)161 $(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR)
159 162
160$(BPF_HELPER_DEFS): $(srctree)/tools/include/uapi/linux/bpf.h163$(BPF_HELPER_DEFS): $(srctree)/tools/include/uapi/linux/bpf.h
@@ -179,7 +182,7 @@ $(OUTPUT)libbpf.pc: · 18 unmodified lines
179 -e "s|@VERSION@|$(LIBBPF_VERSION)|" \182 -e "s|@VERSION@|$(LIBBPF_VERSION)|" \
180 < libbpf.pc.template > $@183 < libbpf.pc.template > $@
181 184
182check: check_abi185check: check_abi check_version
183 186
184check_abi: $(OUTPUT)libbpf.so $(VERSION_SCRIPT)187check_abi: $(OUTPUT)libbpf.so $(VERSION_SCRIPT)
185 @if [ "$(GLOBAL_SYM_COUNT)" != "$(VERSIONED_SYM_COUNT)" ]; then \188 @if [ "$(GLOBAL_SYM_COUNT)" != "$(VERSIONED_SYM_COUNT)" ]; then \
@@ -205,6 +208,21 @@ check_abi: $(OUTPUT)libbpf.so $(VERSION_SCRIPT) · 19 unmodified lines
205 exit 1; \208 exit 1; \
206 fi209 fi
207 210
211HDR_MAJ_VERSION := $(shell grep -oE '^\#define LIBBPF_MAJOR_VERSION ([0-9]+)$$' libbpf_version.h | cut -d' ' -f3)
212HDR_MIN_VERSION := $(shell grep -oE '^\#define LIBBPF_MINOR_VERSION ([0-9]+)$$' libbpf_version.h | cut -d' ' -f3)
213
214check_version: $(VERSION_SCRIPT) libbpf_version.h
215 @if [ "$(HDR_MAJ_VERSION)" != "$(LIBBPF_MAJOR_VERSION)" ]; then \
216 echo "Error: libbpf major version mismatch detected: " \
217 "'$(HDR_MAJ_VERSION)' != '$(LIBBPF_MAJOR_VERSION)'" >&2; \
218 exit 1; \
219 fi
220 @if [ "$(HDR_MIN_VERSION)" != "$(LIBBPF_MINOR_VERSION)" ]; then \
221 echo "Error: libbpf minor version mismatch detected: " \
222 "'$(HDR_MIN_VERSION)' != '$(LIBBPF_MINOR_VERSION)'" >&2; \
223 exit 1; \
224 fi
225
208define do_install_mkdir226define do_install_mkdir
209 if [ ! -d '$(DESTDIR_SQ)$1' ]; then \227 if [ ! -d '$(DESTDIR_SQ)$1' ]; then \
210 $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$1'; \228 $(INSTALL) -d -m 755 '$(DESTDIR_SQ)$1'; \
@@ -224,10 +242,11 @@ install_lib: all_cmd · 13 unmodified lines
224 cp -fpR $(LIB_FILE) $(DESTDIR)$(libdir_SQ)242 cp -fpR $(LIB_FILE) $(DESTDIR)$(libdir_SQ)
225 243
226INSTALL_HEADERS = bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h xsk.h \244INSTALL_HEADERS = bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h xsk.h \
227 bpf_helpers.h $(BPF_HELPER_DEFS) bpf_tracing.h \245 bpf_helpers.h $(BPF_GENERATED) bpf_tracing.h \
228 bpf_endian.h bpf_core_read.h skel_internal.h246 bpf_endian.h bpf_core_read.h skel_internal.h \
247 libbpf_version.h
229 248
230install_headers: $(BPF_HELPER_DEFS)249install_headers: $(BPF_GENERATED)
231 $(call QUIET_INSTALL, headers) \250 $(call QUIET_INSTALL, headers) \
232 $(foreach hdr,$(INSTALL_HEADERS), \251 $(foreach hdr,$(INSTALL_HEADERS), \
233 $(call do_install,$(hdr),$(prefix)/include/bpf,644);)252 $(call do_install,$(hdr),$(prefix)/include/bpf,644);)
@@ -240,12 +259,12 @@ install: install_lib install_pkgconfig install_headers · 6 unmodified lines
240 259
241clean:260clean:
242 $(call QUIET_CLEAN, libbpf) $(RM) -rf $(CMD_TARGETS) \261 $(call QUIET_CLEAN, libbpf) $(RM) -rf $(CMD_TARGETS) \
243 *~ .*.d .*.cmd LIBBPF-CFLAGS $(BPF_HELPER_DEFS) \262 *~ .*.d .*.cmd LIBBPF-CFLAGS $(BPF_GENERATED) \
244 $(SHARED_OBJDIR) $(STATIC_OBJDIR) \263 $(SHARED_OBJDIR) $(STATIC_OBJDIR) \
245 $(addprefix $(OUTPUT), \264 $(addprefix $(OUTPUT), \
246 *.o *.a *.so *.so.$(LIBBPF_MAJOR_VERSION) *.pc)265 *.o *.a *.so *.so.$(LIBBPF_MAJOR_VERSION) *.pc)
247 266
248PHONY += force cscope tags267PHONY += force cscope tags check check_abi check_version
249force:268force:
250 269
251cscope:270cscope:
tools/lib/bpf/btf.c+76−8
@@ -304,6 +304,8 @@ static int btf_type_size(const struct btf_type *t) · 303 unmodified lines
304 return base_size + sizeof(struct btf_var);304 return base_size + sizeof(struct btf_var);
305 case BTF_KIND_DATASEC:305 case BTF_KIND_DATASEC:
306 return base_size + vlen * sizeof(struct btf_var_secinfo);306 return base_size + vlen * sizeof(struct btf_var_secinfo);
307 case BTF_KIND_TAG:
308 return base_size + sizeof(struct btf_tag);
307 default:309 default:
308 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));310 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
309 return -EINVAL;311 return -EINVAL;
@@ -376,6 +378,9 @@ static int btf_bswap_type_rest(struct btf_type *t) · 66 unmodified lines
376 v->size = bswap_32(v->size);378 v->size = bswap_32(v->size);
377 }379 }
378 return 0;380 return 0;
381 case BTF_KIND_TAG:
382 btf_tag(t)->component_idx = bswap_32(btf_tag(t)->component_idx);
383 return 0;
379 default:384 default:
380 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));385 pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
381 return -EINVAL;386 return -EINVAL;
@@ -586,6 +591,7 @@ __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) · 204 unmodified lines
586 case BTF_KIND_CONST:591 case BTF_KIND_CONST:
587 case BTF_KIND_RESTRICT:592 case BTF_KIND_RESTRICT:
588 case BTF_KIND_VAR:593 case BTF_KIND_VAR:
594 case BTF_KIND_TAG:
589 type_id = t->type;595 type_id = t->type;
590 break;596 break;
591 case BTF_KIND_ARRAY:597 case BTF_KIND_ARRAY:
@@ -2440,6 +2446,48 @@ int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __ · 1848 unmodified lines
2440 return 0;2446 return 0;
2441}2447}
2442 2448
2449/*
2450 * Append new BTF_KIND_TAG type with:
2451 * - *value* - non-empty/non-NULL string;
2452 * - *ref_type_id* - referenced type ID, it might not exist yet;
2453 * - *component_idx* - -1 for tagging reference type, otherwise struct/union
2454 * member or function argument index;
2455 * Returns:
2456 * - >0, type ID of newly added BTF type;
2457 * - <0, on error.
2458 */
2459int btf__add_tag(struct btf *btf, const char *value, int ref_type_id,
2460 int component_idx)
2461{
2462 struct btf_type *t;
2463 int sz, value_off;
2464
2465 if (!value || !value[0] || component_idx < -1)
2466 return libbpf_err(-EINVAL);
2467
2468 if (validate_type_id(ref_type_id))
2469 return libbpf_err(-EINVAL);
2470
2471 if (btf_ensure_modifiable(btf))
2472 return libbpf_err(-ENOMEM);
2473
2474 sz = sizeof(struct btf_type) + sizeof(struct btf_tag);
2475 t = btf_add_type_mem(btf, sz);
2476 if (!t)
2477 return libbpf_err(-ENOMEM);
2478
2479 value_off = btf__add_str(btf, value);
2480 if (value_off < 0)
2481 return value_off;
2482
2483 t->name_off = value_off;
2484 t->info = btf_type_info(BTF_KIND_TAG, 0, false);
2485 t->type = ref_type_id;
2486 btf_tag(t)->component_idx = component_idx;
2487
2488 return btf_commit_type(btf, sz);
2489}
2490
2443struct btf_ext_sec_setup_param {2491struct btf_ext_sec_setup_param {
2444 __u32 off;2492 __u32 off;
2445 __u32 len;2493 __u32 len;
@@ -3256,8 +3304,8 @@ static bool btf_equal_common(struct btf_type *t1, struct btf_type *t2) · 810 unmodified lines
3256 t1->size == t2->size;3304 t1->size == t2->size;
3257}3305}
3258 3306
3259/* Calculate type signature hash of INT. */3307/* Calculate type signature hash of INT or TAG. */
3260static long btf_hash_int(struct btf_type *t)3308static long btf_hash_int_tag(struct btf_type *t)
3261{3309{
3262 __u32 info = *(__u32 *)(t + 1);3310 __u32 info = *(__u32 *)(t + 1);
3263 long h;3311 long h;
@@ -3267,8 +3315,8 @@ static long btf_hash_int(struct btf_type *t) · 3 unmodified lines
3267 return h;3315 return h;
3268}3316}
3269 3317
3270/* Check structural equality of two INTs. */3318/* Check structural equality of two INTs or TAGs. */
3271static bool btf_equal_int(struct btf_type *t1, struct btf_type *t2)3319static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2)
3272{3320{
3273 __u32 info1, info2;3321 __u32 info1, info2;
3274 3322
@@ -3535,7 +3583,8 @@ static int btf_dedup_prep(struct btf_dedup *d) · 260 unmodified lines
3535 h = btf_hash_common(t);3583 h = btf_hash_common(t);
3536 break;3584 break;
3537 case BTF_KIND_INT:3585 case BTF_KIND_INT:
3538 h = btf_hash_int(t);3586 case BTF_KIND_TAG:
3587 h = btf_hash_int_tag(t);
3539 break;3588 break;
3540 case BTF_KIND_ENUM:3589 case BTF_KIND_ENUM:
3541 h = btf_hash_enum(t);3590 h = btf_hash_enum(t);
@@ -3590,14 +3639,15 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) · 48 unmodified lines
3590 case BTF_KIND_FUNC_PROTO:3639 case BTF_KIND_FUNC_PROTO:
3591 case BTF_KIND_VAR:3640 case BTF_KIND_VAR:
3592 case BTF_KIND_DATASEC:3641 case BTF_KIND_DATASEC:
3642 case BTF_KIND_TAG:
3593 return 0;3643 return 0;
3594 3644
3595 case BTF_KIND_INT:3645 case BTF_KIND_INT:
3596 h = btf_hash_int(t);3646 h = btf_hash_int_tag(t);
3597 for_each_dedup_cand(d, hash_entry, h) {3647 for_each_dedup_cand(d, hash_entry, h) {
3598 cand_id = (__u32)(long)hash_entry->value;3648 cand_id = (__u32)(long)hash_entry->value;
3599 cand = btf_type_by_id(d->btf, cand_id);3649 cand = btf_type_by_id(d->btf, cand_id);
3600 if (btf_equal_int(t, cand)) {3650 if (btf_equal_int_tag(t, cand)) {
3601 new_id = cand_id;3651 new_id = cand_id;
3602 break;3652 break;
3603 }3653 }
@@ -3881,7 +3931,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, · 277 unmodified lines
3881 3931
3882 switch (cand_kind) {3932 switch (cand_kind) {
3883 case BTF_KIND_INT:3933 case BTF_KIND_INT:
3884 return btf_equal_int(cand_type, canon_type);3934 return btf_equal_int_tag(cand_type, canon_type);
3885 3935
3886 case BTF_KIND_ENUM:3936 case BTF_KIND_ENUM:
3887 if (d->opts.dont_resolve_fwds)3937 if (d->opts.dont_resolve_fwds)
@@ -4210,6 +4260,23 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) · 322 unmodified lines
4210 }4260 }
4211 break;4261 break;
4212 4262
4263 case BTF_KIND_TAG:
4264 ref_type_id = btf_dedup_ref_type(d, t->type);
4265 if (ref_type_id < 0)
4266 return ref_type_id;
4267 t->type = ref_type_id;
4268
4269 h = btf_hash_int_tag(t);
4270 for_each_dedup_cand(d, hash_entry, h) {
4271 cand_id = (__u32)(long)hash_entry->value;
4272 cand = btf_type_by_id(d->btf, cand_id);
4273 if (btf_equal_int_tag(t, cand)) {
4274 new_id = cand_id;
4275 break;
4276 }
4277 }
4278 break;
4279
4213 case BTF_KIND_ARRAY: {4280 case BTF_KIND_ARRAY: {
4214 struct btf_array *info = btf_array(t);4281 struct btf_array *info = btf_array(t);
4215 4282
@@ -4482,6 +4549,7 @@ int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ct · 266 unmodified lines
4482 case BTF_KIND_TYPEDEF:4549 case BTF_KIND_TYPEDEF:
4483 case BTF_KIND_FUNC:4550 case BTF_KIND_FUNC:
4484 case BTF_KIND_VAR:4551 case BTF_KIND_VAR:
4552 case BTF_KIND_TAG:
4485 return visit(&t->type, ctx);4553 return visit(&t->type, ctx);
4486 4554
4487 case BTF_KIND_ARRAY: {4555 case BTF_KIND_ARRAY: {
tools/lib/bpf/btf.h+87−0
@@ -1,5 +1,6 @@
1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2/* Copyright (c) 2018 Facebook */2/* Copyright (c) 2018 Facebook */
3/*! \file */
3 4
4#ifndef __LIBBPF_BTF_H5#ifndef __LIBBPF_BTF_H
5#define __LIBBPF_BTF_H6#define __LIBBPF_BTF_H
@@ -30,11 +31,80 @@ enum btf_endianness { · 24 unmodified lines
30 BTF_BIG_ENDIAN = 1,31 BTF_BIG_ENDIAN = 1,
31};32};
32 33
34/**
35 * @brief **btf__free()** frees all data of a BTF object
36 * @param btf BTF object to free
37 */
33LIBBPF_API void btf__free(struct btf *btf);38LIBBPF_API void btf__free(struct btf *btf);
34 39
40/**
41 * @brief **btf__new()** creates a new instance of a BTF object from the raw
42 * bytes of an ELF's BTF section
43 * @param data raw bytes
44 * @param size number of bytes passed in `data`
45 * @return new BTF object instance which has to be eventually freed with
46 * **btf__free()**
47 *
48 * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract
49 * error code from such a pointer `libbpf_get_error()` should be used. If
50 * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is
51 * returned on error instead. In both cases thread-local `errno` variable is
52 * always set to error code as well.
53 */
35LIBBPF_API struct btf *btf__new(const void *data, __u32 size);54LIBBPF_API struct btf *btf__new(const void *data, __u32 size);
55
56/**
57 * @brief **btf__new_split()** create a new instance of a BTF object from the
58 * provided raw data bytes. It takes another BTF instance, **base_btf**, which
59 * serves as a base BTF, which is extended by types in a newly created BTF
60 * instance
61 * @param data raw bytes
62 * @param size length of raw bytes
63 * @param base_btf the base BTF object
64 * @return new BTF object instance which has to be eventually freed with
65 * **btf__free()**
66 *
67 * If *base_btf* is NULL, `btf__new_split()` is equivalent to `btf__new()` and
68 * creates non-split BTF.
69 *
70 * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract
71 * error code from such a pointer `libbpf_get_error()` should be used. If
72 * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is
73 * returned on error instead. In both cases thread-local `errno` variable is
74 * always set to error code as well.
75 */
36LIBBPF_API struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf);76LIBBPF_API struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf);
77
78/**
79 * @brief **btf__new_empty()** creates an empty BTF object. Use
80 * `btf__add_*()` to populate such BTF object.
81 * @return new BTF object instance which has to be eventually freed with
82 * **btf__free()**
83 *
84 * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract
85 * error code from such a pointer `libbpf_get_error()` should be used. If
86 * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is
87 * returned on error instead. In both cases thread-local `errno` variable is
88 * always set to error code as well.
89 */
37LIBBPF_API struct btf *btf__new_empty(void);90LIBBPF_API struct btf *btf__new_empty(void);
91
92/**
93 * @brief **btf__new_empty_split()** creates an unpopulated BTF object from an
94 * ELF BTF section except with a base BTF on top of which split BTF should be
95 * based
96 * @return new BTF object instance which has to be eventually freed with
97 * **btf__free()**
98 *
99 * If *base_btf* is NULL, `btf__new_empty_split()` is equivalent to
100 * `btf__new_empty()` and creates non-split BTF.
101 *
102 * On error, error-code-encoded-as-pointer is returned, not a NULL. To extract
103 * error code from such a pointer `libbpf_get_error()` should be used. If
104 * `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` is enabled, NULL is
105 * returned on error instead. In both cases thread-local `errno` variable is
106 * always set to error code as well.
107 */
38LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf);108LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf);
39 109
40LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext);110LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext);
@@ -50,9 +120,11 @@ LIBBPF_API struct btf *libbpf_find_kernel_btf(void); · 9 unmodified lines
50 120
51LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id);121LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id);
52LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf);122LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf);
123LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_from_kernel_by_id instead")
53LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);124LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
54 125
55LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);126LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);
127LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_into_kernel instead")
56LIBBPF_API int btf__load(struct btf *btf);128LIBBPF_API int btf__load(struct btf *btf);
57LIBBPF_API int btf__load_into_kernel(struct btf *btf);129LIBBPF_API int btf__load_into_kernel(struct btf *btf);
58LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,130LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,
@@ -141,6 +213,10 @@ LIBBPF_API int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz · 82 unmodified lines
141LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id,213LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id,
142 __u32 offset, __u32 byte_sz);214 __u32 offset, __u32 byte_sz);
143 215
216/* tag construction API */
217LIBBPF_API int btf__add_tag(struct btf *btf, const char *value, int ref_type_id,
218 int component_idx);
219
144struct btf_dedup_opts {220struct btf_dedup_opts {
145 unsigned int dedup_table_size;221 unsigned int dedup_table_size;
146 bool dont_resolve_fwds;222 bool dont_resolve_fwds;
@@ -328,6 +404,11 @@ static inline bool btf_is_float(const struct btf_type *t) · 181 unmodified lines
328 return btf_kind(t) == BTF_KIND_FLOAT;404 return btf_kind(t) == BTF_KIND_FLOAT;
329}405}
330 406
407static inline bool btf_is_tag(const struct btf_type *t)
408{
409 return btf_kind(t) == BTF_KIND_TAG;
410}
411
331static inline __u8 btf_int_encoding(const struct btf_type *t)412static inline __u8 btf_int_encoding(const struct btf_type *t)
332{413{
333 return BTF_INT_ENCODING(*(__u32 *)(t + 1));414 return BTF_INT_ENCODING(*(__u32 *)(t + 1));
@@ -396,6 +477,12 @@ btf_var_secinfos(const struct btf_type *t) · 62 unmodified lines
396 return (struct btf_var_secinfo *)(t + 1);477 return (struct btf_var_secinfo *)(t + 1);
397}478}
398 479
480struct btf_tag;
481static inline struct btf_tag *btf_tag(const struct btf_type *t)
482{
483 return (struct btf_tag *)(t + 1);
484}
485
399#ifdef __cplusplus486#ifdef __cplusplus
400} /* extern "C" */487} /* extern "C" */
401#endif488#endif
tools/lib/bpf/btf_dump.c+3−0
@@ -316,6 +316,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d) · 315 unmodified lines
316 case BTF_KIND_TYPEDEF:316 case BTF_KIND_TYPEDEF:
317 case BTF_KIND_FUNC:317 case BTF_KIND_FUNC:
318 case BTF_KIND_VAR:318 case BTF_KIND_VAR:
319 case BTF_KIND_TAG:
319 d->type_states[t->type].referenced = 1;320 d->type_states[t->type].referenced = 1;
320 break;321 break;
321 322
@@ -583,6 +584,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) · 261 unmodified lines
583 case BTF_KIND_FUNC:584 case BTF_KIND_FUNC:
584 case BTF_KIND_VAR:585 case BTF_KIND_VAR:
585 case BTF_KIND_DATASEC:586 case BTF_KIND_DATASEC:
587 case BTF_KIND_TAG:
586 d->type_states[id].order_state = ORDERED;588 d->type_states[id].order_state = ORDERED;
587 return 0;589 return 0;
588 590
@@ -2215,6 +2217,7 @@ static int btf_dump_dump_type_data(struct btf_dump *d, · 1626 unmodified lines
2215 case BTF_KIND_FWD:2217 case BTF_KIND_FWD:
2216 case BTF_KIND_FUNC:2218 case BTF_KIND_FUNC:
2217 case BTF_KIND_FUNC_PROTO:2219 case BTF_KIND_FUNC_PROTO:
2220 case BTF_KIND_TAG:
2218 err = btf_dump_unsupported_data(d, t, id);2221 err = btf_dump_unsupported_data(d, t, id);
2219 break;2222 break;
2220 case BTF_KIND_INT:2223 case BTF_KIND_INT:
tools/lib/bpf/libbpf.c+288−143
@@ -195,6 +195,8 @@ enum kern_feature_id { · 194 unmodified lines
195 FEAT_BTF_FLOAT,195 FEAT_BTF_FLOAT,
196 /* BPF perf link support */196 /* BPF perf link support */
197 FEAT_PERF_LINK,197 FEAT_PERF_LINK,
198 /* BTF_KIND_TAG support */
199 FEAT_BTF_TAG,
198 __FEAT_CNT,200 __FEAT_CNT,
199};201};
200 202
@@ -218,8 +220,7 @@ struct reloc_desc { · 17 unmodified lines
218 220
219struct bpf_sec_def;221struct bpf_sec_def;
220 222
221typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec,223typedef struct bpf_link *(*attach_fn_t)(const struct bpf_program *prog);
222 struct bpf_program *prog);
223 224
224struct bpf_sec_def {225struct bpf_sec_def {
225 const char *sec;226 const char *sec;
@@ -1987,6 +1988,7 @@ static const char *__btf_kind_str(__u16 kind) · 1761 unmodified lines
1987 case BTF_KIND_VAR: return "var";1988 case BTF_KIND_VAR: return "var";
1988 case BTF_KIND_DATASEC: return "datasec";1989 case BTF_KIND_DATASEC: return "datasec";
1989 case BTF_KIND_FLOAT: return "float";1990 case BTF_KIND_FLOAT: return "float";
1991 case BTF_KIND_TAG: return "tag";
1990 default: return "unknown";1992 default: return "unknown";
1991 }1993 }
1992}1994}
@@ -2486,8 +2488,9 @@ static bool btf_needs_sanitization(struct bpf_object *obj) · 493 unmodified lines
2486 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);2488 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2487 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);2489 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2488 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);2490 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2491 bool has_tag = kernel_supports(obj, FEAT_BTF_TAG);
2489 2492
2490 return !has_func || !has_datasec || !has_func_global || !has_float;2493 return !has_func || !has_datasec || !has_func_global || !has_float || !has_tag;
2491}2494}
2492 2495
2493static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)2496static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
@@ -2496,14 +2499,15 @@ static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) · 2 unmodified lines
2496 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);2499 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2497 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);2500 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2498 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);2501 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2502 bool has_tag = kernel_supports(obj, FEAT_BTF_TAG);
2499 struct btf_type *t;2503 struct btf_type *t;
2500 int i, j, vlen;2504 int i, j, vlen;
2501 2505
2502 for (i = 1; i <= btf__get_nr_types(btf); i++) {2506 for (i = 1; i <= btf__get_nr_types(btf); i++) {
2503 t = (struct btf_type *)btf__type_by_id(btf, i);2507 t = (struct btf_type *)btf__type_by_id(btf, i);
2504 2508
2505 if (!has_datasec && btf_is_var(t)) {2509 if ((!has_datasec && btf_is_var(t)) || (!has_tag && btf_is_tag(t))) {
2506 /* replace VAR with INT */2510 /* replace VAR/TAG with INT */
2507 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);2511 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2508 /*2512 /*
2509 * using size = 1 is the safest choice, 4 will be too2513 * using size = 1 is the safest choice, 4 will be too
@@ -2993,6 +2997,12 @@ static int bpf_object__elf_collect(struct bpf_object *obj) · 483 unmodified lines
2993 }2997 }
2994 }2998 }
2995 2999
3000 if (!obj->efile.symbols) {
3001 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3002 obj->path);
3003 return -ENOENT;
3004 }
3005
2996 scn = NULL;3006 scn = NULL;
2997 while ((scn = elf_nextscn(elf, scn)) != NULL) {3007 while ((scn = elf_nextscn(elf, scn)) != NULL) {
2998 idx++;3008 idx++;
@@ -4207,6 +4217,23 @@ static int probe_kern_btf_float(void) · 1208 unmodified lines
4207 strs, sizeof(strs)));4217 strs, sizeof(strs)));
4208}4218}
4209 4219
4220static int probe_kern_btf_tag(void)
4221{
4222 static const char strs[] = "\0tag";
4223 __u32 types[] = {
4224 /* int */
4225 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
4226 /* VAR x */ /* [2] */
4227 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4228 BTF_VAR_STATIC,
4229 /* attr */
4230 BTF_TYPE_TAG_ENC(1, 2, -1),
4231 };
4232
4233 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4234 strs, sizeof(strs)));
4235}
4236
4210static int probe_kern_array_mmap(void)4237static int probe_kern_array_mmap(void)
4211{4238{
4212 struct bpf_create_map_attr attr = {4239 struct bpf_create_map_attr attr = {
@@ -4423,6 +4450,9 @@ static struct kern_feature_desc { · 210 unmodified lines
4423 [FEAT_PERF_LINK] = {4450 [FEAT_PERF_LINK] = {
4424 "BPF perf link support", probe_perf_link,4451 "BPF perf link support", probe_perf_link,
4425 },4452 },
4453 [FEAT_BTF_TAG] = {
4454 "BTF_KIND_TAG support", probe_kern_btf_tag,
4455 },
4426};4456};
4427 4457
4428static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)4458static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
@@ -6367,12 +6397,40 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) · 1938 unmodified lines
6367 6397
6368static const struct bpf_sec_def *find_sec_def(const char *sec_name);6398static const struct bpf_sec_def *find_sec_def(const char *sec_name);
6369 6399
6400static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
6401{
6402 struct bpf_program *prog;
6403
6404 bpf_object__for_each_program(prog, obj) {
6405 prog->sec_def = find_sec_def(prog->sec_name);
6406 if (!prog->sec_def) {
6407 /* couldn't guess, but user might manually specify */
6408 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
6409 prog->name, prog->sec_name);
6410 continue;
6411 }
6412
6413 if (prog->sec_def->is_sleepable)
6414 prog->prog_flags |= BPF_F_SLEEPABLE;
6415 bpf_program__set_type(prog, prog->sec_def->prog_type);
6416 bpf_program__set_expected_attach_type(prog, prog->sec_def->expected_attach_type);
6417
6418#pragma GCC diagnostic push
6419#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
6420 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
6421 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
6422 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
6423#pragma GCC diagnostic pop
6424 }
6425
6426 return 0;
6427}
6428
6370static struct bpf_object *6429static struct bpf_object *
6371__bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,6430__bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz,
6372 const struct bpf_object_open_opts *opts)6431 const struct bpf_object_open_opts *opts)
6373{6432{
6374 const char *obj_name, *kconfig, *btf_tmp_path;6433 const char *obj_name, *kconfig, *btf_tmp_path;
6375 struct bpf_program *prog;
6376 struct bpf_object *obj;6434 struct bpf_object *obj;
6377 char tmp_name[64];6435 char tmp_name[64];
6378 int err;6436 int err;
@@ -6430,30 +6488,12 @@ __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, · 51 unmodified lines
6430 err = err ? : bpf_object__collect_externs(obj);6488 err = err ? : bpf_object__collect_externs(obj);
6431 err = err ? : bpf_object__finalize_btf(obj);6489 err = err ? : bpf_object__finalize_btf(obj);
6432 err = err ? : bpf_object__init_maps(obj, opts);6490 err = err ? : bpf_object__init_maps(obj, opts);
6491 err = err ? : bpf_object_init_progs(obj, opts);
6433 err = err ? : bpf_object__collect_relos(obj);6492 err = err ? : bpf_object__collect_relos(obj);
6434 if (err)6493 if (err)
6435 goto out;6494 goto out;
6436 bpf_object__elf_finish(obj);
6437 6495
6438 bpf_object__for_each_program(prog, obj) {6496 bpf_object__elf_finish(obj);
6439 prog->sec_def = find_sec_def(prog->sec_name);
6440 if (!prog->sec_def) {
6441 /* couldn't guess, but user might manually specify */
6442 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
6443 prog->name, prog->sec_name);
6444 continue;
6445 }
6446
6447 if (prog->sec_def->is_sleepable)
6448 prog->prog_flags |= BPF_F_SLEEPABLE;
6449 bpf_program__set_type(prog, prog->sec_def->prog_type);
6450 bpf_program__set_expected_attach_type(prog,
6451 prog->sec_def->expected_attach_type);
6452
6453 if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING ||
6454 prog->sec_def->prog_type == BPF_PROG_TYPE_EXT)
6455 prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0);
6456 }
6457 6497
6458 return obj;6498 return obj;
6459out:6499out:
@@ -7907,18 +7947,12 @@ void bpf_program__set_expected_attach_type(struct bpf_program *prog, · 1447 unmodified lines
7907 __VA_ARGS__ \7947 __VA_ARGS__ \
7908}7948}
7909 7949
7910static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,7950static struct bpf_link *attach_kprobe(const struct bpf_program *prog);
7911 struct bpf_program *prog);7951static struct bpf_link *attach_tp(const struct bpf_program *prog);
7912static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,7952static struct bpf_link *attach_raw_tp(const struct bpf_program *prog);
7913 struct bpf_program *prog);7953static struct bpf_link *attach_trace(const struct bpf_program *prog);
7914static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,7954static struct bpf_link *attach_lsm(const struct bpf_program *prog);
7915 struct bpf_program *prog);7955static struct bpf_link *attach_iter(const struct bpf_program *prog);
7916static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,
7917 struct bpf_program *prog);
7918static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,
7919 struct bpf_program *prog);
7920static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,
7921 struct bpf_program *prog);
7922 7956
7923static const struct bpf_sec_def section_defs[] = {7957static const struct bpf_sec_def section_defs[] = {
7924 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),7958 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
@@ -8244,35 +8278,37 @@ static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, · 319 unmodified lines
8244 return -EINVAL;8278 return -EINVAL;
8245 }8279 }
8246 8280
8247 if (prog->type == BPF_PROG_TYPE_UNSPEC) {8281 /* prevent the use of BPF prog with invalid type */
8248 const struct bpf_sec_def *sec_def;8282 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
8249 8283 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
8250 sec_def = find_sec_def(prog->sec_name);8284 map->name, prog->name);
8251 if (sec_def &&8285 return -EINVAL;
8252 sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {8286 }
8253 /* for pr_warn */
8254 prog->type = sec_def->prog_type;
8255 goto invalid_prog;
8256 }
8257 8287
8258 prog->type = BPF_PROG_TYPE_STRUCT_OPS;8288 /* if we haven't yet processed this BPF program, record proper
8289 * attach_btf_id and member_idx
8290 */
8291 if (!prog->attach_btf_id) {
8259 prog->attach_btf_id = st_ops->type_id;8292 prog->attach_btf_id = st_ops->type_id;
8260 prog->expected_attach_type = member_idx;8293 prog->expected_attach_type = member_idx;
8261 } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS ||
8262 prog->attach_btf_id != st_ops->type_id ||
8263 prog->expected_attach_type != member_idx) {
8264 goto invalid_prog;
8265 }8294 }
8295
8296 /* struct_ops BPF prog can be re-used between multiple
8297 * .struct_ops as long as it's the same struct_ops struct
8298 * definition and the same function pointer field
8299 */
8300 if (prog->attach_btf_id != st_ops->type_id ||
8301 prog->expected_attach_type != member_idx) {
8302 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8303 map->name, prog->name, prog->sec_name, prog->type,
8304 prog->attach_btf_id, prog->expected_attach_type, name);
8305 return -EINVAL;
8306 }
8307
8266 st_ops->progs[member_idx] = prog;8308 st_ops->progs[member_idx] = prog;
8267 }8309 }
8268 8310
8269 return 0;8311 return 0;
8270
8271invalid_prog:
8272 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
8273 map->name, prog->name, prog->sec_name, prog->type,
8274 prog->attach_btf_id, prog->expected_attach_type, name);
8275 return -EINVAL;
8276}8312}
8277 8313
8278#define BTF_TRACE_PREFIX "btf_trace_"8314#define BTF_TRACE_PREFIX "btf_trace_"
@@ -8428,28 +8464,15 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, int *btf_obj_fd, · 149 unmodified lines
8428{8464{
8429 enum bpf_attach_type attach_type = prog->expected_attach_type;8465 enum bpf_attach_type attach_type = prog->expected_attach_type;
8430 __u32 attach_prog_fd = prog->attach_prog_fd;8466 __u32 attach_prog_fd = prog->attach_prog_fd;
8431 const char *name = prog->sec_name, *attach_name;8467 const char *attach_name;
8432 const struct bpf_sec_def *sec = NULL;8468 int err = 0;
8433 int i, err = 0;
8434
8435 if (!name)
8436 return -EINVAL;
8437
8438 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {
8439 if (!section_defs[i].is_attach_btf)
8440 continue;
8441 if (strncmp(name, section_defs[i].sec, section_defs[i].len))
8442 continue;
8443
8444 sec = &section_defs[i];
8445 break;
8446 }
8447 8469
8448 if (!sec) {8470 if (!prog->sec_def || !prog->sec_def->is_attach_btf) {
8449 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n", name);8471 pr_warn("failed to identify BTF ID based on ELF section name '%s'\n",
8472 prog->sec_name);
8450 return -ESRCH;8473 return -ESRCH;
8451 }8474 }
8452 attach_name = name + sec->len;8475 attach_name = prog->sec_name + prog->sec_def->len;
8453 8476
8454 /* BPF program's BTF ID */8477 /* BPF program's BTF ID */
8455 if (attach_prog_fd) {8478 if (attach_prog_fd) {
@@ -8483,27 +8506,28 @@ int libbpf_attach_type_by_name(const char *name, · 27 unmodified lines
8483 enum bpf_attach_type *attach_type)8506 enum bpf_attach_type *attach_type)
8484{8507{
8485 char *type_names;8508 char *type_names;
8486 int i;8509 const struct bpf_sec_def *sec_def;
8487 8510
8488 if (!name)8511 if (!name)
8489 return libbpf_err(-EINVAL);8512 return libbpf_err(-EINVAL);
8490 8513
8491 for (i = 0; i < ARRAY_SIZE(section_defs); i++) {8514 sec_def = find_sec_def(name);
8492 if (strncmp(name, section_defs[i].sec, section_defs[i].len))8515 if (!sec_def) {
8493 continue;8516 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);
8494 if (!section_defs[i].is_attachable)8517 type_names = libbpf_get_type_names(true);
8495 return libbpf_err(-EINVAL);8518 if (type_names != NULL) {
8496 *attach_type = section_defs[i].expected_attach_type;8519 pr_debug("attachable section(type) names are:%s\n", type_names);
8497 return 0;8520 free(type_names);
8498 }8521 }
8499 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name);8522
8500 type_names = libbpf_get_type_names(true);8523 return libbpf_err(-EINVAL);
8501 if (type_names != NULL) {
8502 pr_debug("attachable section(type) names are:%s\n", type_names);
8503 free(type_names);
8504 }8524 }
8505 8525
8506 return libbpf_err(-EINVAL);8526 if (!sec_def->is_attachable)
8527 return libbpf_err(-EINVAL);
8528
8529 *attach_type = sec_def->expected_attach_type;
8530 return 0;
8507}8531}
8508 8532
8509int bpf_map__fd(const struct bpf_map *map)8533int bpf_map__fd(const struct bpf_map *map)
@@ -8987,9 +9011,57 @@ int bpf_link__unpin(struct bpf_link *link) · 477 unmodified lines
8987 return 0;9011 return 0;
8988}9012}
8989 9013
9014static int poke_kprobe_events(bool add, const char *name, bool retprobe, uint64_t offset)
9015{
9016 int fd, ret = 0;
9017 pid_t p = getpid();
9018 char cmd[260], probename[128], probefunc[128];
9019 const char *file = "/sys/kernel/debug/tracing/kprobe_events";
9020
9021 if (retprobe)
9022 snprintf(probename, sizeof(probename), "kretprobes/%s_libbpf_%u", name, p);
9023 else
9024 snprintf(probename, sizeof(probename), "kprobes/%s_libbpf_%u", name, p);
9025
9026 if (offset)
9027 snprintf(probefunc, sizeof(probefunc), "%s+%zu", name, (size_t)offset);
9028
9029 if (add) {
9030 snprintf(cmd, sizeof(cmd), "%c:%s %s",
9031 retprobe ? 'r' : 'p',
9032 probename,
9033 offset ? probefunc : name);
9034 } else {
9035 snprintf(cmd, sizeof(cmd), "-:%s", probename);
9036 }
9037
9038 fd = open(file, O_WRONLY | O_APPEND, 0);
9039 if (!fd)
9040 return -errno;
9041 ret = write(fd, cmd, strlen(cmd));
9042 if (ret < 0)
9043 ret = -errno;
9044 close(fd);
9045
9046 return ret;
9047}
9048
9049static inline int add_kprobe_event_legacy(const char *name, bool retprobe, uint64_t offset)
9050{
9051 return poke_kprobe_events(true, name, retprobe, offset);
9052}
9053
9054static inline int remove_kprobe_event_legacy(const char *name, bool retprobe)
9055{
9056 return poke_kprobe_events(false, name, retprobe, 0);
9057}
9058
8990struct bpf_link_perf {9059struct bpf_link_perf {
8991 struct bpf_link link;9060 struct bpf_link link;
8992 int perf_event_fd;9061 int perf_event_fd;
9062 /* legacy kprobe support: keep track of probe identifier and type */
9063 char *legacy_probe_name;
9064 bool legacy_is_retprobe;
8993};9065};
8994 9066
8995static int bpf_link_perf_detach(struct bpf_link *link)9067static int bpf_link_perf_detach(struct bpf_link *link)
@@ -9004,17 +9076,23 @@ static int bpf_link_perf_detach(struct bpf_link *link) · 8 unmodified lines
9004 close(perf_link->perf_event_fd);9076 close(perf_link->perf_event_fd);
9005 close(link->fd);9077 close(link->fd);
9006 9078
9007 return libbpf_err(err);9079 /* legacy kprobe needs to be removed after perf event fd closure */
9080 if (perf_link->legacy_probe_name)
9081 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,
9082 perf_link->legacy_is_retprobe);
9083
9084 return err;
9008}9085}
9009 9086
9010static void bpf_link_perf_dealloc(struct bpf_link *link)9087static void bpf_link_perf_dealloc(struct bpf_link *link)
9011{9088{
9012 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);9089 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9013 9090
9091 free(perf_link->legacy_probe_name);
9014 free(perf_link);9092 free(perf_link);
9015}9093}
9016 9094
9017struct bpf_link *bpf_program__attach_perf_event_opts(struct bpf_program *prog, int pfd,9095struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
9018 const struct bpf_perf_event_opts *opts)9096 const struct bpf_perf_event_opts *opts)
9019{9097{
9020 char errmsg[STRERR_BUFSIZE];9098 char errmsg[STRERR_BUFSIZE];
@@ -9089,7 +9167,7 @@ err_out: · 68 unmodified lines
9089 return libbpf_err_ptr(err);9167 return libbpf_err_ptr(err);
9090}9168}
9091 9169
9092struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, int pfd)9170struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
9093{9171{
9094 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);9172 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
9095}9173}
@@ -9124,6 +9202,18 @@ static int parse_uint_from_file(const char *file, const char *fmt) · 28 unmodified lines
9124 return ret;9202 return ret;
9125}9203}
9126 9204
9205static int determine_kprobe_perf_type_legacy(const char *func_name, bool is_retprobe)
9206{
9207 char file[192];
9208
9209 snprintf(file, sizeof(file),
9210 "/sys/kernel/debug/tracing/events/%s/%s_libbpf_%d/id",
9211 is_retprobe ? "kretprobes" : "kprobes",
9212 func_name, getpid());
9213
9214 return parse_uint_from_file(file, "%d\n");
9215}
9216
9127static int determine_kprobe_perf_type(void)9217static int determine_kprobe_perf_type(void)
9128{9218{
9129 const char *file = "/sys/bus/event_source/devices/kprobe/type";9219 const char *file = "/sys/bus/event_source/devices/kprobe/type";
@@ -9206,16 +9296,52 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 76 unmodified lines
9206 return pfd;9296 return pfd;
9207}9297}
9208 9298
9299static int perf_event_kprobe_open_legacy(bool retprobe, const char *name, uint64_t offset, int pid)
9300{
9301 struct perf_event_attr attr = {};
9302 char errmsg[STRERR_BUFSIZE];
9303 int type, pfd, err;
9304
9305 err = add_kprobe_event_legacy(name, retprobe, offset);
9306 if (err < 0) {
9307 pr_warn("failed to add legacy kprobe event: %s\n",
9308 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9309 return err;
9310 }
9311 type = determine_kprobe_perf_type_legacy(name, retprobe);
9312 if (type < 0) {
9313 pr_warn("failed to determine legacy kprobe event id: %s\n",
9314 libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
9315 return type;
9316 }
9317 attr.size = sizeof(attr);
9318 attr.config = type;
9319 attr.type = PERF_TYPE_TRACEPOINT;
9320
9321 pfd = syscall(__NR_perf_event_open, &attr,
9322 pid < 0 ? -1 : pid, /* pid */
9323 pid == -1 ? 0 : -1, /* cpu */
9324 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
9325 if (pfd < 0) {
9326 err = -errno;
9327 pr_warn("legacy kprobe perf_event_open() failed: %s\n",
9328 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9329 return err;
9330 }
9331 return pfd;
9332}
9333
9209struct bpf_link *9334struct bpf_link *
9210bpf_program__attach_kprobe_opts(struct bpf_program *prog,9335bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
9211 const char *func_name,9336 const char *func_name,
9212 const struct bpf_kprobe_opts *opts)9337 const struct bpf_kprobe_opts *opts)
9213{9338{
9214 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);9339 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts);
9215 char errmsg[STRERR_BUFSIZE];9340 char errmsg[STRERR_BUFSIZE];
9341 char *legacy_probe = NULL;
9216 struct bpf_link *link;9342 struct bpf_link *link;
9217 unsigned long offset;9343 unsigned long offset;
9218 bool retprobe;9344 bool retprobe, legacy;
9219 int pfd, err;9345 int pfd, err;
9220 9346
9221 if (!OPTS_VALID(opts, bpf_kprobe_opts))9347 if (!OPTS_VALID(opts, bpf_kprobe_opts))
@@ -9225,8 +9351,19 @@ bpf_program__attach_kprobe_opts(struct bpf_program *prog, · 3 unmodified lines
9225 offset = OPTS_GET(opts, offset, 0);9351 offset = OPTS_GET(opts, offset, 0);
9226 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);9352 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0);
9227 9353
9228 pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,9354 legacy = determine_kprobe_perf_type() < 0;
9229 offset, -1 /* pid */, 0 /* ref_ctr_off */);9355 if (!legacy) {
9356 pfd = perf_event_open_probe(false /* uprobe */, retprobe,
9357 func_name, offset,
9358 -1 /* pid */, 0 /* ref_ctr_off */);
9359 } else {
9360 legacy_probe = strdup(func_name);
9361 if (!legacy_probe)
9362 return libbpf_err_ptr(-ENOMEM);
9363
9364 pfd = perf_event_kprobe_open_legacy(retprobe, func_name,
9365 offset, -1 /* pid */);
9366 }
9230 if (pfd < 0) {9367 if (pfd < 0) {
9231 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",9368 pr_warn("prog '%s': failed to create %s '%s' perf event: %s\n",
9232 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,9369 prog->name, retprobe ? "kretprobe" : "kprobe", func_name,
@@ -9242,10 +9379,17 @@ bpf_program__attach_kprobe_opts(struct bpf_program *prog, · 9 unmodified lines
9242 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));9379 libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
9243 return libbpf_err_ptr(err);9380 return libbpf_err_ptr(err);
9244 }9381 }
9382 if (legacy) {
9383 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link);
9384
9385 perf_link->legacy_probe_name = legacy_probe;
9386 perf_link->legacy_is_retprobe = retprobe;
9387 }
9388
9245 return link;9389 return link;
9246}9390}
9247 9391
9248struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog,9392struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
9249 bool retprobe,9393 bool retprobe,
9250 const char *func_name)9394 const char *func_name)
9251{9395{
@@ -9256,8 +9400,7 @@ struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, · 4 unmodified lines
9256 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);9400 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
9257}9401}
9258 9402
9259static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec,9403static struct bpf_link *attach_kprobe(const struct bpf_program *prog)
9260 struct bpf_program *prog)
9261{9404{
9262 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);9405 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts);
9263 unsigned long offset = 0;9406 unsigned long offset = 0;
@@ -9266,8 +9409,8 @@ static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, · 2 unmodified lines
9266 char *func;9409 char *func;
9267 int n, err;9410 int n, err;
9268 9411
9269 func_name = prog->sec_name + sec->len;9412 func_name = prog->sec_name + prog->sec_def->len;
9270 opts.retprobe = strcmp(sec->sec, "kretprobe/") == 0;9413 opts.retprobe = strcmp(prog->sec_def->sec, "kretprobe/") == 0;
9271 9414
9272 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);9415 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);
9273 if (n < 1) {9416 if (n < 1) {
@@ -9289,7 +9432,7 @@ static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, · 15 unmodified lines
9289}9432}
9290 9433
9291LIBBPF_API struct bpf_link *9434LIBBPF_API struct bpf_link *
9292bpf_program__attach_uprobe_opts(struct bpf_program *prog, pid_t pid,9435bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
9293 const char *binary_path, size_t func_offset,9436 const char *binary_path, size_t func_offset,
9294 const struct bpf_uprobe_opts *opts)9437 const struct bpf_uprobe_opts *opts)
9295{9438{
@@ -9329,7 +9472,7 @@ bpf_program__attach_uprobe_opts(struct bpf_program *prog, pid_t pid, · 33 unmodified lines
9329 return link;9472 return link;
9330}9473}
9331 9474
9332struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog,9475struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
9333 bool retprobe, pid_t pid,9476 bool retprobe, pid_t pid,
9334 const char *binary_path,9477 const char *binary_path,
9335 size_t func_offset)9478 size_t func_offset)
@@ -9389,7 +9532,7 @@ static int perf_event_open_tracepoint(const char *tp_category, · 53 unmodified lines
9389 return pfd;9532 return pfd;
9390}9533}
9391 9534
9392struct bpf_link *bpf_program__attach_tracepoint_opts(struct bpf_program *prog,9535struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
9393 const char *tp_category,9536 const char *tp_category,
9394 const char *tp_name,9537 const char *tp_name,
9395 const struct bpf_tracepoint_opts *opts)9538 const struct bpf_tracepoint_opts *opts)
@@ -9423,15 +9566,14 @@ struct bpf_link *bpf_program__attach_tracepoint_opts(struct bpf_program *prog, · 27 unmodified lines
9423 return link;9566 return link;
9424}9567}
9425 9568
9426struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog,9569struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
9427 const char *tp_category,9570 const char *tp_category,
9428 const char *tp_name)9571 const char *tp_name)
9429{9572{
9430 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);9573 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
9431}9574}
9432 9575
9433static struct bpf_link *attach_tp(const struct bpf_sec_def *sec,9576static struct bpf_link *attach_tp(const struct bpf_program *prog)
9434 struct bpf_program *prog)
9435{9577{
9436 char *sec_name, *tp_cat, *tp_name;9578 char *sec_name, *tp_cat, *tp_name;
9437 struct bpf_link *link;9579 struct bpf_link *link;
@@ -9441,7 +9583,7 @@ static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, · 3 unmodified lines
9441 return libbpf_err_ptr(-ENOMEM);9583 return libbpf_err_ptr(-ENOMEM);
9442 9584
9443 /* extract "tp/<category>/<name>" */9585 /* extract "tp/<category>/<name>" */
9444 tp_cat = sec_name + sec->len;9586 tp_cat = sec_name + prog->sec_def->len;
9445 tp_name = strchr(tp_cat, '/');9587 tp_name = strchr(tp_cat, '/');
9446 if (!tp_name) {9588 if (!tp_name) {
9447 free(sec_name);9589 free(sec_name);
@@ -9455,7 +9597,7 @@ static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, · 7 unmodified lines
9455 return link;9597 return link;
9456}9598}
9457 9599
9458struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog,9600struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
9459 const char *tp_name)9601 const char *tp_name)
9460{9602{
9461 char errmsg[STRERR_BUFSIZE];9603 char errmsg[STRERR_BUFSIZE];
@@ -9485,16 +9627,15 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, · 23 unmodified lines
9485 return link;9627 return link;
9486}9628}
9487 9629
9488static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec,9630static struct bpf_link *attach_raw_tp(const struct bpf_program *prog)
9489 struct bpf_program *prog)
9490{9631{
9491 const char *tp_name = prog->sec_name + sec->len;9632 const char *tp_name = prog->sec_name + prog->sec_def->len;
9492 9633
9493 return bpf_program__attach_raw_tracepoint(prog, tp_name);9634 return bpf_program__attach_raw_tracepoint(prog, tp_name);
9494}9635}
9495 9636
9496/* Common logic for all BPF program types that attach to a btf_id */9637/* Common logic for all BPF program types that attach to a btf_id */
9497static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog)9638static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog)
9498{9639{
9499 char errmsg[STRERR_BUFSIZE];9640 char errmsg[STRERR_BUFSIZE];
9500 struct bpf_link *link;9641 struct bpf_link *link;
@@ -9523,30 +9664,28 @@ static struct bpf_link *bpf_program__attach_btf_id(struct bpf_program *prog) · 22 unmodified lines
9523 return (struct bpf_link *)link;9664 return (struct bpf_link *)link;
9524}9665}
9525 9666
9526struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog)9667struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
9527{9668{
9528 return bpf_program__attach_btf_id(prog);9669 return bpf_program__attach_btf_id(prog);
9529}9670}
9530 9671
9531struct bpf_link *bpf_program__attach_lsm(struct bpf_program *prog)9672struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
9532{9673{
9533 return bpf_program__attach_btf_id(prog);9674 return bpf_program__attach_btf_id(prog);
9534}9675}
9535 9676
9536static struct bpf_link *attach_trace(const struct bpf_sec_def *sec,9677static struct bpf_link *attach_trace(const struct bpf_program *prog)
9537 struct bpf_program *prog)
9538{9678{
9539 return bpf_program__attach_trace(prog);9679 return bpf_program__attach_trace(prog);
9540}9680}
9541 9681
9542static struct bpf_link *attach_lsm(const struct bpf_sec_def *sec,9682static struct bpf_link *attach_lsm(const struct bpf_program *prog)
9543 struct bpf_program *prog)
9544{9683{
9545 return bpf_program__attach_lsm(prog);9684 return bpf_program__attach_lsm(prog);
9546}9685}
9547 9686
9548static struct bpf_link *9687static struct bpf_link *
9549bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,9688bpf_program__attach_fd(const struct bpf_program *prog, int target_fd, int btf_id,
9550 const char *target_name)9689 const char *target_name)
9551{9690{
9552 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,9691 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
@@ -9582,24 +9721,24 @@ bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id, · 29 unmodified lines
9582}9721}
9583 9722
9584struct bpf_link *9723struct bpf_link *
9585bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)9724bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
9586{9725{
9587 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");9726 return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
9588}9727}
9589 9728
9590struct bpf_link *9729struct bpf_link *
9591bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)9730bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
9592{9731{
9593 return bpf_program__attach_fd(prog, netns_fd, 0, "netns");9732 return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
9594}9733}
9595 9734
9596struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)9735struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
9597{9736{
9598 /* target_fd/target_ifindex use the same field in LINK_CREATE */9737 /* target_fd/target_ifindex use the same field in LINK_CREATE */
9599 return bpf_program__attach_fd(prog, ifindex, 0, "xdp");9738 return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
9600}9739}
9601 9740
9602struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,9741struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
9603 int target_fd,9742 int target_fd,
9604 const char *attach_func_name)9743 const char *attach_func_name)
9605{9744{
@@ -9632,7 +9771,7 @@ struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog, · 26 unmodified lines
9632}9771}
9633 9772
9634struct bpf_link *9773struct bpf_link *
9635bpf_program__attach_iter(struct bpf_program *prog,9774bpf_program__attach_iter(const struct bpf_program *prog,
9636 const struct bpf_iter_attach_opts *opts)9775 const struct bpf_iter_attach_opts *opts)
9637{9776{
9638 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);9777 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts);
@@ -9671,21 +9810,17 @@ bpf_program__attach_iter(struct bpf_program *prog, · 32 unmodified lines
9671 return link;9810 return link;
9672}9811}
9673 9812
9674static struct bpf_link *attach_iter(const struct bpf_sec_def *sec,9813static struct bpf_link *attach_iter(const struct bpf_program *prog)
9675 struct bpf_program *prog)
9676{9814{
9677 return bpf_program__attach_iter(prog, NULL);9815 return bpf_program__attach_iter(prog, NULL);
9678}9816}
9679 9817
9680struct bpf_link *bpf_program__attach(struct bpf_program *prog)9818struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
9681{9819{
9682 const struct bpf_sec_def *sec_def;9820 if (!prog->sec_def || !prog->sec_def->attach_fn)
9683
9684 sec_def = find_sec_def(prog->sec_name);
9685 if (!sec_def || !sec_def->attach_fn)
9686 return libbpf_err_ptr(-ESRCH);9821 return libbpf_err_ptr(-ESRCH);
9687 9822
9688 return sec_def->attach_fn(sec_def, prog);9823 return prog->sec_def->attach_fn(prog);
9689}9824}
9690 9825
9691static int bpf_link__detach_struct_ops(struct bpf_link *link)9826static int bpf_link__detach_struct_ops(struct bpf_link *link)
@@ -9698,7 +9833,7 @@ static int bpf_link__detach_struct_ops(struct bpf_link *link) · 6 unmodified lines
9698 return 0;9833 return 0;
9699}9834}
9700 9835
9701struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map)9836struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map)
9702{9837{
9703 struct bpf_struct_ops *st_ops;9838 struct bpf_struct_ops *st_ops;
9704 struct bpf_link *link;9839 struct bpf_link *link;
@@ -10511,18 +10646,29 @@ int bpf_program__set_attach_target(struct bpf_program *prog, · 806 unmodified lines
10511{10646{
10512 int btf_obj_fd = 0, btf_id = 0, err;10647 int btf_obj_fd = 0, btf_id = 0, err;
10513 10648
10514 if (!prog || attach_prog_fd < 0 || !attach_func_name)10649 if (!prog || attach_prog_fd < 0)
10515 return libbpf_err(-EINVAL);10650 return libbpf_err(-EINVAL);
10516 10651
10517 if (prog->obj->loaded)10652 if (prog->obj->loaded)
10518 return libbpf_err(-EINVAL);10653 return libbpf_err(-EINVAL);
10519 10654
10655 if (attach_prog_fd && !attach_func_name) {
10656 /* remember attach_prog_fd and let bpf_program__load() find
10657 * BTF ID during the program load
10658 */
10659 prog->attach_prog_fd = attach_prog_fd;
10660 return 0;
10661 }
10662
10520 if (attach_prog_fd) {10663 if (attach_prog_fd) {
10521 btf_id = libbpf_find_prog_btf_id(attach_func_name,10664 btf_id = libbpf_find_prog_btf_id(attach_func_name,
10522 attach_prog_fd);10665 attach_prog_fd);
10523 if (btf_id < 0)10666 if (btf_id < 0)
10524 return libbpf_err(btf_id);10667 return libbpf_err(btf_id);
10525 } else {10668 } else {
10669 if (!attach_func_name)
10670 return libbpf_err(-EINVAL);
10671
10526 /* load btf_vmlinux, if not yet */10672 /* load btf_vmlinux, if not yet */
10527 err = bpf_object__load_vmlinux_btf(prog->obj, true);10673 err = bpf_object__load_vmlinux_btf(prog->obj, true);
10528 if (err)10674 if (err)
@@ -10764,16 +10910,15 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) · 235 unmodified lines
10764 for (i = 0; i < s->prog_cnt; i++) {10910 for (i = 0; i < s->prog_cnt; i++) {
10765 struct bpf_program *prog = *s->progs[i].prog;10911 struct bpf_program *prog = *s->progs[i].prog;
10766 struct bpf_link **link = s->progs[i].link;10912 struct bpf_link **link = s->progs[i].link;
10767 const struct bpf_sec_def *sec_def;
10768 10913
10769 if (!prog->load)10914 if (!prog->load)
10770 continue;10915 continue;
10771 10916
10772 sec_def = find_sec_def(prog->sec_name);10917 /* auto-attaching not supported for this program */
10773 if (!sec_def || !sec_def->attach_fn)10918 if (!prog->sec_def || !prog->sec_def->attach_fn)
10774 continue;10919 continue;
10775 10920
10776 *link = sec_def->attach_fn(sec_def, prog);10921 *link = bpf_program__attach(prog);
10777 err = libbpf_get_error(*link);10922 err = libbpf_get_error(*link);
10778 if (err) {10923 if (err) {
10779 pr_warn("failed to auto-attach program '%s': %d\n",10924 pr_warn("failed to auto-attach program '%s': %d\n",
tools/lib/bpf/libbpf.h+22−19
@@ -83,12 +83,15 @@ struct bpf_object_open_opts { · 82 unmodified lines
83 * Non-relocatable instructions are replaced with invalid ones to83 * Non-relocatable instructions are replaced with invalid ones to
84 * prevent accidental errors.84 * prevent accidental errors.
85 * */85 * */
86 LIBBPF_DEPRECATED_SINCE(0, 6, "field has no effect")
86 bool relaxed_core_relocs;87 bool relaxed_core_relocs;
87 /* maps that set the 'pinning' attribute in their definition will have88 /* maps that set the 'pinning' attribute in their definition will have
88 * their pin_path attribute set to a file in this directory, and be89 * their pin_path attribute set to a file in this directory, and be
89 * auto-pinned to that path on load; defaults to "/sys/fs/bpf".90 * auto-pinned to that path on load; defaults to "/sys/fs/bpf".
90 */91 */
91 const char *pin_root_path;92 const char *pin_root_path;
93
94 LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__set_attach_target() on each individual bpf_program")
92 __u32 attach_prog_fd;95 __u32 attach_prog_fd;
93 /* Additional kernel config content that augments and overrides96 /* Additional kernel config content that augments and overrides
94 * system Kconfig for CONFIG_xxx externs.97 * system Kconfig for CONFIG_xxx externs.
@@ -243,7 +246,7 @@ LIBBPF_API int bpf_link__detach(struct bpf_link *link); · 148 unmodified lines
243LIBBPF_API int bpf_link__destroy(struct bpf_link *link);246LIBBPF_API int bpf_link__destroy(struct bpf_link *link);
244 247
245LIBBPF_API struct bpf_link *248LIBBPF_API struct bpf_link *
246bpf_program__attach(struct bpf_program *prog);249bpf_program__attach(const struct bpf_program *prog);
247 250
248struct bpf_perf_event_opts {251struct bpf_perf_event_opts {
249 /* size of this struct, for forward/backward compatiblity */252 /* size of this struct, for forward/backward compatiblity */
@@ -254,10 +257,10 @@ struct bpf_perf_event_opts { · 4 unmodified lines
254#define bpf_perf_event_opts__last_field bpf_cookie257#define bpf_perf_event_opts__last_field bpf_cookie
255 258
256LIBBPF_API struct bpf_link *259LIBBPF_API struct bpf_link *
257bpf_program__attach_perf_event(struct bpf_program *prog, int pfd);260bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd);
258 261
259LIBBPF_API struct bpf_link *262LIBBPF_API struct bpf_link *
260bpf_program__attach_perf_event_opts(struct bpf_program *prog, int pfd,263bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
261 const struct bpf_perf_event_opts *opts);264 const struct bpf_perf_event_opts *opts);
262 265
263struct bpf_kprobe_opts {266struct bpf_kprobe_opts {
@@ -274,10 +277,10 @@ struct bpf_kprobe_opts { · 10 unmodified lines
274#define bpf_kprobe_opts__last_field retprobe277#define bpf_kprobe_opts__last_field retprobe
275 278
276LIBBPF_API struct bpf_link *279LIBBPF_API struct bpf_link *
277bpf_program__attach_kprobe(struct bpf_program *prog, bool retprobe,280bpf_program__attach_kprobe(const struct bpf_program *prog, bool retprobe,
278 const char *func_name);281 const char *func_name);
279LIBBPF_API struct bpf_link *282LIBBPF_API struct bpf_link *
280bpf_program__attach_kprobe_opts(struct bpf_program *prog,283bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
281 const char *func_name,284 const char *func_name,
282 const struct bpf_kprobe_opts *opts);285 const struct bpf_kprobe_opts *opts);
283 286
@@ -297,11 +300,11 @@ struct bpf_uprobe_opts { · 13 unmodified lines
297#define bpf_uprobe_opts__last_field retprobe300#define bpf_uprobe_opts__last_field retprobe
298 301
299LIBBPF_API struct bpf_link *302LIBBPF_API struct bpf_link *
300bpf_program__attach_uprobe(struct bpf_program *prog, bool retprobe,303bpf_program__attach_uprobe(const struct bpf_program *prog, bool retprobe,
301 pid_t pid, const char *binary_path,304 pid_t pid, const char *binary_path,
302 size_t func_offset);305 size_t func_offset);
303LIBBPF_API struct bpf_link *306LIBBPF_API struct bpf_link *
304bpf_program__attach_uprobe_opts(struct bpf_program *prog, pid_t pid,307bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
305 const char *binary_path, size_t func_offset,308 const char *binary_path, size_t func_offset,
306 const struct bpf_uprobe_opts *opts);309 const struct bpf_uprobe_opts *opts);
307 310
@@ -314,35 +317,35 @@ struct bpf_tracepoint_opts { · 6 unmodified lines
314#define bpf_tracepoint_opts__last_field bpf_cookie317#define bpf_tracepoint_opts__last_field bpf_cookie
315 318
316LIBBPF_API struct bpf_link *319LIBBPF_API struct bpf_link *
317bpf_program__attach_tracepoint(struct bpf_program *prog,320bpf_program__attach_tracepoint(const struct bpf_program *prog,
318 const char *tp_category,321 const char *tp_category,
319 const char *tp_name);322 const char *tp_name);
320LIBBPF_API struct bpf_link *323LIBBPF_API struct bpf_link *
321bpf_program__attach_tracepoint_opts(struct bpf_program *prog,324bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
322 const char *tp_category,325 const char *tp_category,
323 const char *tp_name,326 const char *tp_name,
324 const struct bpf_tracepoint_opts *opts);327 const struct bpf_tracepoint_opts *opts);
325 328
326LIBBPF_API struct bpf_link *329LIBBPF_API struct bpf_link *
327bpf_program__attach_raw_tracepoint(struct bpf_program *prog,330bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
328 const char *tp_name);331 const char *tp_name);
329LIBBPF_API struct bpf_link *332LIBBPF_API struct bpf_link *
330bpf_program__attach_trace(struct bpf_program *prog);333bpf_program__attach_trace(const struct bpf_program *prog);
331LIBBPF_API struct bpf_link *334LIBBPF_API struct bpf_link *
332bpf_program__attach_lsm(struct bpf_program *prog);335bpf_program__attach_lsm(const struct bpf_program *prog);
333LIBBPF_API struct bpf_link *336LIBBPF_API struct bpf_link *
334bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd);337bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd);
335LIBBPF_API struct bpf_link *338LIBBPF_API struct bpf_link *
336bpf_program__attach_netns(struct bpf_program *prog, int netns_fd);339bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd);
337LIBBPF_API struct bpf_link *340LIBBPF_API struct bpf_link *
338bpf_program__attach_xdp(struct bpf_program *prog, int ifindex);341bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex);
339LIBBPF_API struct bpf_link *342LIBBPF_API struct bpf_link *
340bpf_program__attach_freplace(struct bpf_program *prog,343bpf_program__attach_freplace(const struct bpf_program *prog,
341 int target_fd, const char *attach_func_name);344 int target_fd, const char *attach_func_name);
342 345
343struct bpf_map;346struct bpf_map;
344 347
345LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map);348LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map);
346 349
347struct bpf_iter_attach_opts {350struct bpf_iter_attach_opts {
348 size_t sz; /* size of this struct for forward/backward compatibility */351 size_t sz; /* size of this struct for forward/backward compatibility */
@@ -352,7 +355,7 @@ struct bpf_iter_attach_opts { · 3 unmodified lines
352#define bpf_iter_attach_opts__last_field link_info_len355#define bpf_iter_attach_opts__last_field link_info_len
353 356
354LIBBPF_API struct bpf_link *357LIBBPF_API struct bpf_link *
355bpf_program__attach_iter(struct bpf_program *prog,358bpf_program__attach_iter(const struct bpf_program *prog,
356 const struct bpf_iter_attach_opts *opts);359 const struct bpf_iter_attach_opts *opts);
357 360
358struct bpf_insn;361struct bpf_insn;
@@ -854,7 +857,7 @@ struct bpf_object_skeleton { · 495 unmodified lines
854 size_t sz; /* size of this struct, for forward/backward compatibility */857 size_t sz; /* size of this struct, for forward/backward compatibility */
855 858
856 const char *name;859 const char *name;
857 void *data;860 const void *data;
858 size_t data_sz;861 size_t data_sz;
859 862
860 struct bpf_object **obj;863 struct bpf_object **obj;
tools/lib/bpf/libbpf.map+5−0
@@ -386,3 +386,8 @@ LIBBPF_0.5.0 { · 385 unmodified lines
386 btf_dump__dump_type_data;386 btf_dump__dump_type_data;
387 libbpf_set_strict_mode;387 libbpf_set_strict_mode;
388} LIBBPF_0.4.0;388} LIBBPF_0.4.0;
389
390LIBBPF_0.6.0 {
391 global:
392 btf__add_tag;
393} LIBBPF_0.5.0;
tools/lib/bpf/libbpf_common.h+24−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
10#define __LIBBPF_LIBBPF_COMMON_H10#define __LIBBPF_LIBBPF_COMMON_H
11 11
12#include <string.h>12#include <string.h>
13#include "libbpf_version.h"
13 14
14#ifndef LIBBPF_API15#ifndef LIBBPF_API
15#define LIBBPF_API __attribute__((visibility("default")))16#define LIBBPF_API __attribute__((visibility("default")))
@@ -17,6 +18,29 @@ · 1 unmodified line
17 18
18#define LIBBPF_DEPRECATED(msg) __attribute__((deprecated(msg)))19#define LIBBPF_DEPRECATED(msg) __attribute__((deprecated(msg)))
19 20
21/* Mark a symbol as deprecated when libbpf version is >= {major}.{minor} */
22#define LIBBPF_DEPRECATED_SINCE(major, minor, msg) \
23 __LIBBPF_MARK_DEPRECATED_ ## major ## _ ## minor \
24 (LIBBPF_DEPRECATED("libbpf v" # major "." # minor "+: " msg))
25
26#define __LIBBPF_CURRENT_VERSION_GEQ(major, minor) \
27 (LIBBPF_MAJOR_VERSION > (major) || \
28 (LIBBPF_MAJOR_VERSION == (major) && LIBBPF_MINOR_VERSION >= (minor)))
29
30/* Add checks for other versions below when planning deprecation of API symbols
31 * with the LIBBPF_DEPRECATED_SINCE macro.
32 */
33#if __LIBBPF_CURRENT_VERSION_GEQ(0, 6)
34#define __LIBBPF_MARK_DEPRECATED_0_6(X) X
35#else
36#define __LIBBPF_MARK_DEPRECATED_0_6(X)
37#endif
38#if __LIBBPF_CURRENT_VERSION_GEQ(0, 7)
39#define __LIBBPF_MARK_DEPRECATED_0_7(X) X
40#else
41#define __LIBBPF_MARK_DEPRECATED_0_7(X)
42#endif
43
20/* Helper macro to declare and initialize libbpf options struct44/* Helper macro to declare and initialize libbpf options struct
21 *45 *
22 * This dance with uninitialized declaration, followed by memset to zero,46 * This dance with uninitialized declaration, followed by memset to zero,
tools/lib/bpf/libbpf_internal.h+21−6
@@ -69,6 +69,8 @@ · 68 unmodified lines
69#define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size)69#define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size)
70#define BTF_TYPE_FLOAT_ENC(name, sz) \70#define BTF_TYPE_FLOAT_ENC(name, sz) \
71 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)71 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)
72#define BTF_TYPE_TAG_ENC(value, type, component_idx) \
73 BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TAG, 0, 0), type), (component_idx)
72 74
73#ifndef likely75#ifndef likely
74#define likely(x) __builtin_expect(!!(x), 1)76#define likely(x) __builtin_expect(!!(x), 1)
@@ -90,17 +92,30 @@ · 15 unmodified lines
90/* Symbol versioning is different between static and shared library.92/* Symbol versioning is different between static and shared library.
91 * Properly versioned symbols are needed for shared library, but93 * Properly versioned symbols are needed for shared library, but
92 * only the symbol of the new version is needed for static library.94 * only the symbol of the new version is needed for static library.
95 * Starting with GNU C 10, use symver attribute instead of .symver assembler
96 * directive, which works better with GCC LTO builds.
93 */97 */
94#ifdef SHARED98#if defined(SHARED) && defined(__GNUC__) && __GNUC__ >= 10
95# define COMPAT_VERSION(internal_name, api_name, version) \99
100#define DEFAULT_VERSION(internal_name, api_name, version) \
101 __attribute__((symver(#api_name "@@" #version)))
102#define COMPAT_VERSION(internal_name, api_name, version) \
103 __attribute__((symver(#api_name "@" #version)))
104
105#elif defined(SHARED)
106
107#define COMPAT_VERSION(internal_name, api_name, version) \
96 asm(".symver " #internal_name "," #api_name "@" #version);108 asm(".symver " #internal_name "," #api_name "@" #version);
97# define DEFAULT_VERSION(internal_name, api_name, version) \109#define DEFAULT_VERSION(internal_name, api_name, version) \
98 asm(".symver " #internal_name "," #api_name "@@" #version);110 asm(".symver " #internal_name "," #api_name "@@" #version);
99#else111
100# define COMPAT_VERSION(internal_name, api_name, version)112#else /* !SHARED */
101# define DEFAULT_VERSION(internal_name, api_name, version) \113
114#define COMPAT_VERSION(internal_name, api_name, version)
115#define DEFAULT_VERSION(internal_name, api_name, version) \
102 extern typeof(internal_name) api_name \116 extern typeof(internal_name) api_name \
103 __attribute__((alias(#internal_name)));117 __attribute__((alias(#internal_name)));
118
104#endif119#endif
105 120
106extern void libbpf_print(enum libbpf_print_level level,121extern void libbpf_print(enum libbpf_print_level level,
tools/lib/bpf/libbpf_version.h+9−0
@@ -0,0 +1,9 @@
1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2/* Copyright (C) 2021 Facebook */
3#ifndef __LIBBPF_VERSION_H
4#define __LIBBPF_VERSION_H
5
6#define LIBBPF_MAJOR_VERSION 0
7#define LIBBPF_MINOR_VERSION 6
8
9#endif /* __LIBBPF_VERSION_H */
tools/lib/bpf/xsk.c+2−2
@@ -281,6 +281,7 @@ out_mmap: · 280 unmodified lines
281 return err;281 return err;
282}282}
283 283
284DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
284int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,285int xsk_umem__create_v0_0_4(struct xsk_umem **umem_ptr, void *umem_area,
285 __u64 size, struct xsk_ring_prod *fill,286 __u64 size, struct xsk_ring_prod *fill,
286 struct xsk_ring_cons *comp,287 struct xsk_ring_cons *comp,
@@ -345,6 +346,7 @@ struct xsk_umem_config_v1 { · 58 unmodified lines
345 __u32 frame_headroom;346 __u32 frame_headroom;
346};347};
347 348
349COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
348int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area,350int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area,
349 __u64 size, struct xsk_ring_prod *fill,351 __u64 size, struct xsk_ring_prod *fill,
350 struct xsk_ring_cons *comp,352 struct xsk_ring_cons *comp,
@@ -358,8 +360,6 @@ int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, · 7 unmodified lines
358 return xsk_umem__create_v0_0_4(umem_ptr, umem_area, size, fill, comp,360 return xsk_umem__create_v0_0_4(umem_ptr, umem_area, size, fill, comp,
359 &config);361 &config);
360}362}
361COMPAT_VERSION(xsk_umem__create_v0_0_2, xsk_umem__create, LIBBPF_0.0.2)
362DEFAULT_VERSION(xsk_umem__create_v0_0_4, xsk_umem__create, LIBBPF_0.0.4)
363 363
364static enum xsk_prog get_xsk_prog(void)364static enum xsk_prog get_xsk_prog(void)
365{365{
tools/testing/selftests/bpf/.gitignore+3−2
@@ -9,8 +9,9 @@ test_tag · 8 unmodified lines
9FEATURE-DUMP.libbpf9FEATURE-DUMP.libbpf
10fixdep10fixdep
11test_dev_cgroup11test_dev_cgroup
12/test_progs*12/test_progs
13!test_progs.h13/test_progs-no_alu32
14/test_progs-bpf_gcc
14test_verifier_log15test_verifier_log
15feature16feature
16test_sock17test_sock
tools/testing/selftests/bpf/Makefile+2−2
@@ -512,14 +512,14 @@ $(OUTPUT)/test_cpp: test_cpp.cpp $(OUTPUT)/test_core_extern.skel.h $(BPFOBJ) · 511 unmodified lines
512 $(Q)$(CXX) $(CFLAGS) $(filter %.a %.o %.cpp,$^) $(LDLIBS) -o $@512 $(Q)$(CXX) $(CFLAGS) $(filter %.a %.o %.cpp,$^) $(LDLIBS) -o $@
513 513
514# Benchmark runner514# Benchmark runner
515$(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h515$(OUTPUT)/bench_%.o: benchs/bench_%.c bench.h $(BPFOBJ)
516 $(call msg,CC,,$@)516 $(call msg,CC,,$@)
517 $(Q)$(CC) $(CFLAGS) -c $(filter %.c,$^) $(LDLIBS) -o $@517 $(Q)$(CC) $(CFLAGS) -c $(filter %.c,$^) $(LDLIBS) -o $@
518$(OUTPUT)/bench_rename.o: $(OUTPUT)/test_overhead.skel.h518$(OUTPUT)/bench_rename.o: $(OUTPUT)/test_overhead.skel.h
519$(OUTPUT)/bench_trigger.o: $(OUTPUT)/trigger_bench.skel.h519$(OUTPUT)/bench_trigger.o: $(OUTPUT)/trigger_bench.skel.h
520$(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \520$(OUTPUT)/bench_ringbufs.o: $(OUTPUT)/ringbuf_bench.skel.h \
521 $(OUTPUT)/perfbuf_bench.skel.h521 $(OUTPUT)/perfbuf_bench.skel.h
522$(OUTPUT)/bench.o: bench.h testing_helpers.h522$(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ)
523$(OUTPUT)/bench: LDLIBS += -lm523$(OUTPUT)/bench: LDLIBS += -lm
524$(OUTPUT)/bench: $(OUTPUT)/bench.o $(OUTPUT)/testing_helpers.o \524$(OUTPUT)/bench: $(OUTPUT)/bench.o $(OUTPUT)/testing_helpers.o \
525 $(OUTPUT)/bench_count.o \525 $(OUTPUT)/bench_count.o \
tools/testing/selftests/bpf/README.rst+14−0
@@ -201,6 +201,20 @@ Without it, the error from compiling bpf selftests looks like: · 200 unmodified lines
201 201
202__ https://reviews.llvm.org/D93563202__ https://reviews.llvm.org/D93563
203 203
204btf_tag test and Clang version
205==============================
206
207The btf_tag selftest require LLVM support to recognize the btf_tag attribute.
208It was introduced in `Clang 14`__.
209
210Without it, the btf_tag selftest will be skipped and you will observe:
211
212.. code-block:: console
213
214 #<test_num> btf_tag:SKIP
215
216__ https://reviews.llvm.org/D106614
217
204Clang dependencies for static linking tests218Clang dependencies for static linking tests
205===========================================219===========================================
206 220
tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c+17−2
@@ -13,6 +13,18 @@ · 12 unmodified lines
13 13
14DEFINE_PER_CPU(int, bpf_testmod_ksym_percpu) = 123;14DEFINE_PER_CPU(int, bpf_testmod_ksym_percpu) = 123;
15 15
16noinline int bpf_testmod_loop_test(int n)
17{
18 int i, sum = 0;
19
20 /* the primary goal of this test is to test LBR. Create a lot of
21 * branches in the function, so we can catch it easily.
22 */
23 for (i = 0; i < n; i++)
24 sum += i;
25 return sum;
26}
27
16noinline ssize_t28noinline ssize_t
17bpf_testmod_test_read(struct file *file, struct kobject *kobj,29bpf_testmod_test_read(struct file *file, struct kobject *kobj,
18 struct bin_attribute *bin_attr,30 struct bin_attribute *bin_attr,
@@ -24,7 +36,11 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj, · 5 unmodified lines
24 .len = len,36 .len = len,
25 };37 };
26 38
27 trace_bpf_testmod_test_read(current, &ctx);39 /* This is always true. Use the check to make sure the compiler
40 * doesn't remove bpf_testmod_loop_test.
41 */
42 if (bpf_testmod_loop_test(101) > 100)
43 trace_bpf_testmod_test_read(current, &ctx);
28 44
29 return -EIO; /* always fail */45 return -EIO; /* always fail */
30}46}
@@ -71,4 +87,3 @@ module_exit(bpf_testmod_exit); · 40 unmodified lines
71MODULE_AUTHOR("Andrii Nakryiko");87MODULE_AUTHOR("Andrii Nakryiko");
72MODULE_DESCRIPTION("BPF selftests module");88MODULE_DESCRIPTION("BPF selftests module");
73MODULE_LICENSE("Dual BSD/GPL");89MODULE_LICENSE("Dual BSD/GPL");
74
tools/testing/selftests/bpf/btf_helpers.c+6−1
@@ -24,11 +24,12 @@ static const char * const btf_kind_str_mapping[] = { · 23 unmodified lines
24 [BTF_KIND_VAR] = "VAR",24 [BTF_KIND_VAR] = "VAR",
25 [BTF_KIND_DATASEC] = "DATASEC",25 [BTF_KIND_DATASEC] = "DATASEC",
26 [BTF_KIND_FLOAT] = "FLOAT",26 [BTF_KIND_FLOAT] = "FLOAT",
27 [BTF_KIND_TAG] = "TAG",
27};28};
28 29
29static const char *btf_kind_str(__u16 kind)30static const char *btf_kind_str(__u16 kind)
30{31{
31 if (kind > BTF_KIND_DATASEC)32 if (kind > BTF_KIND_TAG)
32 return "UNKNOWN";33 return "UNKNOWN";
33 return btf_kind_str_mapping[kind];34 return btf_kind_str_mapping[kind];
34}35}
@@ -177,6 +178,10 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) · 142 unmodified lines
177 case BTF_KIND_FLOAT:178 case BTF_KIND_FLOAT:
178 fprintf(out, " size=%u", t->size);179 fprintf(out, " size=%u", t->size);
179 break;180 break;
181 case BTF_KIND_TAG:
182 fprintf(out, " type_id=%u component_idx=%d",
183 t->type, btf_tag(t)->component_idx);
184 break;
180 default:185 default:
181 break;186 break;
182 }187 }
tools/testing/selftests/bpf/prog_tests/bpf_iter.c+2−4
@@ -589,7 +589,7 @@ out: · 588 unmodified lines
589 589
590static void test_bpf_hash_map(void)590static void test_bpf_hash_map(void)
591{591{
592 __u32 expected_key_a = 0, expected_key_b = 0, expected_key_c = 0;592 __u32 expected_key_a = 0, expected_key_b = 0;
593 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);593 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
594 struct bpf_iter_bpf_hash_map *skel;594 struct bpf_iter_bpf_hash_map *skel;
595 int err, i, len, map_fd, iter_fd;595 int err, i, len, map_fd, iter_fd;
@@ -638,7 +638,6 @@ static void test_bpf_hash_map(void) · 42 unmodified lines
638 val = i + 4;638 val = i + 4;
639 expected_key_a += key.a;639 expected_key_a += key.a;
640 expected_key_b += key.b;640 expected_key_b += key.b;
641 expected_key_c += key.c;
642 expected_val += val;641 expected_val += val;
643 642
644 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);643 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);
@@ -685,7 +684,7 @@ out: · 40 unmodified lines
685 684
686static void test_bpf_percpu_hash_map(void)685static void test_bpf_percpu_hash_map(void)
687{686{
688 __u32 expected_key_a = 0, expected_key_b = 0, expected_key_c = 0;687 __u32 expected_key_a = 0, expected_key_b = 0;
689 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);688 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
690 struct bpf_iter_bpf_percpu_hash_map *skel;689 struct bpf_iter_bpf_percpu_hash_map *skel;
691 int err, i, j, len, map_fd, iter_fd;690 int err, i, j, len, map_fd, iter_fd;
@@ -722,7 +721,6 @@ static void test_bpf_percpu_hash_map(void) · 30 unmodified lines
722 key.c = i + 3;721 key.c = i + 3;
723 expected_key_a += key.a;722 expected_key_a += key.a;
724 expected_key_b += key.b;723 expected_key_b += key.b;
725 expected_key_c += key.c;
726 724
727 for (j = 0; j < bpf_num_possible_cpus(); j++) {725 for (j = 0; j < bpf_num_possible_cpus(); j++) {
728 *(__u32 *)(val + j * 8) = i + j;726 *(__u32 *)(val + j * 8) = i + j;
tools/testing/selftests/bpf/prog_tests/btf.c+422−19
@@ -39,8 +39,8 @@ static bool always_log; · 38 unmodified lines
39#define BTF_END_RAW 0xdeadbeef39#define BTF_END_RAW 0xdeadbeef
40#define NAME_TBD 0xdeadb33f40#define NAME_TBD 0xdeadb33f
41 41
42#define NAME_NTH(N) (0xffff0000 | N)42#define NAME_NTH(N) (0xfffe0000 | N)
43#define IS_NAME_NTH(X) ((X & 0xffff0000) == 0xffff0000)43#define IS_NAME_NTH(X) ((X & 0xffff0000) == 0xfffe0000)
44#define GET_NAME_NTH_IDX(X) (X & 0x0000ffff)44#define GET_NAME_NTH_IDX(X) (X & 0x0000ffff)
45 45
46#define MAX_NR_RAW_U32 102446#define MAX_NR_RAW_U32 1024
@@ -3661,6 +3661,249 @@ static struct btf_raw_test raw_tests[] = { · 3614 unmodified lines
3661 .err_str = "Invalid type_size",3661 .err_str = "Invalid type_size",
3662},3662},
3663 3663
3664{
3665 .descr = "tag test #1, struct/member, well-formed",
3666 .raw_types = {
3667 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3668 BTF_STRUCT_ENC(0, 2, 8), /* [2] */
3669 BTF_MEMBER_ENC(NAME_TBD, 1, 0),
3670 BTF_MEMBER_ENC(NAME_TBD, 1, 32),
3671 BTF_TAG_ENC(NAME_TBD, 2, -1),
3672 BTF_TAG_ENC(NAME_TBD, 2, 0),
3673 BTF_TAG_ENC(NAME_TBD, 2, 1),
3674 BTF_END_RAW,
3675 },
3676 BTF_STR_SEC("\0m1\0m2\0tag1\0tag2\0tag3"),
3677 .map_type = BPF_MAP_TYPE_ARRAY,
3678 .map_name = "tag_type_check_btf",
3679 .key_size = sizeof(int),
3680 .value_size = 8,
3681 .key_type_id = 1,
3682 .value_type_id = 2,
3683 .max_entries = 1,
3684},
3685{
3686 .descr = "tag test #2, union/member, well-formed",
3687 .raw_types = {
3688 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3689 BTF_UNION_ENC(NAME_TBD, 2, 4), /* [2] */
3690 BTF_MEMBER_ENC(NAME_TBD, 1, 0),
3691 BTF_MEMBER_ENC(NAME_TBD, 1, 0),
3692 BTF_TAG_ENC(NAME_TBD, 2, -1),
3693 BTF_TAG_ENC(NAME_TBD, 2, 0),
3694 BTF_TAG_ENC(NAME_TBD, 2, 1),
3695 BTF_END_RAW,
3696 },
3697 BTF_STR_SEC("\0t\0m1\0m2\0tag1\0tag2\0tag3"),
3698 .map_type = BPF_MAP_TYPE_ARRAY,
3699 .map_name = "tag_type_check_btf",
3700 .key_size = sizeof(int),
3701 .value_size = 4,
3702 .key_type_id = 1,
3703 .value_type_id = 2,
3704 .max_entries = 1,
3705},
3706{
3707 .descr = "tag test #3, variable, well-formed",
3708 .raw_types = {
3709 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3710 BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */
3711 BTF_VAR_ENC(NAME_TBD, 1, 1), /* [3] */
3712 BTF_TAG_ENC(NAME_TBD, 2, -1),
3713 BTF_TAG_ENC(NAME_TBD, 3, -1),
3714 BTF_END_RAW,
3715 },
3716 BTF_STR_SEC("\0local\0global\0tag1\0tag2"),
3717 .map_type = BPF_MAP_TYPE_ARRAY,
3718 .map_name = "tag_type_check_btf",
3719 .key_size = sizeof(int),
3720 .value_size = 4,
3721 .key_type_id = 1,
3722 .value_type_id = 1,
3723 .max_entries = 1,
3724},
3725{
3726 .descr = "tag test #4, func/parameter, well-formed",
3727 .raw_types = {
3728 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3729 BTF_FUNC_PROTO_ENC(0, 2), /* [2] */
3730 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
3731 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
3732 BTF_FUNC_ENC(NAME_TBD, 2), /* [3] */
3733 BTF_TAG_ENC(NAME_TBD, 3, -1),
3734 BTF_TAG_ENC(NAME_TBD, 3, 0),
3735 BTF_TAG_ENC(NAME_TBD, 3, 1),
3736 BTF_END_RAW,
3737 },
3738 BTF_STR_SEC("\0arg1\0arg2\0f\0tag1\0tag2\0tag3"),
3739 .map_type = BPF_MAP_TYPE_ARRAY,
3740 .map_name = "tag_type_check_btf",
3741 .key_size = sizeof(int),
3742 .value_size = 4,
3743 .key_type_id = 1,
3744 .value_type_id = 1,
3745 .max_entries = 1,
3746},
3747{
3748 .descr = "tag test #5, invalid value",
3749 .raw_types = {
3750 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3751 BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */
3752 BTF_TAG_ENC(0, 2, -1),
3753 BTF_END_RAW,
3754 },
3755 BTF_STR_SEC("\0local\0tag"),
3756 .map_type = BPF_MAP_TYPE_ARRAY,
3757 .map_name = "tag_type_check_btf",
3758 .key_size = sizeof(int),
3759 .value_size = 4,
3760 .key_type_id = 1,
3761 .value_type_id = 1,
3762 .max_entries = 1,
3763 .btf_load_err = true,
3764 .err_str = "Invalid value",
3765},
3766{
3767 .descr = "tag test #6, invalid target type",
3768 .raw_types = {
3769 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3770 BTF_TAG_ENC(NAME_TBD, 1, -1),
3771 BTF_END_RAW,
3772 },
3773 BTF_STR_SEC("\0tag1"),
3774 .map_type = BPF_MAP_TYPE_ARRAY,
3775 .map_name = "tag_type_check_btf",
3776 .key_size = sizeof(int),
3777 .value_size = 4,
3778 .key_type_id = 1,
3779 .value_type_id = 1,
3780 .max_entries = 1,
3781 .btf_load_err = true,
3782 .err_str = "Invalid type",
3783},
3784{
3785 .descr = "tag test #7, invalid vlen",
3786 .raw_types = {
3787 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3788 BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */
3789 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_TAG, 0, 1), 2), (0),
3790 BTF_END_RAW,
3791 },
3792 BTF_STR_SEC("\0local\0tag1"),
3793 .map_type = BPF_MAP_TYPE_ARRAY,
3794 .map_name = "tag_type_check_btf",
3795 .key_size = sizeof(int),
3796 .value_size = 4,
3797 .key_type_id = 1,
3798 .value_type_id = 1,
3799 .max_entries = 1,
3800 .btf_load_err = true,
3801 .err_str = "vlen != 0",
3802},
3803{
3804 .descr = "tag test #8, invalid kflag",
3805 .raw_types = {
3806 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3807 BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */
3808 BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_TAG, 1, 0), 2), (-1),
3809 BTF_END_RAW,
3810 },
3811 BTF_STR_SEC("\0local\0tag1"),
3812 .map_type = BPF_MAP_TYPE_ARRAY,
3813 .map_name = "tag_type_check_btf",
3814 .key_size = sizeof(int),
3815 .value_size = 4,
3816 .key_type_id = 1,
3817 .value_type_id = 1,
3818 .max_entries = 1,
3819 .btf_load_err = true,
3820 .err_str = "Invalid btf_info kind_flag",
3821},
3822{
3823 .descr = "tag test #9, var, invalid component_idx",
3824 .raw_types = {
3825 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3826 BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */
3827 BTF_TAG_ENC(NAME_TBD, 2, 0),
3828 BTF_END_RAW,
3829 },
3830 BTF_STR_SEC("\0local\0tag"),
3831 .map_type = BPF_MAP_TYPE_ARRAY,
3832 .map_name = "tag_type_check_btf",
3833 .key_size = sizeof(int),
3834 .value_size = 4,
3835 .key_type_id = 1,
3836 .value_type_id = 1,
3837 .max_entries = 1,
3838 .btf_load_err = true,
3839 .err_str = "Invalid component_idx",
3840},
3841{
3842 .descr = "tag test #10, struct member, invalid component_idx",
3843 .raw_types = {
3844 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3845 BTF_STRUCT_ENC(0, 2, 8), /* [2] */
3846 BTF_MEMBER_ENC(NAME_TBD, 1, 0),
3847 BTF_MEMBER_ENC(NAME_TBD, 1, 32),
3848 BTF_TAG_ENC(NAME_TBD, 2, 2),
3849 BTF_END_RAW,
3850 },
3851 BTF_STR_SEC("\0m1\0m2\0tag"),
3852 .map_type = BPF_MAP_TYPE_ARRAY,
3853 .map_name = "tag_type_check_btf",
3854 .key_size = sizeof(int),
3855 .value_size = 8,
3856 .key_type_id = 1,
3857 .value_type_id = 2,
3858 .max_entries = 1,
3859 .btf_load_err = true,
3860 .err_str = "Invalid component_idx",
3861},
3862{
3863 .descr = "tag test #11, func parameter, invalid component_idx",
3864 .raw_types = {
3865 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3866 BTF_FUNC_PROTO_ENC(0, 2), /* [2] */
3867 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
3868 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
3869 BTF_FUNC_ENC(NAME_TBD, 2), /* [3] */
3870 BTF_TAG_ENC(NAME_TBD, 3, 2),
3871 BTF_END_RAW,
3872 },
3873 BTF_STR_SEC("\0arg1\0arg2\0f\0tag"),
3874 .map_type = BPF_MAP_TYPE_ARRAY,
3875 .map_name = "tag_type_check_btf",
3876 .key_size = sizeof(int),
3877 .value_size = 4,
3878 .key_type_id = 1,
3879 .value_type_id = 1,
3880 .max_entries = 1,
3881 .btf_load_err = true,
3882 .err_str = "Invalid component_idx",
3883},
3884{
3885 .descr = "tag test #12, < -1 component_idx",
3886 .raw_types = {
3887 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
3888 BTF_FUNC_PROTO_ENC(0, 2), /* [2] */
3889 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
3890 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1),
3891 BTF_FUNC_ENC(NAME_TBD, 2), /* [3] */
3892 BTF_TAG_ENC(NAME_TBD, 3, -2),
3893 BTF_END_RAW,
3894 },
3895 BTF_STR_SEC("\0arg1\0arg2\0f\0tag"),
3896 .map_type = BPF_MAP_TYPE_ARRAY,
3897 .map_name = "tag_type_check_btf",
3898 .key_size = sizeof(int),
3899 .value_size = 4,
3900 .key_type_id = 1,
3901 .value_type_id = 1,
3902 .max_entries = 1,
3903 .btf_load_err = true,
3904 .err_str = "Invalid component_idx",
3905},
3906
3664}; /* struct btf_raw_test raw_tests[] */3907}; /* struct btf_raw_test raw_tests[] */
3665 3908
3666static const char *get_next_str(const char *start, const char *end)3909static const char *get_next_str(const char *start, const char *end)
@@ -6421,27 +6664,33 @@ const struct btf_dedup_test dedup_tests[] = { · 2754 unmodified lines
6421 BTF_MEMBER_ENC(NAME_NTH(4), 5, 64), /* const int *a; */6664 BTF_MEMBER_ENC(NAME_NTH(4), 5, 64), /* const int *a; */
6422 BTF_MEMBER_ENC(NAME_NTH(5), 2, 128), /* int b[16]; */6665 BTF_MEMBER_ENC(NAME_NTH(5), 2, 128), /* int b[16]; */
6423 BTF_MEMBER_ENC(NAME_NTH(6), 1, 640), /* int c; */6666 BTF_MEMBER_ENC(NAME_NTH(6), 1, 640), /* int c; */
6424 BTF_MEMBER_ENC(NAME_NTH(8), 13, 672), /* float d; */6667 BTF_MEMBER_ENC(NAME_NTH(8), 15, 672), /* float d; */
6425 /* ptr -> [3] struct s */6668 /* ptr -> [3] struct s */
6426 BTF_PTR_ENC(3), /* [4] */6669 BTF_PTR_ENC(3), /* [4] */
6427 /* ptr -> [6] const int */6670 /* ptr -> [6] const int */
6428 BTF_PTR_ENC(6), /* [5] */6671 BTF_PTR_ENC(6), /* [5] */
6429 /* const -> [1] int */6672 /* const -> [1] int */
6430 BTF_CONST_ENC(1), /* [6] */6673 BTF_CONST_ENC(1), /* [6] */
6674 /* tag -> [3] struct s */
6675 BTF_TAG_ENC(NAME_NTH(2), 3, -1), /* [7] */
6676 /* tag -> [3] struct s, member 1 */
6677 BTF_TAG_ENC(NAME_NTH(2), 3, 1), /* [8] */
6431 6678
6432 /* full copy of the above */6679 /* full copy of the above */
6433 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [7] */6680 BTF_TYPE_INT_ENC(NAME_NTH(1), BTF_INT_SIGNED, 0, 32, 4), /* [9] */
6434 BTF_TYPE_ARRAY_ENC(7, 7, 16), /* [8] */6681 BTF_TYPE_ARRAY_ENC(9, 9, 16), /* [10] */
6435 BTF_STRUCT_ENC(NAME_NTH(2), 5, 88), /* [9] */6682 BTF_STRUCT_ENC(NAME_NTH(2), 5, 88), /* [11] */
6436 BTF_MEMBER_ENC(NAME_NTH(3), 10, 0),6683 BTF_MEMBER_ENC(NAME_NTH(3), 12, 0),
6437 BTF_MEMBER_ENC(NAME_NTH(4), 11, 64),6684 BTF_MEMBER_ENC(NAME_NTH(4), 13, 64),
6438 BTF_MEMBER_ENC(NAME_NTH(5), 8, 128),6685 BTF_MEMBER_ENC(NAME_NTH(5), 10, 128),
6439 BTF_MEMBER_ENC(NAME_NTH(6), 7, 640),6686 BTF_MEMBER_ENC(NAME_NTH(6), 9, 640),
6440 BTF_MEMBER_ENC(NAME_NTH(8), 13, 672),6687 BTF_MEMBER_ENC(NAME_NTH(8), 15, 672),
6441 BTF_PTR_ENC(9), /* [10] */6688 BTF_PTR_ENC(11), /* [12] */
6442 BTF_PTR_ENC(12), /* [11] */6689 BTF_PTR_ENC(14), /* [13] */
6443 BTF_CONST_ENC(7), /* [12] */6690 BTF_CONST_ENC(9), /* [14] */
6444 BTF_TYPE_FLOAT_ENC(NAME_NTH(7), 4), /* [13] */6691 BTF_TYPE_FLOAT_ENC(NAME_NTH(7), 4), /* [15] */
6692 BTF_TAG_ENC(NAME_NTH(2), 11, -1), /* [16] */
6693 BTF_TAG_ENC(NAME_NTH(2), 11, 1), /* [17] */
6445 BTF_END_RAW,6694 BTF_END_RAW,
6446 },6695 },
6447 BTF_STR_SEC("\0int\0s\0next\0a\0b\0c\0float\0d"),6696 BTF_STR_SEC("\0int\0s\0next\0a\0b\0c\0float\0d"),
@@ -6458,14 +6707,16 @@ const struct btf_dedup_test dedup_tests[] = { · 10 unmodified lines
6458 BTF_MEMBER_ENC(NAME_NTH(1), 5, 64), /* const int *a; */6707 BTF_MEMBER_ENC(NAME_NTH(1), 5, 64), /* const int *a; */
6459 BTF_MEMBER_ENC(NAME_NTH(2), 2, 128), /* int b[16]; */6708 BTF_MEMBER_ENC(NAME_NTH(2), 2, 128), /* int b[16]; */
6460 BTF_MEMBER_ENC(NAME_NTH(3), 1, 640), /* int c; */6709 BTF_MEMBER_ENC(NAME_NTH(3), 1, 640), /* int c; */
6461 BTF_MEMBER_ENC(NAME_NTH(4), 7, 672), /* float d; */6710 BTF_MEMBER_ENC(NAME_NTH(4), 9, 672), /* float d; */
6462 /* ptr -> [3] struct s */6711 /* ptr -> [3] struct s */
6463 BTF_PTR_ENC(3), /* [4] */6712 BTF_PTR_ENC(3), /* [4] */
6464 /* ptr -> [6] const int */6713 /* ptr -> [6] const int */
6465 BTF_PTR_ENC(6), /* [5] */6714 BTF_PTR_ENC(6), /* [5] */
6466 /* const -> [1] int */6715 /* const -> [1] int */
6467 BTF_CONST_ENC(1), /* [6] */6716 BTF_CONST_ENC(1), /* [6] */
6468 BTF_TYPE_FLOAT_ENC(NAME_NTH(7), 4), /* [7] */6717 BTF_TAG_ENC(NAME_NTH(2), 3, -1), /* [7] */
6718 BTF_TAG_ENC(NAME_NTH(2), 3, 1), /* [8] */
6719 BTF_TYPE_FLOAT_ENC(NAME_NTH(7), 4), /* [9] */
6469 BTF_END_RAW,6720 BTF_END_RAW,
6470 },6721 },
6471 BTF_STR_SEC("\0a\0b\0c\0d\0int\0float\0next\0s"),6722 BTF_STR_SEC("\0a\0b\0c\0d\0int\0float\0next\0s"),
@@ -6590,9 +6841,11 @@ const struct btf_dedup_test dedup_tests[] = { · 118 unmodified lines
6590 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8),6841 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8),
6591 BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */6842 BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */
6592 BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */6843 BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */
6844 BTF_TAG_ENC(NAME_TBD, 13, -1), /* [15] tag */
6845 BTF_TAG_ENC(NAME_TBD, 13, 1), /* [16] tag */
6593 BTF_END_RAW,6846 BTF_END_RAW,
6594 },6847 },
6595 BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N"),6848 BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P"),
6596 },6849 },
6597 .expect = {6850 .expect = {
6598 .raw_types = {6851 .raw_types = {
@@ -6616,9 +6869,11 @@ const struct btf_dedup_test dedup_tests[] = { · 17 unmodified lines
6616 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8),6869 BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8),
6617 BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */6870 BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */
6618 BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */6871 BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */
6872 BTF_TAG_ENC(NAME_TBD, 13, -1), /* [15] tag */
6873 BTF_TAG_ENC(NAME_TBD, 13, 1), /* [16] tag */
6619 BTF_END_RAW,6874 BTF_END_RAW,
6620 },6875 },
6621 BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N"),6876 BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P"),
6622 },6877 },
6623 .opts = {6878 .opts = {
6624 .dont_resolve_fwds = false,6879 .dont_resolve_fwds = false,
@@ -6767,6 +7022,152 @@ const struct btf_dedup_test dedup_tests[] = { · 142 unmodified lines
6767 .dedup_table_size = 17022 .dedup_table_size = 1
6768 },7023 },
6769},7024},
7025{
7026 .descr = "dedup: func/func_arg/var tags",
7027 .input = {
7028 .raw_types = {
7029 /* int */
7030 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
7031 /* static int t */
7032 BTF_VAR_ENC(NAME_NTH(1), 1, 0), /* [2] */
7033 /* void f(int a1, int a2) */
7034 BTF_FUNC_PROTO_ENC(0, 2), /* [3] */
7035 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(2), 1),
7036 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(3), 1),
7037 BTF_FUNC_ENC(NAME_NTH(4), 2), /* [4] */
7038 /* tag -> t */
7039 BTF_TAG_ENC(NAME_NTH(5), 2, -1), /* [5] */
7040 BTF_TAG_ENC(NAME_NTH(5), 2, -1), /* [6] */
7041 /* tag -> func */
7042 BTF_TAG_ENC(NAME_NTH(5), 4, -1), /* [7] */
7043 BTF_TAG_ENC(NAME_NTH(5), 4, -1), /* [8] */
7044 /* tag -> func arg a1 */
7045 BTF_TAG_ENC(NAME_NTH(5), 4, 1), /* [9] */
7046 BTF_TAG_ENC(NAME_NTH(5), 4, 1), /* [10] */
7047 BTF_END_RAW,
7048 },
7049 BTF_STR_SEC("\0t\0a1\0a2\0f\0tag"),
7050 },
7051 .expect = {
7052 .raw_types = {
7053 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
7054 BTF_VAR_ENC(NAME_NTH(1), 1, 0), /* [2] */
7055 BTF_FUNC_PROTO_ENC(0, 2), /* [3] */
7056 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(2), 1),
7057 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(3), 1),
7058 BTF_FUNC_ENC(NAME_NTH(4), 2), /* [4] */
7059 BTF_TAG_ENC(NAME_NTH(5), 2, -1), /* [5] */
7060 BTF_TAG_ENC(NAME_NTH(5), 4, -1), /* [6] */
7061 BTF_TAG_ENC(NAME_NTH(5), 4, 1), /* [7] */
7062 BTF_END_RAW,
7063 },
7064 BTF_STR_SEC("\0t\0a1\0a2\0f\0tag"),
7065 },
7066 .opts = {
7067 .dont_resolve_fwds = false,
7068 },
7069},
7070{
7071 .descr = "dedup: func/func_param tags",
7072 .input = {
7073 .raw_types = {
7074 /* int */
7075 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
7076 /* void f(int a1, int a2) */
7077 BTF_FUNC_PROTO_ENC(0, 2), /* [2] */
7078 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(1), 1),
7079 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(2), 1),
7080 BTF_FUNC_ENC(NAME_NTH(3), 2), /* [3] */
7081 /* void f(int a1, int a2) */
7082 BTF_FUNC_PROTO_ENC(0, 2), /* [4] */
7083 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(1), 1),
7084 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(2), 1),
7085 BTF_FUNC_ENC(NAME_NTH(3), 4), /* [5] */
7086 /* tag -> f: tag1, tag2 */
7087 BTF_TAG_ENC(NAME_NTH(4), 3, -1), /* [6] */
7088 BTF_TAG_ENC(NAME_NTH(5), 3, -1), /* [7] */
7089 /* tag -> f/a2: tag1, tag2 */
7090 BTF_TAG_ENC(NAME_NTH(4), 3, 1), /* [8] */
7091 BTF_TAG_ENC(NAME_NTH(5), 3, 1), /* [9] */
7092 /* tag -> f: tag1, tag3 */
7093 BTF_TAG_ENC(NAME_NTH(4), 5, -1), /* [10] */
7094 BTF_TAG_ENC(NAME_NTH(6), 5, -1), /* [11] */
7095 /* tag -> f/a2: tag1, tag3 */
7096 BTF_TAG_ENC(NAME_NTH(4), 5, 1), /* [12] */
7097 BTF_TAG_ENC(NAME_NTH(6), 5, 1), /* [13] */
7098 BTF_END_RAW,
7099 },
7100 BTF_STR_SEC("\0a1\0a2\0f\0tag1\0tag2\0tag3"),
7101 },
7102 .expect = {
7103 .raw_types = {
7104 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
7105 BTF_FUNC_PROTO_ENC(0, 2), /* [2] */
7106 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(1), 1),
7107 BTF_FUNC_PROTO_ARG_ENC(NAME_NTH(2), 1),
7108 BTF_FUNC_ENC(NAME_NTH(3), 2), /* [3] */
7109 BTF_TAG_ENC(NAME_NTH(4), 3, -1), /* [4] */
7110 BTF_TAG_ENC(NAME_NTH(5), 3, -1), /* [5] */
7111 BTF_TAG_ENC(NAME_NTH(6), 3, -1), /* [6] */
7112 BTF_TAG_ENC(NAME_NTH(4), 3, 1), /* [7] */
7113 BTF_TAG_ENC(NAME_NTH(5), 3, 1), /* [8] */
7114 BTF_TAG_ENC(NAME_NTH(6), 3, 1), /* [9] */
7115 BTF_END_RAW,
7116 },
7117 BTF_STR_SEC("\0a1\0a2\0f\0tag1\0tag2\0tag3"),
7118 },
7119 .opts = {
7120 .dont_resolve_fwds = false,
7121 },
7122},
7123{
7124 .descr = "dedup: struct/struct_member tags",
7125 .input = {
7126 .raw_types = {
7127 /* int */
7128 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
7129 BTF_STRUCT_ENC(NAME_NTH(1), 2, 8), /* [2] */
7130 BTF_MEMBER_ENC(NAME_NTH(2), 1, 0),
7131 BTF_MEMBER_ENC(NAME_NTH(3), 1, 32),
7132 BTF_STRUCT_ENC(NAME_NTH(1), 2, 8), /* [3] */
7133 BTF_MEMBER_ENC(NAME_NTH(2), 1, 0),
7134 BTF_MEMBER_ENC(NAME_NTH(3), 1, 32),
7135 /* tag -> t: tag1, tag2 */
7136 BTF_TAG_ENC(NAME_NTH(4), 2, -1), /* [4] */
7137 BTF_TAG_ENC(NAME_NTH(5), 2, -1), /* [5] */
7138 /* tag -> t/m2: tag1, tag2 */
7139 BTF_TAG_ENC(NAME_NTH(4), 2, 1), /* [6] */
7140 BTF_TAG_ENC(NAME_NTH(5), 2, 1), /* [7] */
7141 /* tag -> t: tag1, tag3 */
7142 BTF_TAG_ENC(NAME_NTH(4), 3, -1), /* [8] */
7143 BTF_TAG_ENC(NAME_NTH(6), 3, -1), /* [9] */
7144 /* tag -> t/m2: tag1, tag3 */
7145 BTF_TAG_ENC(NAME_NTH(4), 3, 1), /* [10] */
7146 BTF_TAG_ENC(NAME_NTH(6), 3, 1), /* [11] */
7147 BTF_END_RAW,
7148 },
7149 BTF_STR_SEC("\0t\0m1\0m2\0tag1\0tag2\0tag3"),
7150 },
7151 .expect = {
7152 .raw_types = {
7153 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
7154 BTF_STRUCT_ENC(NAME_NTH(1), 2, 8), /* [2] */
7155 BTF_MEMBER_ENC(NAME_NTH(2), 1, 0),
7156 BTF_MEMBER_ENC(NAME_NTH(3), 1, 32),
7157 BTF_TAG_ENC(NAME_NTH(4), 2, -1), /* [3] */
7158 BTF_TAG_ENC(NAME_NTH(5), 2, -1), /* [4] */
7159 BTF_TAG_ENC(NAME_NTH(6), 2, -1), /* [5] */
7160 BTF_TAG_ENC(NAME_NTH(4), 2, 1), /* [6] */
7161 BTF_TAG_ENC(NAME_NTH(5), 2, 1), /* [7] */
7162 BTF_TAG_ENC(NAME_NTH(6), 2, 1), /* [8] */
7163 BTF_END_RAW,
7164 },
7165 BTF_STR_SEC("\0t\0m1\0m2\0tag1\0tag2\0tag3"),
7166 },
7167 .opts = {
7168 .dont_resolve_fwds = false,
7169 },
7170},
6770 7171
6771};7172};
6772 7173
@@ -6801,6 +7202,8 @@ static int btf_type_size(const struct btf_type *t) · 28 unmodified lines
6801 return base_size + sizeof(struct btf_var);7202 return base_size + sizeof(struct btf_var);
6802 case BTF_KIND_DATASEC:7203 case BTF_KIND_DATASEC:
6803 return base_size + vlen * sizeof(struct btf_var_secinfo);7204 return base_size + vlen * sizeof(struct btf_var_secinfo);
7205 case BTF_KIND_TAG:
7206 return base_size + sizeof(struct btf_tag);
6804 default:7207 default:
6805 fprintf(stderr, "Unsupported BTF_KIND:%u\n", kind);7208 fprintf(stderr, "Unsupported BTF_KIND:%u\n", kind);
6806 return -EINVAL;7209 return -EINVAL;
tools/testing/selftests/bpf/prog_tests/btf_tag.c+20−0
@@ -0,0 +1,20 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2021 Facebook */
3#include <test_progs.h>
4#include "tag.skel.h"
5
6void test_btf_tag(void)
7{
8 struct tag *skel;
9
10 skel = tag__open_and_load();
11 if (!ASSERT_OK_PTR(skel, "btf_tag"))
12 return;
13
14 if (skel->rodata->skip_tests) {
15 printf("%s:SKIP: btf_tag attribute not supported", __func__);
16 test__skip();
17 }
18
19 tag__destroy(skel);
20}
tools/testing/selftests/bpf/prog_tests/btf_write.c+21−0
@@ -281,5 +281,26 @@ void test_btf_write() { · 280 unmodified lines
281 "[17] DATASEC 'datasec1' size=12 vlen=1\n"281 "[17] DATASEC 'datasec1' size=12 vlen=1\n"
282 "\ttype_id=1 offset=4 size=8", "raw_dump");282 "\ttype_id=1 offset=4 size=8", "raw_dump");
283 283
284 /* TAG */
285 id = btf__add_tag(btf, "tag1", 16, -1);
286 ASSERT_EQ(id, 18, "tag_id");
287 t = btf__type_by_id(btf, 18);
288 ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "tag1", "tag_value");
289 ASSERT_EQ(btf_kind(t), BTF_KIND_TAG, "tag_kind");
290 ASSERT_EQ(t->type, 16, "tag_type");
291 ASSERT_EQ(btf_tag(t)->component_idx, -1, "tag_component_idx");
292 ASSERT_STREQ(btf_type_raw_dump(btf, 18),
293 "[18] TAG 'tag1' type_id=16 component_idx=-1", "raw_dump");
294
295 id = btf__add_tag(btf, "tag2", 14, 1);
296 ASSERT_EQ(id, 19, "tag_id");
297 t = btf__type_by_id(btf, 19);
298 ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "tag2", "tag_value");
299 ASSERT_EQ(btf_kind(t), BTF_KIND_TAG, "tag_kind");
300 ASSERT_EQ(t->type, 14, "tag_type");
301 ASSERT_EQ(btf_tag(t)->component_idx, 1, "tag_component_idx");
302 ASSERT_STREQ(btf_type_raw_dump(btf, 19),
303 "[19] TAG 'tag2' type_id=14 component_idx=1", "raw_dump");
304
284 btf__free(btf);305 btf__free(btf);
285}306}
tools/testing/selftests/bpf/prog_tests/core_reloc.c+4−13
@@ -30,7 +30,7 @@ static int duration = 0; · 29 unmodified lines
30 .output_len = sizeof(struct core_reloc_module_output), \30 .output_len = sizeof(struct core_reloc_module_output), \
31 .prog_sec_name = sec_name, \31 .prog_sec_name = sec_name, \
32 .raw_tp_name = tp_name, \32 .raw_tp_name = tp_name, \
33 .trigger = trigger_module_test_read, \33 .trigger = __trigger_module_test_read, \
34 .needs_testmod = true, \34 .needs_testmod = true, \
35}35}
36 36
@@ -249,8 +249,7 @@ static int duration = 0; · 212 unmodified lines
249#define SIZE_CASE_COMMON(name) \249#define SIZE_CASE_COMMON(name) \
250 .case_name = #name, \250 .case_name = #name, \
251 .bpf_obj_file = "test_core_reloc_size.o", \251 .bpf_obj_file = "test_core_reloc_size.o", \
252 .btf_src_file = "btf__core_reloc_" #name ".o", \252 .btf_src_file = "btf__core_reloc_" #name ".o"
253 .relaxed_core_relocs = true
254 253
255#define SIZE_OUTPUT_DATA(type) \254#define SIZE_OUTPUT_DATA(type) \
256 STRUCT_TO_CHAR_PTR(core_reloc_size_output) { \255 STRUCT_TO_CHAR_PTR(core_reloc_size_output) { \
@@ -475,19 +474,11 @@ static int setup_type_id_case_failure(struct core_reloc_test_case *test) · 218 unmodified lines
475 return 0;474 return 0;
476}475}
477 476
478static int trigger_module_test_read(const struct core_reloc_test_case *test)477static int __trigger_module_test_read(const struct core_reloc_test_case *test)
479{478{
480 struct core_reloc_module_output *exp = (void *)test->output;479 struct core_reloc_module_output *exp = (void *)test->output;
481 int fd, err;
482
483 fd = open("/sys/kernel/bpf_testmod", O_RDONLY);
484 err = -errno;
485 if (CHECK(fd < 0, "testmod_file_open", "failed: %d\n", err))
486 return err;
487
488 read(fd, NULL, exp->len); /* request expected number of bytes */
489 close(fd);
490 480
481 trigger_module_test_read(exp->len);
491 return 0;482 return 0;
492}483}
493 484
tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c+26−17
@@ -60,7 +60,7 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, · 59 unmodified lines
60 struct bpf_object *obj = NULL, *tgt_obj;60 struct bpf_object *obj = NULL, *tgt_obj;
61 __u32 retval, tgt_prog_id, info_len;61 __u32 retval, tgt_prog_id, info_len;
62 struct bpf_prog_info prog_info = {};62 struct bpf_prog_info prog_info = {};
63 struct bpf_program **prog = NULL;63 struct bpf_program **prog = NULL, *p;
64 struct bpf_link **link = NULL;64 struct bpf_link **link = NULL;
65 int err, tgt_fd, i;65 int err, tgt_fd, i;
66 struct btf *btf;66 struct btf *btf;
@@ -69,9 +69,6 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, · 2 unmodified lines
69 &tgt_obj, &tgt_fd);69 &tgt_obj, &tgt_fd);
70 if (!ASSERT_OK(err, "tgt_prog_load"))70 if (!ASSERT_OK(err, "tgt_prog_load"))
71 return;71 return;
72 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
73 .attach_prog_fd = tgt_fd,
74 );
75 72
76 info_len = sizeof(prog_info);73 info_len = sizeof(prog_info);
77 err = bpf_obj_get_info_by_fd(tgt_fd, &prog_info, &info_len);74 err = bpf_obj_get_info_by_fd(tgt_fd, &prog_info, &info_len);
@@ -89,10 +86,15 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, · 11 unmodified lines
89 if (!ASSERT_OK_PTR(prog, "prog_ptr"))86 if (!ASSERT_OK_PTR(prog, "prog_ptr"))
90 goto close_prog;87 goto close_prog;
91 88
92 obj = bpf_object__open_file(obj_file, &opts);89 obj = bpf_object__open_file(obj_file, NULL);
93 if (!ASSERT_OK_PTR(obj, "obj_open"))90 if (!ASSERT_OK_PTR(obj, "obj_open"))
94 goto close_prog;91 goto close_prog;
95 92
93 bpf_object__for_each_program(p, obj) {
94 err = bpf_program__set_attach_target(p, tgt_fd, NULL);
95 ASSERT_OK(err, "set_attach_target");
96 }
97
96 err = bpf_object__load(obj);98 err = bpf_object__load(obj);
97 if (!ASSERT_OK(err, "obj_load"))99 if (!ASSERT_OK(err, "obj_load"))
98 goto close_prog;100 goto close_prog;
@@ -270,7 +272,7 @@ static void test_fmod_ret_freplace(void) · 171 unmodified lines
270 struct bpf_link *freplace_link = NULL;272 struct bpf_link *freplace_link = NULL;
271 struct bpf_program *prog;273 struct bpf_program *prog;
272 __u32 duration = 0;274 __u32 duration = 0;
273 int err, pkt_fd;275 int err, pkt_fd, attach_prog_fd;
274 276
275 err = bpf_prog_load(tgt_name, BPF_PROG_TYPE_UNSPEC,277 err = bpf_prog_load(tgt_name, BPF_PROG_TYPE_UNSPEC,
276 &pkt_obj, &pkt_fd);278 &pkt_obj, &pkt_fd);
@@ -278,26 +280,32 @@ static void test_fmod_ret_freplace(void) · 1 unmodified line
278 if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n",280 if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n",
279 tgt_name, err, errno))281 tgt_name, err, errno))
280 return;282 return;
281 opts.attach_prog_fd = pkt_fd;
282 283
283 freplace_obj = bpf_object__open_file(freplace_name, &opts);284 freplace_obj = bpf_object__open_file(freplace_name, NULL);
284 if (!ASSERT_OK_PTR(freplace_obj, "freplace_obj_open"))285 if (!ASSERT_OK_PTR(freplace_obj, "freplace_obj_open"))
285 goto out;286 goto out;
286 287
288 prog = bpf_program__next(NULL, freplace_obj);
289 err = bpf_program__set_attach_target(prog, pkt_fd, NULL);
290 ASSERT_OK(err, "freplace__set_attach_target");
291
287 err = bpf_object__load(freplace_obj);292 err = bpf_object__load(freplace_obj);
288 if (CHECK(err, "freplace_obj_load", "err %d\n", err))293 if (CHECK(err, "freplace_obj_load", "err %d\n", err))
289 goto out;294 goto out;
290 295
291 prog = bpf_program__next(NULL, freplace_obj);
292 freplace_link = bpf_program__attach_trace(prog);296 freplace_link = bpf_program__attach_trace(prog);
293 if (!ASSERT_OK_PTR(freplace_link, "freplace_attach_trace"))297 if (!ASSERT_OK_PTR(freplace_link, "freplace_attach_trace"))
294 goto out;298 goto out;
295 299
296 opts.attach_prog_fd = bpf_program__fd(prog);300 fmod_obj = bpf_object__open_file(fmod_ret_name, NULL);
297 fmod_obj = bpf_object__open_file(fmod_ret_name, &opts);
298 if (!ASSERT_OK_PTR(fmod_obj, "fmod_obj_open"))301 if (!ASSERT_OK_PTR(fmod_obj, "fmod_obj_open"))
299 goto out;302 goto out;
300 303
304 attach_prog_fd = bpf_program__fd(prog);
305 prog = bpf_program__next(NULL, fmod_obj);
306 err = bpf_program__set_attach_target(prog, attach_prog_fd, NULL);
307 ASSERT_OK(err, "fmod_ret_set_attach_target");
308
301 err = bpf_object__load(fmod_obj);309 err = bpf_object__load(fmod_obj);
302 if (CHECK(!err, "fmod_obj_load", "loading fmod_ret should fail\n"))310 if (CHECK(!err, "fmod_obj_load", "loading fmod_ret should fail\n"))
303 goto out;311 goto out;
@@ -322,14 +330,14 @@ static void test_func_sockmap_update(void) · 18 unmodified lines
322}330}
323 331
324static void test_obj_load_failure_common(const char *obj_file,332static void test_obj_load_failure_common(const char *obj_file,
325 const char *target_obj_file)333 const char *target_obj_file)
326
327{334{
328 /*335 /*
329 * standalone test that asserts failure to load freplace prog336 * standalone test that asserts failure to load freplace prog
330 * because of invalid return code.337 * because of invalid return code.
331 */338 */
332 struct bpf_object *obj = NULL, *pkt_obj;339 struct bpf_object *obj = NULL, *pkt_obj;
340 struct bpf_program *prog;
333 int err, pkt_fd;341 int err, pkt_fd;
334 __u32 duration = 0;342 __u32 duration = 0;
335 343
@@ -339,14 +347,15 @@ static void test_obj_load_failure_common(const char *obj_file, · 3 unmodified lines
339 if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n",347 if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n",
340 target_obj_file, err, errno))348 target_obj_file, err, errno))
341 return;349 return;
342 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
343 .attach_prog_fd = pkt_fd,
344 );
345 350
346 obj = bpf_object__open_file(obj_file, &opts);351 obj = bpf_object__open_file(obj_file, NULL);
347 if (!ASSERT_OK_PTR(obj, "obj_open"))352 if (!ASSERT_OK_PTR(obj, "obj_open"))
348 goto close_prog;353 goto close_prog;
349 354
355 prog = bpf_program__next(NULL, obj);
356 err = bpf_program__set_attach_target(prog, pkt_fd, NULL);
357 ASSERT_OK(err, "set_attach_target");
358
350 /* It should fail to load the program */359 /* It should fail to load the program */
351 err = bpf_object__load(obj);360 err = bpf_object__load(obj);
352 if (CHECK(!err, "bpf_obj_load should fail", "err %d\n", err))361 if (CHECK(!err, "bpf_obj_load should fail", "err %d\n", err))
tools/testing/selftests/bpf/prog_tests/get_branch_snapshot.c+100−0
@@ -0,0 +1,100 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2021 Facebook */
3#include <test_progs.h>
4#include "get_branch_snapshot.skel.h"
5
6static int *pfd_array;
7static int cpu_cnt;
8
9static int create_perf_events(void)
10{
11 struct perf_event_attr attr = {0};
12 int cpu;
13
14 /* create perf event */
15 attr.size = sizeof(attr);
16 attr.type = PERF_TYPE_RAW;
17 attr.config = 0x1b00;
18 attr.sample_type = PERF_SAMPLE_BRANCH_STACK;
19 attr.branch_sample_type = PERF_SAMPLE_BRANCH_KERNEL |
20 PERF_SAMPLE_BRANCH_USER | PERF_SAMPLE_BRANCH_ANY;
21
22 cpu_cnt = libbpf_num_possible_cpus();
23 pfd_array = malloc(sizeof(int) * cpu_cnt);
24 if (!pfd_array) {
25 cpu_cnt = 0;
26 return 1;
27 }
28
29 for (cpu = 0; cpu < cpu_cnt; cpu++) {
30 pfd_array[cpu] = syscall(__NR_perf_event_open, &attr,
31 -1, cpu, -1, PERF_FLAG_FD_CLOEXEC);
32 if (pfd_array[cpu] < 0)
33 break;
34 }
35
36 return cpu == 0;
37}
38
39static void close_perf_events(void)
40{
41 int cpu = 0;
42 int fd;
43
44 while (cpu++ < cpu_cnt) {
45 fd = pfd_array[cpu];
46 if (fd < 0)
47 break;
48 close(fd);
49 }
50 free(pfd_array);
51}
52
53void test_get_branch_snapshot(void)
54{
55 struct get_branch_snapshot *skel = NULL;
56 int err;
57
58 if (create_perf_events()) {
59 test__skip(); /* system doesn't support LBR */
60 goto cleanup;
61 }
62
63 skel = get_branch_snapshot__open_and_load();
64 if (!ASSERT_OK_PTR(skel, "get_branch_snapshot__open_and_load"))
65 goto cleanup;
66
67 err = kallsyms_find("bpf_testmod_loop_test", &skel->bss->address_low);
68 if (!ASSERT_OK(err, "kallsyms_find"))
69 goto cleanup;
70
71 err = kallsyms_find_next("bpf_testmod_loop_test", &skel->bss->address_high);
72 if (!ASSERT_OK(err, "kallsyms_find_next"))
73 goto cleanup;
74
75 err = get_branch_snapshot__attach(skel);
76 if (!ASSERT_OK(err, "get_branch_snapshot__attach"))
77 goto cleanup;
78
79 trigger_module_test_read(100);
80
81 if (skel->bss->total_entries < 16) {
82 /* too few entries for the hit/waste test */
83 test__skip();
84 goto cleanup;
85 }
86
87 ASSERT_GT(skel->bss->test1_hits, 6, "find_looptest_in_lbr");
88
89 /* Given we stop LBR in software, we will waste a few entries.
90 * But we should try to waste as few as possible entries. We are at
91 * about 7 on x86_64 systems.
92 * Add a check for < 10 so that we get heads-up when something
93 * changes and wastes too many entries.
94 */
95 ASSERT_LT(skel->bss->wasted_entries, 10, "check_wasted_entries");
96
97cleanup:
98 get_branch_snapshot__destroy(skel);
99 close_perf_events();
100}
tools/testing/selftests/bpf/prog_tests/module_attach.c+0−39
@@ -6,45 +6,6 @@ · 5 unmodified lines
6 6
7static int duration;7static int duration;
8 8
9static int trigger_module_test_read(int read_sz)
10{
11 int fd, err;
12
13 fd = open("/sys/kernel/bpf_testmod", O_RDONLY);
14 err = -errno;
15 if (CHECK(fd < 0, "testmod_file_open", "failed: %d\n", err))
16 return err;
17
18 read(fd, NULL, read_sz);
19 close(fd);
20
21 return 0;
22}
23
24static int trigger_module_test_write(int write_sz)
25{
26 int fd, err;
27 char *buf = malloc(write_sz);
28
29 if (!buf)
30 return -ENOMEM;
31
32 memset(buf, 'a', write_sz);
33 buf[write_sz-1] = '\0';
34
35 fd = open("/sys/kernel/bpf_testmod", O_WRONLY);
36 err = -errno;
37 if (CHECK(fd < 0, "testmod_file_open", "failed: %d\n", err)) {
38 free(buf);
39 return err;
40 }
41
42 write(fd, buf, write_sz);
43 close(fd);
44 free(buf);
45 return 0;
46}
47
48static int delete_module(const char *name, int flags)9static int delete_module(const char *name, int flags)
49{10{
50 return syscall(__NR_delete_module, name, flags);11 return syscall(__NR_delete_module, name, flags);
tools/testing/selftests/bpf/prog_tests/skb_ctx.c+6−0
@@ -11,12 +11,14 @@ void test_skb_ctx(void) · 10 unmodified lines
11 .cb[3] = 4,11 .cb[3] = 4,
12 .cb[4] = 5,12 .cb[4] = 5,
13 .priority = 6,13 .priority = 6,
14 .ingress_ifindex = 11,
14 .ifindex = 1,15 .ifindex = 1,
15 .tstamp = 7,16 .tstamp = 7,
16 .wire_len = 100,17 .wire_len = 100,
17 .gso_segs = 8,18 .gso_segs = 8,
18 .mark = 9,19 .mark = 9,
19 .gso_size = 10,20 .gso_size = 10,
21 .hwtstamp = 11,
20 };22 };
21 struct bpf_prog_test_run_attr tattr = {23 struct bpf_prog_test_run_attr tattr = {
22 .data_in = &pkt_v4,24 .data_in = &pkt_v4,
@@ -97,6 +99,10 @@ void test_skb_ctx(void) · 74 unmodified lines
97 "ctx_out_ifindex",99 "ctx_out_ifindex",
98 "skb->ifindex == %d, expected %d\n",100 "skb->ifindex == %d, expected %d\n",
99 skb.ifindex, 1);101 skb.ifindex, 1);
102 CHECK_ATTR(skb.ingress_ifindex != 11,
103 "ctx_out_ingress_ifindex",
104 "skb->ingress_ifindex == %d, expected %d\n",
105 skb.ingress_ifindex, 11);
100 CHECK_ATTR(skb.tstamp != 8,106 CHECK_ATTR(skb.tstamp != 8,
101 "ctx_out_tstamp",107 "ctx_out_tstamp",
102 "skb->tstamp == %lld, expected %d\n",108 "skb->tstamp == %lld, expected %d\n",
tools/testing/selftests/bpf/prog_tests/skeleton.c+6−0
@@ -18,6 +18,8 @@ void test_skeleton(void) · 17 unmodified lines
18 struct test_skeleton__data *data;18 struct test_skeleton__data *data;
19 struct test_skeleton__rodata *rodata;19 struct test_skeleton__rodata *rodata;
20 struct test_skeleton__kconfig *kcfg;20 struct test_skeleton__kconfig *kcfg;
21 const void *elf_bytes;
22 size_t elf_bytes_sz = 0;
21 23
22 skel = test_skeleton__open();24 skel = test_skeleton__open();
23 if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))25 if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
@@ -91,6 +93,10 @@ void test_skeleton(void) · 67 unmodified lines
91 CHECK(bss->kern_ver != kcfg->LINUX_KERNEL_VERSION, "ext2",93 CHECK(bss->kern_ver != kcfg->LINUX_KERNEL_VERSION, "ext2",
92 "got %d != exp %d\n", bss->kern_ver, kcfg->LINUX_KERNEL_VERSION);94 "got %d != exp %d\n", bss->kern_ver, kcfg->LINUX_KERNEL_VERSION);
93 95
96 elf_bytes = test_skeleton__elf_bytes(&elf_bytes_sz);
97 ASSERT_OK_PTR(elf_bytes, "elf_bytes");
98 ASSERT_GE(elf_bytes_sz, 0, "elf_bytes_sz");
99
94cleanup:100cleanup:
95 test_skeleton__destroy(skel);101 test_skeleton__destroy(skel);
96}102}
tools/testing/selftests/bpf/prog_tests/tailcalls.c+20−5
@@ -219,10 +219,7 @@ out: · 218 unmodified lines
219 bpf_object__close(obj);219 bpf_object__close(obj);
220}220}
221 221
222/* test_tailcall_3 checks that the count value of the tail call limit222static void test_tailcall_count(const char *which)
223 * enforcement matches with expectations.
224 */
225static void test_tailcall_3(void)
226{223{
227 int err, map_fd, prog_fd, main_fd, data_fd, i, val;224 int err, map_fd, prog_fd, main_fd, data_fd, i, val;
228 struct bpf_map *prog_array, *data_map;225 struct bpf_map *prog_array, *data_map;
@@ -231,7 +228,7 @@ static void test_tailcall_3(void) · 2 unmodified lines
231 __u32 retval, duration;228 __u32 retval, duration;
232 char buff[128] = {};229 char buff[128] = {};
233 230
234 err = bpf_prog_load("tailcall3.o", BPF_PROG_TYPE_SCHED_CLS, &obj,231 err = bpf_prog_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj,
235 &prog_fd);232 &prog_fd);
236 if (CHECK_FAIL(err))233 if (CHECK_FAIL(err))
237 return;234 return;
@@ -296,6 +293,22 @@ out: · 58 unmodified lines
296 bpf_object__close(obj);293 bpf_object__close(obj);
297}294}
298 295
296/* test_tailcall_3 checks that the count value of the tail call limit
297 * enforcement matches with expectations. JIT uses direct jump.
298 */
299static void test_tailcall_3(void)
300{
301 test_tailcall_count("tailcall3.o");
302}
303
304/* test_tailcall_6 checks that the count value of the tail call limit
305 * enforcement matches with expectations. JIT uses indirect jump.
306 */
307static void test_tailcall_6(void)
308{
309 test_tailcall_count("tailcall6.o");
310}
311
299/* test_tailcall_4 checks that the kernel properly selects indirect jump312/* test_tailcall_4 checks that the kernel properly selects indirect jump
300 * for the case where the key is not known. Latter is passed via global313 * for the case where the key is not known. Latter is passed via global
301 * data to select different targets we can compare return value of.314 * data to select different targets we can compare return value of.
@@ -822,6 +835,8 @@ void test_tailcalls(void) · 520 unmodified lines
822 test_tailcall_4();835 test_tailcall_4();
823 if (test__start_subtest("tailcall_5"))836 if (test__start_subtest("tailcall_5"))
824 test_tailcall_5();837 test_tailcall_5();
838 if (test__start_subtest("tailcall_6"))
839 test_tailcall_6();
825 if (test__start_subtest("tailcall_bpf2bpf_1"))840 if (test__start_subtest("tailcall_bpf2bpf_1"))
826 test_tailcall_bpf2bpf_1();841 test_tailcall_bpf2bpf_1();
827 if (test__start_subtest("tailcall_bpf2bpf_2"))842 if (test__start_subtest("tailcall_bpf2bpf_2"))
tools/testing/selftests/bpf/prog_tests/tc_redirect.c+1−1
@@ -633,7 +633,7 @@ static void test_tc_redirect_peer_l3(struct netns_setup_result *setup_result) · 632 unmodified lines
633 struct nstoken *nstoken = NULL;633 struct nstoken *nstoken = NULL;
634 int err;634 int err;
635 int tunnel_pid = -1;635 int tunnel_pid = -1;
636 int src_fd, target_fd;636 int src_fd, target_fd = -1;
637 int ifindex;637 int ifindex;
638 638
639 /* Start a L3 TUN/TAP tunnel between the src and dst namespaces.639 /* Start a L3 TUN/TAP tunnel between the src and dst namespaces.
tools/testing/selftests/bpf/progs/bpf_cubic.c+3−9
@@ -169,11 +169,7 @@ static __always_inline void bictcp_hystart_reset(struct sock *sk) · 168 unmodified lines
169 ca->sample_cnt = 0;169 ca->sample_cnt = 0;
170}170}
171 171
172/* "struct_ops/" prefix is not a requirement172/* "struct_ops/" prefix is a requirement */
173 * It will be recognized as BPF_PROG_TYPE_STRUCT_OPS
174 * as long as it is used in one of the func ptr
175 * under SEC(".struct_ops").
176 */
177SEC("struct_ops/bpf_cubic_init")173SEC("struct_ops/bpf_cubic_init")
178void BPF_PROG(bpf_cubic_init, struct sock *sk)174void BPF_PROG(bpf_cubic_init, struct sock *sk)
179{175{
@@ -188,10 +184,8 @@ void BPF_PROG(bpf_cubic_init, struct sock *sk) · 8 unmodified lines
188 tcp_sk(sk)->snd_ssthresh = initial_ssthresh;184 tcp_sk(sk)->snd_ssthresh = initial_ssthresh;
189}185}
190 186
191/* No prefix in SEC will also work.187/* "struct_ops" prefix is a requirement */
192 * The remaining tcp-cubic functions have an easier way.188SEC("struct_ops/bpf_cubic_cwnd_event")
193 */
194SEC("no-sec-prefix-bictcp_cwnd_event")
195void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)189void BPF_PROG(bpf_cubic_cwnd_event, struct sock *sk, enum tcp_ca_event event)
196{190{
197 if (event == CA_EVENT_TX_START) {191 if (event == CA_EVENT_TX_START) {
tools/testing/selftests/bpf/progs/get_branch_snapshot.c+40−0
@@ -0,0 +1,40 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2021 Facebook */
3#include "vmlinux.h"
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6
7char _license[] SEC("license") = "GPL";
8
9__u64 test1_hits = 0;
10__u64 address_low = 0;
11__u64 address_high = 0;
12int wasted_entries = 0;
13long total_entries = 0;
14
15#define ENTRY_CNT 32
16struct perf_branch_entry entries[ENTRY_CNT] = {};
17
18static inline bool in_range(__u64 val)
19{
20 return (val >= address_low) && (val < address_high);
21}
22
23SEC("fexit/bpf_testmod_loop_test")
24int BPF_PROG(test1, int n, int ret)
25{
26 long i;
27
28 total_entries = bpf_get_branch_snapshot(entries, sizeof(entries), 0);
29 total_entries /= sizeof(struct perf_branch_entry);
30
31 for (i = 0; i < ENTRY_CNT; i++) {
32 if (i >= total_entries)
33 break;
34 if (in_range(entries[i].from) && in_range(entries[i].to))
35 test1_hits++;
36 else if (!test1_hits)
37 wasted_entries++;
38 }
39 return 0;
40}
tools/testing/selftests/bpf/progs/tag.c+49−0
@@ -0,0 +1,49 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2021 Facebook */
3#include "vmlinux.h"
4#include <bpf/bpf_helpers.h>
5#include <bpf/bpf_tracing.h>
6
7#ifndef __has_attribute
8#define __has_attribute(x) 0
9#endif
10
11#if __has_attribute(btf_tag)
12#define __tag1 __attribute__((btf_tag("tag1")))
13#define __tag2 __attribute__((btf_tag("tag2")))
14volatile const bool skip_tests __tag1 __tag2 = false;
15#else
16#define __tag1
17#define __tag2
18volatile const bool skip_tests = true;
19#endif
20
21struct key_t {
22 int a;
23 int b __tag1 __tag2;
24 int c;
25} __tag1 __tag2;
26
27struct {
28 __uint(type, BPF_MAP_TYPE_HASH);
29 __uint(max_entries, 3);
30 __type(key, struct key_t);
31 __type(value, __u64);
32} hashmap1 SEC(".maps");
33
34
35static __noinline int foo(int x __tag1 __tag2) __tag1 __tag2
36{
37 struct key_t key;
38 __u64 val = 1;
39
40 key.a = key.b = key.c = x;
41 bpf_map_update_elem(&hashmap1, &key, &val, 0);
42 return 0;
43}
44
45SEC("fentry/bpf_fentry_test1")
46int BPF_PROG(sub, int x)
47{
48 return foo(x);
49}
tools/testing/selftests/bpf/progs/tailcall6.c+34−0
@@ -0,0 +1,34 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/bpf.h>
3
4#include <bpf/bpf_helpers.h>
5
6struct {
7 __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
8 __uint(max_entries, 1);
9 __uint(key_size, sizeof(__u32));
10 __uint(value_size, sizeof(__u32));
11} jmp_table SEC(".maps");
12
13int count, which;
14
15SEC("classifier/0")
16int bpf_func_0(struct __sk_buff *skb)
17{
18 count++;
19 if (__builtin_constant_p(which))
20 __bpf_unreachable();
21 bpf_tail_call(skb, &jmp_table, which);
22 return 1;
23}
24
25SEC("classifier")
26int entry(struct __sk_buff *skb)
27{
28 if (__builtin_constant_p(which))
29 __bpf_unreachable();
30 bpf_tail_call(skb, &jmp_table, which);
31 return 0;
32}
33
34char __license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/test_skb_ctx.c+6−0
@@ -25,6 +25,12 @@ int process(struct __sk_buff *skb) · 24 unmodified lines
25 return 1;25 return 1;
26 if (skb->gso_size != 10)26 if (skb->gso_size != 10)
27 return 1;27 return 1;
28 if (skb->ingress_ifindex != 11)
29 return 1;
30 if (skb->ifindex != 1)
31 return 1;
32 if (skb->hwtstamp != 11)
33 return 1;
28 34
29 return 0;35 return 0;
30}36}
tools/testing/selftests/bpf/test_btf.h+3−0
@@ -69,4 +69,7 @@ · 68 unmodified lines
69#define BTF_TYPE_FLOAT_ENC(name, sz) \69#define BTF_TYPE_FLOAT_ENC(name, sz) \
70 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)70 BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)
71 71
72#define BTF_TAG_ENC(value, type, component_idx) \
73 BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TAG, 0, 0), type), (component_idx)
74
72#endif /* _TEST_BTF_H */75#endif /* _TEST_BTF_H */
tools/testing/selftests/bpf/test_progs.c+39−0
@@ -743,6 +743,45 @@ int cd_flavor_subdir(const char *exec_name) · 742 unmodified lines
743 return chdir(flavor);743 return chdir(flavor);
744}744}
745 745
746int trigger_module_test_read(int read_sz)
747{
748 int fd, err;
749
750 fd = open("/sys/kernel/bpf_testmod", O_RDONLY);
751 err = -errno;
752 if (!ASSERT_GE(fd, 0, "testmod_file_open"))
753 return err;
754
755 read(fd, NULL, read_sz);
756 close(fd);
757
758 return 0;
759}
760
761int trigger_module_test_write(int write_sz)
762{
763 int fd, err;
764 char *buf = malloc(write_sz);
765
766 if (!buf)
767 return -ENOMEM;
768
769 memset(buf, 'a', write_sz);
770 buf[write_sz-1] = '\0';
771
772 fd = open("/sys/kernel/bpf_testmod", O_WRONLY);
773 err = -errno;
774 if (!ASSERT_GE(fd, 0, "testmod_file_open")) {
775 free(buf);
776 return err;
777 }
778
779 write(fd, buf, write_sz);
780 close(fd);
781 free(buf);
782 return 0;
783}
784
746#define MAX_BACKTRACE_SZ 128785#define MAX_BACKTRACE_SZ 128
747void crash_handler(int signum)786void crash_handler(int signum)
748{787{
tools/testing/selftests/bpf/test_progs.h+2−0
@@ -291,6 +291,8 @@ int compare_map_keys(int map1_fd, int map2_fd); · 290 unmodified lines
291int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len);291int compare_stack_ips(int smap_fd, int amap_fd, int stack_trace_len);
292int extract_build_id(char *build_id, size_t size);292int extract_build_id(char *build_id, size_t size);
293int kern_sync_rcu(void);293int kern_sync_rcu(void);
294int trigger_module_test_read(int read_sz);
295int trigger_module_test_write(int write_sz);
294 296
295#ifdef __x86_64__297#ifdef __x86_64__
296#define SYS_NANOSLEEP_KPROBE_NAME "__x64_sys_nanosleep"298#define SYS_NANOSLEEP_KPROBE_NAME "__x64_sys_nanosleep"
tools/testing/selftests/bpf/trace_helpers.c+37−0
@@ -1,4 +1,5 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2#include <ctype.h>
2#include <stdio.h>3#include <stdio.h>
3#include <stdlib.h>4#include <stdlib.h>
4#include <string.h>5#include <string.h>
@@ -117,6 +118,42 @@ out: · 112 unmodified lines
117 return err;118 return err;
118}119}
119 120
121/* find the address of the next symbol of the same type, this can be used
122 * to determine the end of a function.
123 */
124int kallsyms_find_next(const char *sym, unsigned long long *addr)
125{
126 char type, found_type, name[500];
127 unsigned long long value;
128 bool found = false;
129 int err = 0;
130 FILE *f;
131
132 f = fopen("/proc/kallsyms", "r");
133 if (!f)
134 return -EINVAL;
135
136 while (fscanf(f, "%llx %c %499s%*[^\n]\n", &value, &type, name) > 0) {
137 /* Different types of symbols in kernel modules are mixed
138 * in /proc/kallsyms. Only return the next matching type.
139 * Use tolower() for type so that 'T' matches 't'.
140 */
141 if (found && found_type == tolower(type)) {
142 *addr = value;
143 goto out;
144 }
145 if (strcmp(name, sym) == 0) {
146 found = true;
147 found_type = tolower(type);
148 }
149 }
150 err = -ENOENT;
151
152out:
153 fclose(f);
154 return err;
155}
156
120void read_trace_pipe(void)157void read_trace_pipe(void)
121{158{
122 int trace_fd;159 int trace_fd;
tools/testing/selftests/bpf/trace_helpers.h+5−0
@@ -16,6 +16,11 @@ long ksym_get_addr(const char *name); · 15 unmodified lines
16/* open kallsyms and find addresses on the fly, faster than load + search. */16/* open kallsyms and find addresses on the fly, faster than load + search. */
17int kallsyms_find(const char *sym, unsigned long long *addr);17int kallsyms_find(const char *sym, unsigned long long *addr);
18 18
19/* find the address of the next symbol, this can be used to determine the
20 * end of a function
21 */
22int kallsyms_find_next(const char *sym, unsigned long long *addr);
23
19void read_trace_pipe(void);24void read_trace_pipe(void);
20 25
21ssize_t get_uprobe_offset(const void *addr, ssize_t base);26ssize_t get_uprobe_offset(const void *addr, ssize_t base);
tools/testing/selftests/bpf/verifier/ctx_skb.c+60−0
@@ -1057,6 +1057,66 @@ · 1056 unmodified lines
1057 .result = ACCEPT,1057 .result = ACCEPT,
1058 .prog_type = BPF_PROG_TYPE_SCHED_CLS,1058 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
1059},1059},
1060{
1061 "padding after gso_size is not accessible",
1062 .insns = {
1063 BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
1064 offsetofend(struct __sk_buff, gso_size)),
1065 BPF_MOV64_IMM(BPF_REG_0, 0),
1066 BPF_EXIT_INSN(),
1067 },
1068 .result = REJECT,
1069 .result_unpriv = REJECT,
1070 .errstr = "invalid bpf_context access off=180 size=4",
1071 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
1072},
1073{
1074 "read hwtstamp from CGROUP_SKB",
1075 .insns = {
1076 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1,
1077 offsetof(struct __sk_buff, hwtstamp)),
1078 BPF_MOV64_IMM(BPF_REG_0, 0),
1079 BPF_EXIT_INSN(),
1080 },
1081 .result = ACCEPT,
1082 .prog_type = BPF_PROG_TYPE_CGROUP_SKB,
1083},
1084{
1085 "read hwtstamp from CGROUP_SKB",
1086 .insns = {
1087 BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1,
1088 offsetof(struct __sk_buff, hwtstamp)),
1089 BPF_MOV64_IMM(BPF_REG_0, 0),
1090 BPF_EXIT_INSN(),
1091 },
1092 .result = ACCEPT,
1093 .prog_type = BPF_PROG_TYPE_CGROUP_SKB,
1094},
1095{
1096 "write hwtstamp from CGROUP_SKB",
1097 .insns = {
1098 BPF_MOV64_IMM(BPF_REG_0, 0),
1099 BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_0,
1100 offsetof(struct __sk_buff, hwtstamp)),
1101 BPF_MOV64_IMM(BPF_REG_0, 0),
1102 BPF_EXIT_INSN(),
1103 },
1104 .result = REJECT,
1105 .result_unpriv = REJECT,
1106 .errstr = "invalid bpf_context access off=184 size=8",
1107 .prog_type = BPF_PROG_TYPE_CGROUP_SKB,
1108},
1109{
1110 "read hwtstamp from CLS",
1111 .insns = {
1112 BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1,
1113 offsetof(struct __sk_buff, hwtstamp)),
1114 BPF_MOV64_IMM(BPF_REG_0, 0),
1115 BPF_EXIT_INSN(),
1116 },
1117 .result = ACCEPT,
1118 .prog_type = BPF_PROG_TYPE_SCHED_CLS,
1119},
1060{1120{
1061 "check wire_len is not readable by sockets",1121 "check wire_len is not readable by sockets",
1062 .insns = {1122 .insns = {
tools/testing/selftests/bpf/verifier/jit.c+19−3
@@ -62,6 +62,11 @@ · 61 unmodified lines
62 BPF_JMP_REG(BPF_JEQ, BPF_REG_3, BPF_REG_2, 2),62 BPF_JMP_REG(BPF_JEQ, BPF_REG_3, BPF_REG_2, 2),
63 BPF_MOV64_IMM(BPF_REG_0, 1),63 BPF_MOV64_IMM(BPF_REG_0, 1),
64 BPF_EXIT_INSN(),64 BPF_EXIT_INSN(),
65 BPF_LD_IMM64(BPF_REG_3, 0xfefefeULL),
66 BPF_ALU64_IMM(BPF_MUL, BPF_REG_3, 0xefefef),
67 BPF_JMP_REG(BPF_JEQ, BPF_REG_3, BPF_REG_2, 2),
68 BPF_MOV64_IMM(BPF_REG_0, 1),
69 BPF_EXIT_INSN(),
65 BPF_MOV32_REG(BPF_REG_2, BPF_REG_2),70 BPF_MOV32_REG(BPF_REG_2, BPF_REG_2),
66 BPF_LD_IMM64(BPF_REG_0, 0xfefefeULL),71 BPF_LD_IMM64(BPF_REG_0, 0xfefefeULL),
67 BPF_ALU32_REG(BPF_MUL, BPF_REG_0, BPF_REG_1),72 BPF_ALU32_REG(BPF_MUL, BPF_REG_0, BPF_REG_1),
@@ -73,11 +78,22 @@ · 5 unmodified lines
73 BPF_JMP_REG(BPF_JEQ, BPF_REG_3, BPF_REG_2, 2),78 BPF_JMP_REG(BPF_JEQ, BPF_REG_3, BPF_REG_2, 2),
74 BPF_MOV64_IMM(BPF_REG_0, 1),79 BPF_MOV64_IMM(BPF_REG_0, 1),
75 BPF_EXIT_INSN(),80 BPF_EXIT_INSN(),
81 BPF_LD_IMM64(BPF_REG_3, 0xfefefeULL),
82 BPF_ALU32_IMM(BPF_MUL, BPF_REG_3, 0xefefef),
83 BPF_JMP_REG(BPF_JEQ, BPF_REG_3, BPF_REG_2, 2),
84 BPF_MOV64_IMM(BPF_REG_0, 1),
85 BPF_EXIT_INSN(),
86 BPF_LD_IMM64(BPF_REG_0, 0xfefefeULL),
87 BPF_LD_IMM64(BPF_REG_2, 0x2ad4d4aaULL),
88 BPF_ALU32_IMM(BPF_MUL, BPF_REG_0, 0x2b),
89 BPF_JMP_REG(BPF_JEQ, BPF_REG_0, BPF_REG_2, 2),
90 BPF_MOV64_IMM(BPF_REG_0, 1),
91 BPF_EXIT_INSN(),
76 BPF_LD_IMM64(BPF_REG_0, 0x952a7bbcULL),92 BPF_LD_IMM64(BPF_REG_0, 0x952a7bbcULL),
77 BPF_LD_IMM64(BPF_REG_1, 0xfefefeULL),93 BPF_LD_IMM64(BPF_REG_1, 0xfefefeULL),
78 BPF_LD_IMM64(BPF_REG_2, 0xeeff0d413122ULL),94 BPF_LD_IMM64(BPF_REG_5, 0xeeff0d413122ULL),
79 BPF_ALU32_REG(BPF_MUL, BPF_REG_2, BPF_REG_1),95 BPF_ALU32_REG(BPF_MUL, BPF_REG_5, BPF_REG_1),
80 BPF_JMP_REG(BPF_JEQ, BPF_REG_2, BPF_REG_0, 2),96 BPF_JMP_REG(BPF_JEQ, BPF_REG_5, BPF_REG_0, 2),
81 BPF_MOV64_IMM(BPF_REG_0, 1),97 BPF_MOV64_IMM(BPF_REG_0, 1),
82 BPF_EXIT_INSN(),98 BPF_EXIT_INSN(),
83 BPF_MOV64_IMM(BPF_REG_0, 2),99 BPF_MOV64_IMM(BPF_REG_0, 2),
tools/testing/selftests/bpf/xdpxceiver.c+570−302
@@ -19,7 +19,7 @@ · 18 unmodified lines
19 * Virtual Ethernet interfaces.19 * Virtual Ethernet interfaces.
20 *20 *
21 * For each mode, the following tests are run:21 * For each mode, the following tests are run:
22 * a. nopoll - soft-irq processing22 * a. nopoll - soft-irq processing in run-to-completion mode
23 * b. poll - using poll() syscall23 * b. poll - using poll() syscall
24 * c. Socket Teardown24 * c. Socket Teardown
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
@@ -45,6 +45,10 @@ · 19 unmodified lines
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 147 * finally run a traffic on queues ids 1
48 * g. unaligned mode
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
51 * i. 2K frame size tests
48 *52 *
49 * Total tests: 1253 * Total tests: 12
50 *54 *
@@ -112,13 +116,10 @@ static void __exit_with_error(int error, const char *file, const char *func, int · 61 unmodified lines
112 116
113#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)117#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
114 118
115#define print_ksft_result(void)\119#define mode_string(test) (test)->ifobj_tx->xdp_flags & XDP_FLAGS_SKB_MODE ? "SKB" : "DRV"
116 (ksft_test_result_pass("PASS: %s %s %s%s%s%s\n", configured_mode ? "DRV" : "SKB",\120
117 test_type == TEST_TYPE_POLL ? "POLL" : "NOPOLL",\121#define print_ksft_result(test) \
118 test_type == TEST_TYPE_TEARDOWN ? "Socket Teardown" : "",\122 (ksft_test_result_pass("PASS: %s %s\n", mode_string(test), (test)->name))
119 test_type == TEST_TYPE_BIDI ? "Bi-directional Sockets" : "",\
120 test_type == TEST_TYPE_STATS ? "Stats" : "",\
121 test_type == TEST_TYPE_BPF_RES ? "BPF RES" : ""))
122 123
123static void memset32_htonl(void *dest, u32 val, u32 size)124static void memset32_htonl(void *dest, u32 val, u32 size)
124{125{
@@ -235,80 +236,46 @@ static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr) · 110 unmodified lines
235 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);236 udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, IPPROTO_UDP, (u16 *)udp_hdr);
236}237}
237 238
238static void xsk_configure_umem(struct ifobject *data, void *buffer, u64 size, int idx)239static int xsk_configure_umem(struct xsk_umem_info *umem, void *buffer, u64 size)
239{240{
240 struct xsk_umem_config cfg = {241 struct xsk_umem_config cfg = {
241 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,242 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
242 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,243 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
243 .frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE,244 .frame_size = umem->frame_size,
244 .frame_headroom = frame_headroom,245 .frame_headroom = umem->frame_headroom,
245 .flags = XSK_UMEM__DEFAULT_FLAGS246 .flags = XSK_UMEM__DEFAULT_FLAGS
246 };247 };
247 struct xsk_umem_info *umem;
248 int ret;248 int ret;
249 249
250 umem = calloc(1, sizeof(struct xsk_umem_info));250 if (umem->unaligned_mode)
251 if (!umem)251 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
252 exit_with_error(errno);
253 252
254 ret = xsk_umem__create(&umem->umem, buffer, size,253 ret = xsk_umem__create(&umem->umem, buffer, size,
255 &umem->fq, &umem->cq, &cfg);254 &umem->fq, &umem->cq, &cfg);
256 if (ret)255 if (ret)
257 exit_with_error(-ret);256 return ret;
258 257
259 umem->buffer = buffer;258 umem->buffer = buffer;
260 259 return 0;
261 data->umem_arr[idx] = umem;
262}
263
264static void xsk_populate_fill_ring(struct xsk_umem_info *umem)
265{
266 int ret, i;
267 u32 idx = 0;
268
269 ret = xsk_ring_prod__reserve(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
270 if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
271 exit_with_error(-ret);
272 for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
273 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = i * XSK_UMEM__DEFAULT_FRAME_SIZE;
274 xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
275}260}
276 261
277static int xsk_configure_socket(struct ifobject *ifobject, int idx)262static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
263 struct ifobject *ifobject, u32 qid)
278{264{
279 struct xsk_socket_config cfg;265 struct xsk_socket_config cfg;
280 struct xsk_socket_info *xsk;
281 struct xsk_ring_cons *rxr;266 struct xsk_ring_cons *rxr;
282 struct xsk_ring_prod *txr;267 struct xsk_ring_prod *txr;
283 int ret;
284 268
285 xsk = calloc(1, sizeof(struct xsk_socket_info));269 xsk->umem = umem;
286 if (!xsk)270 cfg.rx_size = xsk->rxqsize;
287 exit_with_error(errno);
288
289 xsk->umem = ifobject->umem;
290 cfg.rx_size = rxqsize;
291 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;271 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
292 cfg.libbpf_flags = 0;272 cfg.libbpf_flags = 0;
293 cfg.xdp_flags = xdp_flags;273 cfg.xdp_flags = ifobject->xdp_flags;
294 cfg.bind_flags = xdp_bind_flags;274 cfg.bind_flags = ifobject->bind_flags;
295
296 if (test_type != TEST_TYPE_BIDI) {
297 rxr = (ifobject->fv.vector == rx) ? &xsk->rx : NULL;
298 txr = (ifobject->fv.vector == tx) ? &xsk->tx : NULL;
299 } else {
300 rxr = &xsk->rx;
301 txr = &xsk->tx;
302 }
303
304 ret = xsk_socket__create(&xsk->xsk, ifobject->ifname, idx,
305 ifobject->umem->umem, rxr, txr, &cfg);
306 if (ret)
307 return 1;
308 275
309 ifobject->xsk_arr[idx] = xsk;276 txr = ifobject->tx_on ? &xsk->tx : NULL;
310 277 rxr = ifobject->rx_on ? &xsk->rx : NULL;
311 return 0;278 return xsk_socket__create(&xsk->xsk, ifobject->ifname, qid, umem->umem, rxr, txr, &cfg);
312}279}
313 280
314static struct option long_options[] = {281static struct option long_options[] = {
@@ -354,45 +321,44 @@ static int switch_namespace(const char *nsname) · 39 unmodified lines
354 return nsfd;321 return nsfd;
355}322}
356 323
357static int validate_interfaces(void)324static bool validate_interface(struct ifobject *ifobj)
358{325{
359 bool ret = true;326 if (!strcmp(ifobj->ifname, ""))
360 327 return false;
361 for (int i = 0; i < MAX_INTERFACES; i++) {328 return true;
362 if (!strcmp(ifdict[i]->ifname, "")) {
363 ret = false;
364 ksft_test_result_fail("ERROR: interfaces: -i <int>,<ns> -i <int>,<ns>.");
365 }
366 }
367 return ret;
368}329}
369 330
370static void parse_command_line(int argc, char **argv)331static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
332 char **argv)
371{333{
372 int option_index, interface_index = 0, c;334 struct ifobject *ifobj;
335 u32 interface_nb = 0;
336 int option_index, c;
373 337
374 opterr = 0;338 opterr = 0;
375 339
376 for (;;) {340 for (;;) {
377 c = getopt_long(argc, argv, "i:Dv", long_options, &option_index);341 char *sptr, *token;
378 342
343 c = getopt_long(argc, argv, "i:Dv", long_options, &option_index);
379 if (c == -1)344 if (c == -1)
380 break;345 break;
381 346
382 switch (c) {347 switch (c) {
383 case 'i':348 case 'i':
384 if (interface_index == MAX_INTERFACES)349 if (interface_nb == 0)
350 ifobj = ifobj_tx;
351 else if (interface_nb == 1)
352 ifobj = ifobj_rx;
353 else
385 break;354 break;
386 char *sptr, *token;
387 355
388 sptr = strndupa(optarg, strlen(optarg));356 sptr = strndupa(optarg, strlen(optarg));
389 memcpy(ifdict[interface_index]->ifname,357 memcpy(ifobj->ifname, strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS);
390 strsep(&sptr, ","), MAX_INTERFACE_NAME_CHARS);
391 token = strsep(&sptr, ",");358 token = strsep(&sptr, ",");
392 if (token)359 if (token)
393 memcpy(ifdict[interface_index]->nsname, token,360 memcpy(ifobj->nsname, token, MAX_INTERFACES_NAMESPACE_CHARS);
394 MAX_INTERFACES_NAMESPACE_CHARS);361 interface_nb++;
395 interface_index++;
396 break;362 break;
397 case 'D':363 case 'D':
398 opt_pkt_dump = true;364 opt_pkt_dump = true;
@@ -405,11 +371,78 @@ static void parse_command_line(int argc, char **argv) · 6 unmodified lines
405 ksft_exit_xfail();371 ksft_exit_xfail();
406 }372 }
407 }373 }
374}
408 375
409 if (!validate_interfaces()) {376static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
410 usage(basename(argv[0]));377 struct ifobject *ifobj_rx)
411 ksft_exit_xfail();378{
379 u32 i, j;
380
381 for (i = 0; i < MAX_INTERFACES; i++) {
382 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
383
384 ifobj->umem = &ifobj->umem_arr[0];
385 ifobj->xsk = &ifobj->xsk_arr[0];
386 ifobj->use_poll = false;
387 ifobj->pkt_stream = test->pkt_stream_default;
388
389 if (i == 0) {
390 ifobj->rx_on = false;
391 ifobj->tx_on = true;
392 } else {
393 ifobj->rx_on = true;
394 ifobj->tx_on = false;
395 }
396
397 for (j = 0; j < MAX_SOCKETS; j++) {
398 memset(&ifobj->umem_arr[j], 0, sizeof(ifobj->umem_arr[j]));
399 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
400 ifobj->umem_arr[j].num_frames = DEFAULT_UMEM_BUFFERS;
401 ifobj->umem_arr[j].frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
402 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
403 }
412 }404 }
405
406 test->ifobj_tx = ifobj_tx;
407 test->ifobj_rx = ifobj_rx;
408 test->current_step = 0;
409 test->total_steps = 1;
410 test->nb_sockets = 1;
411}
412
413static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
414 struct ifobject *ifobj_rx, enum test_mode mode)
415{
416 struct pkt_stream *pkt_stream;
417 u32 i;
418
419 pkt_stream = test->pkt_stream_default;
420 memset(test, 0, sizeof(*test));
421 test->pkt_stream_default = pkt_stream;
422
423 for (i = 0; i < MAX_INTERFACES; i++) {
424 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
425
426 ifobj->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
427 if (mode == TEST_MODE_SKB)
428 ifobj->xdp_flags |= XDP_FLAGS_SKB_MODE;
429 else
430 ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE;
431
432 ifobj->bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY;
433 }
434
435 __test_spec_init(test, ifobj_tx, ifobj_rx);
436}
437
438static void test_spec_reset(struct test_spec *test)
439{
440 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
441}
442
443static void test_spec_set_name(struct test_spec *test, const char *name)
444{
445 strncpy(test->name, name, MAX_TEST_NAME_SIZE);
413}446}
414 447
415static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb)448static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb)
@@ -420,29 +453,105 @@ static struct pkt *pkt_stream_get_pkt(struct pkt_stream *pkt_stream, u32 pkt_nb) · 4 unmodified lines
420 return &pkt_stream->pkts[pkt_nb];453 return &pkt_stream->pkts[pkt_nb];
421}454}
422 455
423static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len)456static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream)
457{
458 while (pkt_stream->rx_pkt_nb < pkt_stream->nb_pkts) {
459 if (pkt_stream->pkts[pkt_stream->rx_pkt_nb].valid)
460 return &pkt_stream->pkts[pkt_stream->rx_pkt_nb++];
461 pkt_stream->rx_pkt_nb++;
462 }
463 return NULL;
464}
465
466static void pkt_stream_delete(struct pkt_stream *pkt_stream)
467{
468 free(pkt_stream->pkts);
469 free(pkt_stream);
470}
471
472static void pkt_stream_restore_default(struct test_spec *test)
473{
474 if (test->ifobj_tx->pkt_stream != test->pkt_stream_default) {
475 pkt_stream_delete(test->ifobj_tx->pkt_stream);
476 test->ifobj_tx->pkt_stream = test->pkt_stream_default;
477 }
478 test->ifobj_rx->pkt_stream = test->pkt_stream_default;
479}
480
481static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
424{482{
425 struct pkt_stream *pkt_stream;483 struct pkt_stream *pkt_stream;
426 u32 i;
427 484
428 pkt_stream = malloc(sizeof(*pkt_stream));485 pkt_stream = calloc(1, sizeof(*pkt_stream));
429 if (!pkt_stream)486 if (!pkt_stream)
430 exit_with_error(ENOMEM);487 return NULL;
431 488
432 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));489 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
433 if (!pkt_stream->pkts)490 if (!pkt_stream->pkts) {
491 free(pkt_stream);
492 return NULL;
493 }
494
495 pkt_stream->nb_pkts = nb_pkts;
496 return pkt_stream;
497}
498
499static struct pkt_stream *pkt_stream_generate(struct xsk_umem_info *umem, u32 nb_pkts, u32 pkt_len)
500{
501 struct pkt_stream *pkt_stream;
502 u32 i;
503
504 pkt_stream = __pkt_stream_alloc(nb_pkts);
505 if (!pkt_stream)
434 exit_with_error(ENOMEM);506 exit_with_error(ENOMEM);
435 507
436 pkt_stream->nb_pkts = nb_pkts;508 pkt_stream->nb_pkts = nb_pkts;
437 for (i = 0; i < nb_pkts; i++) {509 for (i = 0; i < nb_pkts; i++) {
438 pkt_stream->pkts[i].addr = (i % num_frames) * XSK_UMEM__DEFAULT_FRAME_SIZE;510 pkt_stream->pkts[i].addr = (i % umem->num_frames) * umem->frame_size +
511 DEFAULT_OFFSET;
439 pkt_stream->pkts[i].len = pkt_len;512 pkt_stream->pkts[i].len = pkt_len;
440 pkt_stream->pkts[i].payload = i;513 pkt_stream->pkts[i].payload = i;
514
515 if (pkt_len > umem->frame_size)
516 pkt_stream->pkts[i].valid = false;
517 else
518 pkt_stream->pkts[i].valid = true;
441 }519 }
442 520
443 return pkt_stream;521 return pkt_stream;
444}522}
445 523
524static struct pkt_stream *pkt_stream_clone(struct xsk_umem_info *umem,
525 struct pkt_stream *pkt_stream)
526{
527 return pkt_stream_generate(umem, pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
528}
529
530static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
531{
532 struct pkt_stream *pkt_stream;
533
534 pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
535 test->ifobj_tx->pkt_stream = pkt_stream;
536 test->ifobj_rx->pkt_stream = pkt_stream;
537}
538
539static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, u32 offset)
540{
541 struct xsk_umem_info *umem = test->ifobj_tx->umem;
542 struct pkt_stream *pkt_stream;
543 u32 i;
544
545 pkt_stream = pkt_stream_clone(umem, test->pkt_stream_default);
546 for (i = 0; i < test->pkt_stream_default->nb_pkts; i += 2) {
547 pkt_stream->pkts[i].addr = (i % umem->num_frames) * umem->frame_size + offset;
548 pkt_stream->pkts[i].len = pkt_len;
549 }
550
551 test->ifobj_tx->pkt_stream = pkt_stream;
552 test->ifobj_rx->pkt_stream = pkt_stream;
553}
554
446static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)555static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)
447{556{
448 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);557 struct pkt *pkt = pkt_stream_get_pkt(ifobject->pkt_stream, pkt_nb);
@@ -453,6 +562,8 @@ static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb) · 4 unmodified lines
453 562
454 if (!pkt)563 if (!pkt)
455 return NULL;564 return NULL;
565 if (!pkt->valid || pkt->len < PKT_SIZE)
566 return pkt;
456 567
457 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);568 data = xsk_umem__get_data(ifobject->umem->buffer, pkt->addr);
458 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr));569 udp_hdr = (struct udphdr *)(data + sizeof(struct ethhdr) + sizeof(struct iphdr));
@@ -467,6 +578,26 @@ static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb) · 8 unmodified lines
467 return pkt;578 return pkt;
468}579}
469 580
581static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
582{
583 struct pkt_stream *pkt_stream;
584 u32 i;
585
586 pkt_stream = __pkt_stream_alloc(nb_pkts);
587 if (!pkt_stream)
588 exit_with_error(ENOMEM);
589
590 test->ifobj_tx->pkt_stream = pkt_stream;
591 test->ifobj_rx->pkt_stream = pkt_stream;
592
593 for (i = 0; i < nb_pkts; i++) {
594 pkt_stream->pkts[i].addr = pkts[i].addr;
595 pkt_stream->pkts[i].len = pkts[i].len;
596 pkt_stream->pkts[i].payload = i;
597 pkt_stream->pkts[i].valid = pkts[i].valid;
598 }
599}
600
470static void pkt_dump(void *pkt, u32 len)601static void pkt_dump(void *pkt, u32 len)
471{602{
472 char s[INET_ADDRSTRLEN];603 char s[INET_ADDRSTRLEN];
@@ -504,9 +635,9 @@ static void pkt_dump(void *pkt, u32 len) · 31 unmodified lines
504 fprintf(stdout, "---------------------------------------\n");635 fprintf(stdout, "---------------------------------------\n");
505}636}
506 637
507static bool is_pkt_valid(struct pkt *pkt, void *buffer, const struct xdp_desc *desc)638static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
508{639{
509 void *data = xsk_umem__get_data(buffer, desc->addr);640 void *data = xsk_umem__get_data(buffer, addr);
510 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr));641 struct iphdr *iphdr = (struct iphdr *)(data + sizeof(struct ethhdr));
511 642
512 if (!pkt) {643 if (!pkt) {
@@ -514,19 +645,24 @@ static bool is_pkt_valid(struct pkt *pkt, void *buffer, const struct xdp_desc *d · 1 unmodified line
514 return false;645 return false;
515 }646 }
516 647
648 if (len < PKT_SIZE) {
649 /*Do not try to verify packets that are smaller than minimum size. */
650 return true;
651 }
652
653 if (pkt->len != len) {
654 ksft_test_result_fail
655 ("ERROR: [%s] expected length [%d], got length [%d]\n",
656 __func__, pkt->len, len);
657 return false;
658 }
659
517 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {660 if (iphdr->version == IP_PKT_VER && iphdr->tos == IP_PKT_TOS) {
518 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE)));661 u32 seqnum = ntohl(*((u32 *)(data + PKT_HDR_SIZE)));
519 662
520 if (opt_pkt_dump && test_type != TEST_TYPE_STATS)663 if (opt_pkt_dump)
521 pkt_dump(data, PKT_SIZE);664 pkt_dump(data, PKT_SIZE);
522 665
523 if (pkt->len != desc->len) {
524 ksft_test_result_fail
525 ("ERROR: [%s] expected length [%d], got length [%d]\n",
526 __func__, pkt->len, desc->len);
527 return false;
528 }
529
530 if (pkt->payload != seqnum) {666 if (pkt->payload != seqnum) {
531 ksft_test_result_fail667 ksft_test_result_fail
532 ("ERROR: [%s] expected seqnum [%d], got seqnum [%d]\n",668 ("ERROR: [%s] expected seqnum [%d], got seqnum [%d]\n",
@@ -558,14 +694,20 @@ static void complete_pkts(struct xsk_socket_info *xsk, int batch_size) · 25 unmodified lines
558 unsigned int rcvd;694 unsigned int rcvd;
559 u32 idx;695 u32 idx;
560 696
561 if (!xsk->outstanding_tx)
562 return;
563
564 if (xsk_ring_prod__needs_wakeup(&xsk->tx))697 if (xsk_ring_prod__needs_wakeup(&xsk->tx))
565 kick_tx(xsk);698 kick_tx(xsk);
566 699
567 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);700 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
568 if (rcvd) {701 if (rcvd) {
702 if (rcvd > xsk->outstanding_tx) {
703 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
704
705 ksft_test_result_fail("ERROR: [%s] Too many packets completed\n",
706 __func__);
707 ksft_print_msg("Last completion address: %llx\n", addr);
708 return;
709 }
710
569 xsk_ring_cons__release(&xsk->umem->cq, rcvd);711 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
570 xsk->outstanding_tx -= rcvd;712 xsk->outstanding_tx -= rcvd;
571 }713 }
@@ -574,11 +716,10 @@ static void complete_pkts(struct xsk_socket_info *xsk, int batch_size) · 2 unmodified lines
574static void receive_pkts(struct pkt_stream *pkt_stream, struct xsk_socket_info *xsk,716static void receive_pkts(struct pkt_stream *pkt_stream, struct xsk_socket_info *xsk,
575 struct pollfd *fds)717 struct pollfd *fds)
576{718{
577 u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkt_count = 0;719 struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream);
578 struct pkt *pkt;720 u32 idx_rx = 0, idx_fq = 0, rcvd, i;
579 int ret;721 int ret;
580 722
581 pkt = pkt_stream_get_pkt(pkt_stream, pkt_count++);
582 while (pkt) {723 while (pkt) {
583 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);724 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
584 if (!rcvd) {725 if (!rcvd) {
@@ -606,13 +747,21 @@ static void receive_pkts(struct pkt_stream *pkt_stream, struct xsk_socket_info * · 21 unmodified lines
606 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);747 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
607 u64 addr = desc->addr, orig;748 u64 addr = desc->addr, orig;
608 749
750 if (!pkt) {
751 ksft_test_result_fail("ERROR: [%s] Received too many packets.\n",
752 __func__);
753 ksft_print_msg("Last packet has addr: %llx len: %u\n",
754 addr, desc->len);
755 return;
756 }
757
609 orig = xsk_umem__extract_addr(addr);758 orig = xsk_umem__extract_addr(addr);
610 addr = xsk_umem__add_offset_to_addr(addr);759 addr = xsk_umem__add_offset_to_addr(addr);
611 if (!is_pkt_valid(pkt, xsk->umem->buffer, desc))760 if (!is_pkt_valid(pkt, xsk->umem->buffer, addr, desc->len))
612 return;761 return;
613 762
614 *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig;763 *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig;
615 pkt = pkt_stream_get_pkt(pkt_stream, pkt_count++);764 pkt = pkt_stream_get_next_rx_pkt(pkt_stream);
616 }765 }
617 766
618 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);767 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
@@ -623,7 +772,7 @@ static void receive_pkts(struct pkt_stream *pkt_stream, struct xsk_socket_info * · 4 unmodified lines
623static u32 __send_pkts(struct ifobject *ifobject, u32 pkt_nb)772static u32 __send_pkts(struct ifobject *ifobject, u32 pkt_nb)
624{773{
625 struct xsk_socket_info *xsk = ifobject->xsk;774 struct xsk_socket_info *xsk = ifobject->xsk;
626 u32 i, idx;775 u32 i, idx, valid_pkts = 0;
627 776
628 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE)777 while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE)
629 complete_pkts(xsk, BATCH_SIZE);778 complete_pkts(xsk, BATCH_SIZE);
@@ -638,14 +787,13 @@ static u32 __send_pkts(struct ifobject *ifobject, u32 pkt_nb) · 8 unmodified lines
638 tx_desc->addr = pkt->addr;787 tx_desc->addr = pkt->addr;
639 tx_desc->len = pkt->len;788 tx_desc->len = pkt->len;
640 pkt_nb++;789 pkt_nb++;
790 if (pkt->valid)
791 valid_pkts++;
641 }792 }
642 793
643 xsk_ring_prod__submit(&xsk->tx, i);794 xsk_ring_prod__submit(&xsk->tx, i);
644 if (stat_test_type != STAT_TEST_TX_INVALID)795 xsk->outstanding_tx += valid_pkts;
645 xsk->outstanding_tx += i;796 complete_pkts(xsk, BATCH_SIZE);
646 else if (xsk_ring_prod__needs_wakeup(&xsk->tx))
647 kick_tx(xsk);
648 complete_pkts(xsk, i);
649 797
650 return i;798 return i;
651}799}
@@ -658,23 +806,23 @@ static void wait_for_tx_completion(struct xsk_socket_info *xsk) · 6 unmodified lines
658 806
659static void send_pkts(struct ifobject *ifobject)807static void send_pkts(struct ifobject *ifobject)
660{808{
661 struct pollfd fds[MAX_SOCKS] = { };809 struct pollfd fds = { };
662 u32 pkt_cnt = 0;810 u32 pkt_cnt = 0;
663 811
664 fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);812 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
665 fds[0].events = POLLOUT;813 fds.events = POLLOUT;
666 814
667 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {815 while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {
668 u32 sent;816 u32 sent;
669 817
670 if (test_type == TEST_TYPE_POLL) {818 if (ifobject->use_poll) {
671 int ret;819 int ret;
672 820
673 ret = poll(fds, 1, POLL_TMOUT);821 ret = poll(&fds, 1, POLL_TMOUT);
674 if (ret <= 0)822 if (ret <= 0)
675 continue;823 continue;
676 824
677 if (!(fds[0].revents & POLLOUT))825 if (!(fds.revents & POLLOUT))
678 continue;826 continue;
679 }827 }
680 828
@@ -698,7 +846,7 @@ static bool rx_stats_are_valid(struct ifobject *ifobject) · 17 unmodified lines
698 optlen = sizeof(stats);846 optlen = sizeof(stats);
699 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);847 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
700 if (err) {848 if (err) {
701 ksft_test_result_fail("ERROR: [%s] getsockopt(XDP_STATISTICS) error %u %s\n",849 ksft_test_result_fail("ERROR Rx: [%s] getsockopt(XDP_STATISTICS) error %u %s\n",
702 __func__, -err, strerror(-err));850 __func__, -err, strerror(-err));
703 return true;851 return true;
704 }852 }
@@ -739,7 +887,7 @@ static void tx_stats_validate(struct ifobject *ifobject) · 34 unmodified lines
739 optlen = sizeof(stats);887 optlen = sizeof(stats);
740 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);888 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
741 if (err) {889 if (err) {
742 ksft_test_result_fail("ERROR: [%s] getsockopt(XDP_STATISTICS) error %u %s\n",890 ksft_test_result_fail("ERROR Tx: [%s] getsockopt(XDP_STATISTICS) error %u %s\n",
743 __func__, -err, strerror(-err));891 __func__, -err, strerror(-err));
744 return;892 return;
745 }893 }
@@ -751,71 +899,63 @@ static void tx_stats_validate(struct ifobject *ifobject) · 5 unmodified lines
751 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);899 __func__, stats.tx_invalid_descs, ifobject->pkt_stream->nb_pkts);
752}900}
753 901
754static void thread_common_ops(struct ifobject *ifobject, void *bufs)902static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
755{903{
756 u64 umem_sz = num_frames * XSK_UMEM__DEFAULT_FRAME_SIZE;
757 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;904 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
758 size_t mmap_sz = umem_sz;905 u32 i;
759 int ctr = 0;
760 int ret;
761 906
762 ifobject->ns_fd = switch_namespace(ifobject->nsname);907 ifobject->ns_fd = switch_namespace(ifobject->nsname);
763 908
764 if (test_type == TEST_TYPE_BPF_RES)909 if (ifobject->umem->unaligned_mode)
765 mmap_sz *= 2;910 mmap_flags |= MAP_HUGETLB;
766 911
767 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);912 for (i = 0; i < test->nb_sockets; i++) {
768 if (bufs == MAP_FAILED)913 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
769 exit_with_error(errno);914 u32 ctr = 0;
915 void *bufs;
770 916
771 while (ctr++ < SOCK_RECONF_CTR) {917 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
772 xsk_configure_umem(ifobject, bufs, umem_sz, 0);918 if (bufs == MAP_FAILED)
773 ifobject->umem = ifobject->umem_arr[0];919 exit_with_error(errno);
774 ret = xsk_configure_socket(ifobject, 0);
775 if (!ret)
776 break;
777 920
778 /* Retry Create Socket if it fails as xsk_socket__create() is asynchronous */921 while (ctr++ < SOCK_RECONF_CTR) {
779 usleep(USLEEP_MAX);922 int ret;
780 if (ctr >= SOCK_RECONF_CTR)
781 exit_with_error(-ret);
782 }
783 923
784 ifobject->umem = ifobject->umem_arr[0];924 ret = xsk_configure_umem(&ifobject->umem_arr[i], bufs, umem_sz);
785 ifobject->xsk = ifobject->xsk_arr[0];925 if (ret)
926 exit_with_error(-ret);
786 927
787 if (test_type == TEST_TYPE_BPF_RES) {928 ret = xsk_configure_socket(&ifobject->xsk_arr[i], &ifobject->umem_arr[i],
788 xsk_configure_umem(ifobject, (u8 *)bufs + umem_sz, umem_sz, 1);929 ifobject, i);
789 ifobject->umem = ifobject->umem_arr[1];930 if (!ret)
790 ret = xsk_configure_socket(ifobject, 1);931 break;
791 }
792 932
793 ifobject->umem = ifobject->umem_arr[0];933 /* Retry if it fails as xsk_socket__create() is asynchronous */
794 ifobject->xsk = ifobject->xsk_arr[0];934 if (ctr >= SOCK_RECONF_CTR)
795 print_verbose("Interface [%s] vector [%s]\n",935 exit_with_error(-ret);
796 ifobject->ifname, ifobject->fv.vector == tx ? "Tx" : "Rx");936 usleep(USLEEP_MAX);
797}937 }
938 }
798 939
799static bool testapp_is_test_two_stepped(void)940 ifobject->umem = &ifobject->umem_arr[0];
800{941 ifobject->xsk = &ifobject->xsk_arr[0];
801 return (test_type != TEST_TYPE_BIDI && test_type != TEST_TYPE_BPF_RES) || second_step;
802}942}
803 943
804static void testapp_cleanup_xsk_res(struct ifobject *ifobj)944static void testapp_cleanup_xsk_res(struct ifobject *ifobj)
805{945{
806 if (testapp_is_test_two_stepped()) {946 print_verbose("Destroying socket\n");
807 xsk_socket__delete(ifobj->xsk->xsk);947 xsk_socket__delete(ifobj->xsk->xsk);
808 (void)xsk_umem__delete(ifobj->umem->umem);948 munmap(ifobj->umem->buffer, ifobj->umem->num_frames * ifobj->umem->frame_size);
809 }949 xsk_umem__delete(ifobj->umem->umem);
810}950}
811 951
812static void *worker_testapp_validate_tx(void *arg)952static void *worker_testapp_validate_tx(void *arg)
813{953{
814 struct ifobject *ifobject = (struct ifobject *)arg;954 struct test_spec *test = (struct test_spec *)arg;
815 void *bufs = NULL;955 struct ifobject *ifobject = test->ifobj_tx;
816 956
817 if (!second_step)957 if (test->current_step == 1)
818 thread_common_ops(ifobject, bufs);958 thread_common_ops(test, ifobject);
819 959
820 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,960 print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
821 ifobject->ifname);961 ifobject->ifname);
@@ -824,24 +964,50 @@ static void *worker_testapp_validate_tx(void *arg) · 2 unmodified lines
824 if (stat_test_type == STAT_TEST_TX_INVALID)964 if (stat_test_type == STAT_TEST_TX_INVALID)
825 tx_stats_validate(ifobject);965 tx_stats_validate(ifobject);
826 966
827 testapp_cleanup_xsk_res(ifobject);967 if (test->total_steps == test->current_step)
968 testapp_cleanup_xsk_res(ifobject);
828 pthread_exit(NULL);969 pthread_exit(NULL);
829}970}
830 971
972static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
973{
974 u32 idx = 0, i;
975 int ret;
976
977 ret = xsk_ring_prod__reserve(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
978 if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
979 exit_with_error(ENOSPC);
980 for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++) {
981 u64 addr;
982
983 if (pkt_stream->use_addr_for_fill) {
984 struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
985
986 if (!pkt)
987 break;
988 addr = pkt->addr;
989 } else {
990 addr = (i % umem->num_frames) * umem->frame_size + DEFAULT_OFFSET;
991 }
992
993 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
994 }
995 xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
996}
997
831static void *worker_testapp_validate_rx(void *arg)998static void *worker_testapp_validate_rx(void *arg)
832{999{
833 struct ifobject *ifobject = (struct ifobject *)arg;1000 struct test_spec *test = (struct test_spec *)arg;
834 struct pollfd fds[MAX_SOCKS] = { };1001 struct ifobject *ifobject = test->ifobj_rx;
835 void *bufs = NULL;1002 struct pollfd fds = { };
836 1003
837 if (!second_step)1004 if (test->current_step == 1)
838 thread_common_ops(ifobject, bufs);1005 thread_common_ops(test, ifobject);
839 1006
840 if (stat_test_type != STAT_TEST_RX_FILL_EMPTY)1007 xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
841 xsk_populate_fill_ring(ifobject->umem);
842 1008
843 fds[0].fd = xsk_socket__fd(ifobject->xsk->xsk);1009 fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
844 fds[0].events = POLLIN;1010 fds.events = POLLIN;
845 1011
846 pthread_barrier_wait(&barr);1012 pthread_barrier_wait(&barr);
847 1013
@@ -849,151 +1015,219 @@ static void *worker_testapp_validate_rx(void *arg) · 1 unmodified line
849 while (!rx_stats_are_valid(ifobject))1015 while (!rx_stats_are_valid(ifobject))
850 continue;1016 continue;
851 else1017 else
852 receive_pkts(ifobject->pkt_stream, ifobject->xsk, fds);1018 receive_pkts(ifobject->pkt_stream, ifobject->xsk, &fds);
853
854 if (test_type == TEST_TYPE_TEARDOWN)
855 print_verbose("Destroying socket\n");
856 1019
857 testapp_cleanup_xsk_res(ifobject);1020 if (test->total_steps == test->current_step)
1021 testapp_cleanup_xsk_res(ifobject);
858 pthread_exit(NULL);1022 pthread_exit(NULL);
859}1023}
860 1024
861static void testapp_validate(void)1025static void testapp_validate_traffic(struct test_spec *test)
862{1026{
863 bool bidi = test_type == TEST_TYPE_BIDI;1027 struct ifobject *ifobj_tx = test->ifobj_tx;
864 bool bpf = test_type == TEST_TYPE_BPF_RES;1028 struct ifobject *ifobj_rx = test->ifobj_rx;
865 struct pkt_stream *pkt_stream;1029 pthread_t t0, t1;
866 1030
867 if (pthread_barrier_init(&barr, NULL, 2))1031 if (pthread_barrier_init(&barr, NULL, 2))
868 exit_with_error(errno);1032 exit_with_error(errno);
869 1033
870 if (stat_test_type == STAT_TEST_TX_INVALID)1034 test->current_step++;
871 pkt_stream = pkt_stream_generate(DEFAULT_PKT_CNT, XSK_UMEM__INVALID_FRAME_SIZE);
872 else
873 pkt_stream = pkt_stream_generate(DEFAULT_PKT_CNT, PKT_SIZE);
874 ifdict_tx->pkt_stream = pkt_stream;
875 ifdict_rx->pkt_stream = pkt_stream;
876 1035
877 /*Spawn RX thread */1036 /*Spawn RX thread */
878 pthread_create(&t0, NULL, ifdict_rx->func_ptr, ifdict_rx);1037 pthread_create(&t0, NULL, ifobj_rx->func_ptr, test);
879 1038
880 pthread_barrier_wait(&barr);1039 pthread_barrier_wait(&barr);
881 if (pthread_barrier_destroy(&barr))1040 if (pthread_barrier_destroy(&barr))
882 exit_with_error(errno);1041 exit_with_error(errno);
883 1042
884 /*Spawn TX thread */1043 /*Spawn TX thread */
885 pthread_create(&t1, NULL, ifdict_tx->func_ptr, ifdict_tx);1044 pthread_create(&t1, NULL, ifobj_tx->func_ptr, test);
886 1045
887 pthread_join(t1, NULL);1046 pthread_join(t1, NULL);
888 pthread_join(t0, NULL);1047 pthread_join(t0, NULL);
889
890 if (!(test_type == TEST_TYPE_TEARDOWN) && !bidi && !bpf && !(test_type == TEST_TYPE_STATS))
891 print_ksft_result();
892}1048}
893 1049
894static void testapp_teardown(void)1050static void testapp_teardown(struct test_spec *test)
895{1051{
896 int i;1052 int i;
897 1053
1054 test_spec_set_name(test, "TEARDOWN");
898 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {1055 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
899 print_verbose("Creating socket\n");1056 testapp_validate_traffic(test);
900 testapp_validate();1057 test_spec_reset(test);
901 }1058 }
902
903 print_ksft_result();
904}1059}
905 1060
906static void swap_vectors(struct ifobject *ifobj1, struct ifobject *ifobj2)1061static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
907{1062{
908 void *(*tmp_func_ptr)(void *) = ifobj1->func_ptr;1063 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
909 enum fvector tmp_vector = ifobj1->fv.vector;1064 struct ifobject *tmp_ifobj = (*ifobj1);
910 1065
911 ifobj1->func_ptr = ifobj2->func_ptr;1066 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
912 ifobj1->fv.vector = ifobj2->fv.vector;1067 (*ifobj2)->func_ptr = tmp_func_ptr;
913 1068
914 ifobj2->func_ptr = tmp_func_ptr;1069 *ifobj1 = *ifobj2;
915 ifobj2->fv.vector = tmp_vector;1070 *ifobj2 = tmp_ifobj;
916
917 ifdict_tx = ifobj1;
918 ifdict_rx = ifobj2;
919}1071}
920 1072
921static void testapp_bidi(void)1073static void testapp_bidi(struct test_spec *test)
922{1074{
923 for (int i = 0; i < MAX_BIDI_ITER; i++) {1075 test_spec_set_name(test, "BIDIRECTIONAL");
924 print_verbose("Creating socket\n");1076 test->ifobj_tx->rx_on = true;
925 testapp_validate();1077 test->ifobj_rx->tx_on = true;
926 if (!second_step) {1078 test->total_steps = 2;
927 print_verbose("Switching Tx/Rx vectors\n");1079 testapp_validate_traffic(test);
928 swap_vectors(ifdict[1], ifdict[0]);
929 }
930 second_step = true;
931 }
932 1080
933 swap_vectors(ifdict[0], ifdict[1]);1081 print_verbose("Switching Tx/Rx vectors\n");
1082 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
1083 testapp_validate_traffic(test);
934 1084
935 print_ksft_result();1085 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
936}1086}
937 1087
938static void swap_xsk_res(void)1088static void swap_xsk_resources(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx)
939{1089{
940 xsk_socket__delete(ifdict_tx->xsk->xsk);1090 xsk_socket__delete(ifobj_tx->xsk->xsk);
941 xsk_umem__delete(ifdict_tx->umem->umem);1091 xsk_umem__delete(ifobj_tx->umem->umem);
942 xsk_socket__delete(ifdict_rx->xsk->xsk);1092 xsk_socket__delete(ifobj_rx->xsk->xsk);
943 xsk_umem__delete(ifdict_rx->umem->umem);1093 xsk_umem__delete(ifobj_rx->umem->umem);
944 ifdict_tx->umem = ifdict_tx->umem_arr[1];1094 ifobj_tx->umem = &ifobj_tx->umem_arr[1];
945 ifdict_tx->xsk = ifdict_tx->xsk_arr[1];1095 ifobj_tx->xsk = &ifobj_tx->xsk_arr[1];
946 ifdict_rx->umem = ifdict_rx->umem_arr[1];1096 ifobj_rx->umem = &ifobj_rx->umem_arr[1];
947 ifdict_rx->xsk = ifdict_rx->xsk_arr[1];1097 ifobj_rx->xsk = &ifobj_rx->xsk_arr[1];
948}1098}
949 1099
950static void testapp_bpf_res(void)1100static void testapp_bpf_res(struct test_spec *test)
951{1101{
952 int i;1102 test_spec_set_name(test, "BPF_RES");
1103 test->total_steps = 2;
1104 test->nb_sockets = 2;
1105 testapp_validate_traffic(test);
953 1106
954 for (i = 0; i < MAX_BPF_ITER; i++) {1107 swap_xsk_resources(test->ifobj_tx, test->ifobj_rx);
955 print_verbose("Creating socket\n");1108 testapp_validate_traffic(test);
956 testapp_validate();
957 if (!second_step)
958 swap_xsk_res();
959 second_step = true;
960 }
961
962 print_ksft_result();
963}1109}
964 1110
965static void testapp_stats(void)1111static void testapp_stats(struct test_spec *test)
966{1112{
967 for (int i = 0; i < STAT_TEST_TYPE_MAX; i++) {1113 int i;
968 stat_test_type = i;
969 1114
970 /* reset defaults */1115 for (i = 0; i < STAT_TEST_TYPE_MAX; i++) {
971 rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;1116 test_spec_reset(test);
972 frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;1117 stat_test_type = i;
973 1118
974 switch (stat_test_type) {1119 switch (stat_test_type) {
975 case STAT_TEST_RX_DROPPED:1120 case STAT_TEST_RX_DROPPED:
976 frame_headroom = XSK_UMEM__DEFAULT_FRAME_SIZE -1121 test_spec_set_name(test, "STAT_RX_DROPPED");
977 XDP_PACKET_HEADROOM - 1;1122 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1123 XDP_PACKET_HEADROOM - 1;
1124 testapp_validate_traffic(test);
978 break;1125 break;
979 case STAT_TEST_RX_FULL:1126 case STAT_TEST_RX_FULL:
980 rxqsize = RX_FULL_RXQSIZE;1127 test_spec_set_name(test, "STAT_RX_FULL");
1128 test->ifobj_rx->xsk->rxqsize = RX_FULL_RXQSIZE;
1129 testapp_validate_traffic(test);
981 break;1130 break;
982 case STAT_TEST_TX_INVALID:1131 case STAT_TEST_TX_INVALID:
983 continue;1132 test_spec_set_name(test, "STAT_TX_INVALID");
1133 pkt_stream_replace(test, DEFAULT_PKT_CNT, XSK_UMEM__INVALID_FRAME_SIZE);
1134 testapp_validate_traffic(test);
1135
1136 pkt_stream_restore_default(test);
1137 break;
1138 case STAT_TEST_RX_FILL_EMPTY:
1139 test_spec_set_name(test, "STAT_RX_FILL_EMPTY");
1140 test->ifobj_rx->pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, 0,
1141 MIN_PKT_SIZE);
1142 if (!test->ifobj_rx->pkt_stream)
1143 exit_with_error(ENOMEM);
1144 test->ifobj_rx->pkt_stream->use_addr_for_fill = true;
1145 testapp_validate_traffic(test);
1146
1147 pkt_stream_restore_default(test);
1148 break;
984 default:1149 default:
985 break;1150 break;
986 }1151 }
987 testapp_validate();
988 }1152 }
989 1153
990 print_ksft_result();1154 /* To only see the whole stat set being completed unless an individual test fails. */
1155 test_spec_set_name(test, "STATS");
1156}
1157
1158/* Simple test */
1159static bool hugepages_present(struct ifobject *ifobject)
1160{
1161 const size_t mmap_sz = 2 * ifobject->umem->num_frames * ifobject->umem->frame_size;
1162 void *bufs;
1163
1164 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1165 MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_HUGETLB, -1, 0);
1166 if (bufs == MAP_FAILED)
1167 return false;
1168
1169 munmap(bufs, mmap_sz);
1170 return true;
1171}
1172
1173static bool testapp_unaligned(struct test_spec *test)
1174{
1175 if (!hugepages_present(test->ifobj_tx)) {
1176 ksft_test_result_skip("No 2M huge pages present.\n");
1177 return false;
1178 }
1179
1180 test_spec_set_name(test, "UNALIGNED_MODE");
1181 test->ifobj_tx->umem->unaligned_mode = true;
1182 test->ifobj_rx->umem->unaligned_mode = true;
1183 /* Let half of the packets straddle a buffer boundrary */
1184 pkt_stream_replace_half(test, PKT_SIZE, test->ifobj_tx->umem->frame_size - 32);
1185 test->ifobj_rx->pkt_stream->use_addr_for_fill = true;
1186 testapp_validate_traffic(test);
1187
1188 pkt_stream_restore_default(test);
1189 return true;
1190}
1191
1192static void testapp_invalid_desc(struct test_spec *test)
1193{
1194 struct pkt pkts[] = {
1195 /* Zero packet length at address zero allowed */
1196 {0, 0, 0, true},
1197 /* Zero packet length allowed */
1198 {0x1000, 0, 0, true},
1199 /* Straddling the start of umem */
1200 {-2, PKT_SIZE, 0, false},
1201 /* Packet too large */
1202 {0x2000, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
1203 /* After umem ends */
1204 {UMEM_SIZE, PKT_SIZE, 0, false},
1205 /* Straddle the end of umem */
1206 {UMEM_SIZE - PKT_SIZE / 2, PKT_SIZE, 0, false},
1207 /* Straddle a page boundrary */
1208 {0x3000 - PKT_SIZE / 2, PKT_SIZE, 0, false},
1209 /* Straddle a 2K boundrary */
1210 {0x3800 - PKT_SIZE / 2, PKT_SIZE, 0, true},
1211 /* Valid packet for synch so that something is received */
1212 {0x4000, PKT_SIZE, 0, true}};
1213
1214 if (test->ifobj_tx->umem->unaligned_mode) {
1215 /* Crossing a page boundrary allowed */
1216 pkts[6].valid = true;
1217 }
1218 if (test->ifobj_tx->umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
1219 /* Crossing a 2K frame size boundrary not allowed */
1220 pkts[7].valid = false;
1221 }
1222
1223 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1224 testapp_validate_traffic(test);
1225 pkt_stream_restore_default(test);
991}1226}
992 1227
993static void init_iface(struct ifobject *ifobj, const char *dst_mac,1228static void init_iface(struct ifobject *ifobj, const char *dst_mac, const char *src_mac,
994 const char *src_mac, const char *dst_ip,1229 const char *dst_ip, const char *src_ip, const u16 dst_port,
995 const char *src_ip, const u16 dst_port,1230 const u16 src_port, thread_func_t func_ptr)
996 const u16 src_port, enum fvector vector)
997{1231{
998 struct in_addr ip;1232 struct in_addr ip;
999 1233
@@ -1009,58 +1243,73 @@ static void init_iface(struct ifobject *ifobj, const char *dst_mac, · 9 unmodified lines
1009 ifobj->dst_port = dst_port;1243 ifobj->dst_port = dst_port;
1010 ifobj->src_port = src_port;1244 ifobj->src_port = src_port;
1011 1245
1012 if (vector == tx) {1246 ifobj->func_ptr = func_ptr;
1013 ifobj->fv.vector = tx;
1014 ifobj->func_ptr = worker_testapp_validate_tx;
1015 ifdict_tx = ifobj;
1016 } else {
1017 ifobj->fv.vector = rx;
1018 ifobj->func_ptr = worker_testapp_validate_rx;
1019 ifdict_rx = ifobj;
1020 }
1021}1247}
1022 1248
1023static void run_pkt_test(int mode, int type)1249static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_type type)
1024{1250{
1025 test_type = type;1251 test_type = type;
1026 1252
1027 /* reset defaults after potential previous test */1253 /* reset defaults after potential previous test */
1028 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
1029 second_step = 0;
1030 stat_test_type = -1;1254 stat_test_type = -1;
1031 rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
1032 frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM;
1033
1034 configured_mode = mode;
1035
1036 switch (mode) {
1037 case (TEST_MODE_SKB):
1038 xdp_flags |= XDP_FLAGS_SKB_MODE;
1039 break;
1040 case (TEST_MODE_DRV):
1041 xdp_flags |= XDP_FLAGS_DRV_MODE;
1042 break;
1043 default:
1044 break;
1045 }
1046 1255
1047 switch (test_type) {1256 switch (test_type) {
1048 case TEST_TYPE_STATS:1257 case TEST_TYPE_STATS:
1049 testapp_stats();1258 testapp_stats(test);
1050 break;1259 break;
1051 case TEST_TYPE_TEARDOWN:1260 case TEST_TYPE_TEARDOWN:
1052 testapp_teardown();1261 testapp_teardown(test);
1053 break;1262 break;
1054 case TEST_TYPE_BIDI:1263 case TEST_TYPE_BIDI:
1055 testapp_bidi();1264 testapp_bidi(test);
1056 break;1265 break;
1057 case TEST_TYPE_BPF_RES:1266 case TEST_TYPE_BPF_RES:
1058 testapp_bpf_res();1267 testapp_bpf_res(test);
1268 break;
1269 case TEST_TYPE_RUN_TO_COMPLETION:
1270 test_spec_set_name(test, "RUN_TO_COMPLETION");
1271 testapp_validate_traffic(test);
1272 break;
1273 case TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME:
1274 test_spec_set_name(test, "RUN_TO_COMPLETION_2K_FRAME_SIZE");
1275 test->ifobj_tx->umem->frame_size = 2048;
1276 test->ifobj_rx->umem->frame_size = 2048;
1277 pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
1278 testapp_validate_traffic(test);
1279
1280 pkt_stream_restore_default(test);
1281 break;
1282 case TEST_TYPE_POLL:
1283 test->ifobj_tx->use_poll = true;
1284 test->ifobj_rx->use_poll = true;
1285 test_spec_set_name(test, "POLL");
1286 testapp_validate_traffic(test);
1287 break;
1288 case TEST_TYPE_ALIGNED_INV_DESC:
1289 test_spec_set_name(test, "ALIGNED_INV_DESC");
1290 testapp_invalid_desc(test);
1291 break;
1292 case TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME:
1293 test_spec_set_name(test, "ALIGNED_INV_DESC_2K_FRAME_SIZE");
1294 test->ifobj_tx->umem->frame_size = 2048;
1295 test->ifobj_rx->umem->frame_size = 2048;
1296 testapp_invalid_desc(test);
1297 break;
1298 case TEST_TYPE_UNALIGNED_INV_DESC:
1299 test_spec_set_name(test, "UNALIGNED_INV_DESC");
1300 test->ifobj_tx->umem->unaligned_mode = true;
1301 test->ifobj_rx->umem->unaligned_mode = true;
1302 testapp_invalid_desc(test);
1303 break;
1304 case TEST_TYPE_UNALIGNED:
1305 if (!testapp_unaligned(test))
1306 return;
1059 break;1307 break;
1060 default:1308 default:
1061 testapp_validate();
1062 break;1309 break;
1063 }1310 }
1311
1312 print_ksft_result(test);
1064}1313}
1065 1314
1066static struct ifobject *ifobject_create(void)1315static struct ifobject *ifobject_create(void)
@@ -1071,11 +1320,11 @@ static struct ifobject *ifobject_create(void) · 4 unmodified lines
1071 if (!ifobj)1320 if (!ifobj)
1072 return NULL;1321 return NULL;
1073 1322
1074 ifobj->xsk_arr = calloc(2, sizeof(struct xsk_socket_info *));1323 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
1075 if (!ifobj->xsk_arr)1324 if (!ifobj->xsk_arr)
1076 goto out_xsk_arr;1325 goto out_xsk_arr;
1077 1326
1078 ifobj->umem_arr = calloc(2, sizeof(struct xsk_umem_info *));1327 ifobj->umem_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->umem_arr));
1079 if (!ifobj->umem_arr)1328 if (!ifobj->umem_arr)
1080 goto out_umem_arr;1329 goto out_umem_arr;
1081 1330
@@ -1098,34 +1347,53 @@ static void ifobject_delete(struct ifobject *ifobj) · 16 unmodified lines
1098int main(int argc, char **argv)1347int main(int argc, char **argv)
1099{1348{
1100 struct rlimit _rlim = { RLIM_INFINITY, RLIM_INFINITY };1349 struct rlimit _rlim = { RLIM_INFINITY, RLIM_INFINITY };
1101 int i, j;1350 struct pkt_stream *pkt_stream_default;
1351 struct ifobject *ifobj_tx, *ifobj_rx;
1352 struct test_spec test;
1353 u32 i, j;
1102 1354
1103 if (setrlimit(RLIMIT_MEMLOCK, &_rlim))1355 if (setrlimit(RLIMIT_MEMLOCK, &_rlim))
1104 exit_with_error(errno);1356 exit_with_error(errno);
1105 1357
1106 for (i = 0; i < MAX_INTERFACES; i++) {1358 ifobj_tx = ifobject_create();
1107 ifdict[i] = ifobject_create();1359 if (!ifobj_tx)
1108 if (!ifdict[i])1360 exit_with_error(ENOMEM);
1109 exit_with_error(ENOMEM);1361 ifobj_rx = ifobject_create();
1110 }1362 if (!ifobj_rx)
1363 exit_with_error(ENOMEM);
1111 1364
1112 setlocale(LC_ALL, "");1365 setlocale(LC_ALL, "");
1113 1366
1114 parse_command_line(argc, argv);1367 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
1115 1368
1116 init_iface(ifdict[tx], MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2, tx);1369 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
1117 init_iface(ifdict[rx], MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1, rx);1370 usage(basename(argv[0]));
1371 ksft_exit_xfail();
1372 }
1373
1374 init_iface(ifobj_tx, MAC1, MAC2, IP1, IP2, UDP_PORT1, UDP_PORT2,
1375 worker_testapp_validate_tx);
1376 init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
1377 worker_testapp_validate_rx);
1378
1379 test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
1380 pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1381 if (!pkt_stream_default)
1382 exit_with_error(ENOMEM);
1383 test.pkt_stream_default = pkt_stream_default;
1118 1384
1119 ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);1385 ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);
1120 1386
1121 for (i = 0; i < TEST_MODE_MAX; i++)1387 for (i = 0; i < TEST_MODE_MAX; i++)
1122 for (j = 0; j < TEST_TYPE_MAX; j++) {1388 for (j = 0; j < TEST_TYPE_MAX; j++) {
1123 run_pkt_test(i, j);1389 test_spec_init(&test, ifobj_tx, ifobj_rx, i);
1390 run_pkt_test(&test, i, j);
1124 usleep(USLEEP_MAX);1391 usleep(USLEEP_MAX);
1125 }1392 }
1126 1393
1127 for (i = 0; i < MAX_INTERFACES; i++)1394 pkt_stream_delete(pkt_stream_default);
1128 ifobject_delete(ifdict[i]);1395 ifobject_delete(ifobj_tx);
1396 ifobject_delete(ifobj_rx);
1129 1397
1130 ksft_exit_pass();1398 ksft_exit_pass();
1131 return 0;1399 return 0;
tools/testing/selftests/bpf/xdpxceiver.h+38−28
@@ -20,10 +20,9 @@ · 19 unmodified lines
20#define MAX_INTERFACES 220#define MAX_INTERFACES 2
21#define MAX_INTERFACE_NAME_CHARS 721#define MAX_INTERFACE_NAME_CHARS 7
22#define MAX_INTERFACES_NAMESPACE_CHARS 1022#define MAX_INTERFACES_NAMESPACE_CHARS 10
23#define MAX_SOCKS 123#define MAX_SOCKETS 2
24#define MAX_TEST_NAME_SIZE 32
24#define MAX_TEARDOWN_ITER 1025#define MAX_TEARDOWN_ITER 10
25#define MAX_BIDI_ITER 2
26#define MAX_BPF_ITER 2
27#define PKT_HDR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr) + \26#define PKT_HDR_SIZE (sizeof(struct ethhdr) + sizeof(struct iphdr) + \
28 sizeof(struct udphdr))27 sizeof(struct udphdr))
29#define MIN_PKT_SIZE 6428#define MIN_PKT_SIZE 64
@@ -39,7 +38,10 @@ · 9 unmodified lines
39#define BATCH_SIZE 838#define BATCH_SIZE 8
40#define POLL_TMOUT 100039#define POLL_TMOUT 1000
41#define DEFAULT_PKT_CNT (4 * 1024)40#define DEFAULT_PKT_CNT (4 * 1024)
41#define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4)
42#define UMEM_SIZE (DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE)
42#define RX_FULL_RXQSIZE 3243#define RX_FULL_RXQSIZE 32
44#define DEFAULT_OFFSET 256
43#define XSK_UMEM__INVALID_FRAME_SIZE (XSK_UMEM__DEFAULT_FRAME_SIZE + 1)45#define XSK_UMEM__INVALID_FRAME_SIZE (XSK_UMEM__DEFAULT_FRAME_SIZE + 1)
44 46
45#define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0)47#define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0)
@@ -51,8 +53,13 @@ enum test_mode { · 5 unmodified lines
51};53};
52 54
53enum test_type {55enum test_type {
54 TEST_TYPE_NOPOLL,56 TEST_TYPE_RUN_TO_COMPLETION,
57 TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME,
55 TEST_TYPE_POLL,58 TEST_TYPE_POLL,
59 TEST_TYPE_UNALIGNED,
60 TEST_TYPE_ALIGNED_INV_DESC,
61 TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME,
62 TEST_TYPE_UNALIGNED_INV_DESC,
56 TEST_TYPE_TEARDOWN,63 TEST_TYPE_TEARDOWN,
57 TEST_TYPE_BIDI,64 TEST_TYPE_BIDI,
58 TEST_TYPE_STATS,65 TEST_TYPE_STATS,
@@ -68,25 +75,21 @@ enum stat_test_type { · 9 unmodified lines
68 STAT_TEST_TYPE_MAX75 STAT_TEST_TYPE_MAX
69};76};
70 77
71static int configured_mode;
72static bool opt_pkt_dump;78static bool opt_pkt_dump;
73static u32 num_frames = DEFAULT_PKT_CNT / 4;
74static bool second_step;
75static int test_type;79static int test_type;
76 80
77static bool opt_verbose;81static bool opt_verbose;
78
79static u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
80static u32 xdp_bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY;
81static int stat_test_type;82static int stat_test_type;
82static u32 rxqsize;
83static u32 frame_headroom;
84 83
85struct xsk_umem_info {84struct xsk_umem_info {
86 struct xsk_ring_prod fq;85 struct xsk_ring_prod fq;
87 struct xsk_ring_cons cq;86 struct xsk_ring_cons cq;
88 struct xsk_umem *umem;87 struct xsk_umem *umem;
88 u32 num_frames;
89 u32 frame_headroom;
89 void *buffer;90 void *buffer;
91 u32 frame_size;
92 bool unaligned_mode;
90};93};
91 94
92struct xsk_socket_info {95struct xsk_socket_info {
@@ -95,51 +98,58 @@ struct xsk_socket_info { · 2 unmodified lines
95 struct xsk_umem_info *umem;98 struct xsk_umem_info *umem;
96 struct xsk_socket *xsk;99 struct xsk_socket *xsk;
97 u32 outstanding_tx;100 u32 outstanding_tx;
98};101 u32 rxqsize;
99
100struct flow_vector {
101 enum fvector {
102 tx,
103 rx,
104 } vector;
105};102};
106 103
107struct pkt {104struct pkt {
108 u64 addr;105 u64 addr;
109 u32 len;106 u32 len;
110 u32 payload;107 u32 payload;
108 bool valid;
111};109};
112 110
113struct pkt_stream {111struct pkt_stream {
114 u32 nb_pkts;112 u32 nb_pkts;
113 u32 rx_pkt_nb;
115 struct pkt *pkts;114 struct pkt *pkts;
115 bool use_addr_for_fill;
116};116};
117 117
118typedef void *(*thread_func_t)(void *arg);
119
118struct ifobject {120struct ifobject {
119 char ifname[MAX_INTERFACE_NAME_CHARS];121 char ifname[MAX_INTERFACE_NAME_CHARS];
120 char nsname[MAX_INTERFACES_NAMESPACE_CHARS];122 char nsname[MAX_INTERFACES_NAMESPACE_CHARS];
121 struct xsk_socket_info *xsk;123 struct xsk_socket_info *xsk;
122 struct xsk_socket_info **xsk_arr;124 struct xsk_socket_info *xsk_arr;
123 struct xsk_umem_info **umem_arr;
124 struct xsk_umem_info *umem;125 struct xsk_umem_info *umem;
125 void *(*func_ptr)(void *arg);126 struct xsk_umem_info *umem_arr;
126 struct flow_vector fv;127 thread_func_t func_ptr;
127 struct pkt_stream *pkt_stream;128 struct pkt_stream *pkt_stream;
128 int ns_fd;129 int ns_fd;
129 u32 dst_ip;130 u32 dst_ip;
130 u32 src_ip;131 u32 src_ip;
132 u32 xdp_flags;
133 u32 bind_flags;
131 u16 src_port;134 u16 src_port;
132 u16 dst_port;135 u16 dst_port;
136 bool tx_on;
137 bool rx_on;
138 bool use_poll;
133 u8 dst_mac[ETH_ALEN];139 u8 dst_mac[ETH_ALEN];
134 u8 src_mac[ETH_ALEN];140 u8 src_mac[ETH_ALEN];
135};141};
136 142
137static struct ifobject *ifdict[MAX_INTERFACES];143struct test_spec {
138static struct ifobject *ifdict_rx;144 struct ifobject *ifobj_tx;
139static struct ifobject *ifdict_tx;145 struct ifobject *ifobj_rx;
146 struct pkt_stream *pkt_stream_default;
147 u16 total_steps;
148 u16 current_step;
149 u16 nb_sockets;
150 char name[MAX_TEST_NAME_SIZE];
151};
140 152
141/*threads*/
142pthread_barrier_t barr;153pthread_barrier_t barr;
143pthread_t t0, t1;
144 154
145#endif /* XDPXCEIVER_H */155#endif /* XDPXCEIVER_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t