Merge tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2024-03-25 We've added 38 non-merge commits during the last 13 day(s) which contain a total of 50 files changed, 867 insertions(+), 274 deletions(-). The main changes are: 1) Add the ability to specify and retrieve BPF cookie also for raw tracepoint programs in order to ease migration from classic to raw tracepoints, from Andrii Nakryiko. 2) Allow the use of bpf_get_{ns_,}current_pid_tgid() helper for all program types and add additional BPF selftests, from Yonghong Song. 3) Several improvements to bpftool and its build, for example, enabling libbpf logs when loading pid_iter in debug mode, from Quentin Monnet. 4) Check the return code of all BPF-related set_memory_*() functions during load and bail out in case they fail, from Christophe Leroy. 5) Avoid a goto in regs_refine_cond_op() such that the verifier can be better integrated into Agni tool which doesn't support backedges yet, from Harishankar Vishwanathan. 6) Add a small BPF trie perf improvement by always inlining longest_prefix_match, from Jesper Dangaard Brouer. 7) Small BPF selftest refactor in bpf_tcp_ca.c to utilize start_server() helper instead of open-coding it, from Geliang Tang. 8) Improve test_tc_tunnel.sh BPF selftest to prevent client connect before the server bind, from Alessandro Carminati. 9) Fix BPF selftest benchmark for older glibc and use syscall(SYS_gettid) instead of gettid(), from Alan Maguire. 10) Implement a backward-compatible method for struct_ops types with additional fields which are not present in older kernels, from Kui-Feng Lee. 11) Add a small helper to check if an instruction is addr_space_cast from as(0) to as(1) and utilize it in x86-64 JIT, from Puranjay Mohan. 12) Small cleanup to remove unnecessary error check in bpf_struct_ops_map_update_elem, from Martin KaFai Lau. 13) Improvements to libbpf fd validity checks for BPF map/programs, from Mykyta Yatsenko. * tag 'for-netdev' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (38 commits) selftests/bpf: Fix flaky test btf_map_in_map/lookup_update bpf: implement insn_is_cast_user() helper for JITs bpf: Avoid get_kernel_nofault() to fetch kprobe entry IP selftests/bpf: Use start_server in bpf_tcp_ca bpf: Sync uapi bpf.h to tools directory libbpf: Add new sec_def "sk_skb/verdict" selftests/bpf: Mark uprobe trigger functions with nocf_check attribute selftests/bpf: Use syscall(SYS_gettid) instead of gettid() wrapper in bench bpf-next: Avoid goto in regs_refine_cond_op() bpftool: Clean up HOST_CFLAGS, HOST_LDFLAGS for bootstrap bpftool selftests/bpf: scale benchmark counting by using per-CPU counters bpftool: Remove unnecessary source files from bootstrap version bpftool: Enable libbpf logs when loading pid_iter in debug mode selftests/bpf: add raw_tp/tp_btf BPF cookie subtests libbpf: add support for BPF cookie for raw_tp/tp_btf programs bpf: support BPF cookie in raw tracepoint (raw_tp, tp_btf) programs bpf: pass whole link instead of prog when triggering raw tracepoint bpf: flatten bpf_probe_register call chain selftests/bpf: Prevent client connect before server bind in test_tc_tunnel.sh selftests/bpf: Add a sk_msg prog bpf_get_ns_current_pid_tgid() test ... ====================

Jakub Kicinski committed 2 years ago

Commit 2a702c22 parents ee36b1e 14bb1e850 files · +867 −274Browse files

Unified · Split

50 files

arch/arm/net/bpf_jit_32.c +12 −13
arch/arm64/net/bpf_jit_comp.c +2 −5
arch/loongarch/net/bpf_jit.c +16 −6
arch/mips/net/bpf_jit_comp.c +2 −1
arch/parisc/net/bpf_jit_core.c +7 −1
arch/s390/net/bpf_jit_comp.c +5 −1
arch/sparc/net/bpf_jit_comp_64.c +5 −1
arch/x86/net/bpf_jit_comp.c +3 −7
arch/x86/net/bpf_jit_comp32.c +1 −2
include/linux/bpf.h +7 −2
include/linux/filter.h +16 −4
include/linux/trace_events.h +20 −16
include/trace/bpf_probe.h +1 −2
include/uapi/linux/bpf.h +4 −2
kernel/bpf/bpf_struct_ops.c +6 −4
kernel/bpf/cgroup.c +0 −2
kernel/bpf/core.c +24 −8
kernel/bpf/helpers.c +4 −0
kernel/bpf/lpm_trie.c +13 −5
kernel/bpf/syscall.c +11 −11
kernel/bpf/trampoline.c +5 −10
kernel/bpf/verifier.c +20 −11
kernel/trace/bpf_trace.c +32 −20
net/bpf/bpf_dummy_struct_ops.c +3 −1
net/core/filter.c +0 −2
tools/bpf/bpftool/Makefile +6 −8
tools/bpf/bpftool/gen.c +2 −1
tools/bpf/bpftool/pids.c +12 −7
tools/bpf/bpftool/skeleton/pid_iter.bpf.c +2 −2
tools/include/linux/compiler.h +4 −0
tools/include/uapi/linux/bpf.h +4 −2
tools/lib/bpf/bpf.c +14 −2
tools/lib/bpf/bpf.h +9 −0
tools/lib/bpf/libbpf.c +92 −19
tools/lib/bpf/libbpf.h +11 −0
tools/lib/bpf/libbpf.map +2 −0
tools/testing/selftests/bpf/benchs/bench_local_storage_create.c +1 −1
tools/testing/selftests/bpf/benchs/bench_trigger.c +40 −8
tools/testing/selftests/bpf/prog_tests/bpf_cookie.c +113 −1
tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +1 −12
tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c +1 −25
tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c +183 −31
tools/testing/selftests/bpf/prog_tests/test_struct_ops_module.c +47 −0
tools/testing/selftests/bpf/progs/iters.c +1 −1
tools/testing/selftests/bpf/progs/struct_ops_module.c +15 −1
tools/testing/selftests/bpf/progs/test_bpf_cookie.c +16 −0
tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c +28 −3
tools/testing/selftests/bpf/progs/trigger_bench.c +27 −12
tools/testing/selftests/bpf/test_cpp.cpp +5 −0
tools/testing/selftests/bpf/test_tc_tunnel.sh +12 −1

arch/arm/net/bpf_jit_32.c+12−13

 	/* If building the body of the JITed code fails somehow,
 	 * we fall back to the interpretation.
 	 */
-	if (build_body(&ctx) < 0) {
+	if (build_body(&ctx) < 0)
-		image_ptr = NULL;
+		goto out_free;
-		bpf_jit_binary_free(header);
-		prog = orig_prog;
-		goto out_imms;
-	}
 	build_epilogue(&ctx);
 	/* 3.) Extra pass to validate JITed Code */
-	if (validate_code(&ctx)) {
+	if (validate_code(&ctx))
-		image_ptr = NULL;
+		goto out_free;
-		bpf_jit_binary_free(header);
-		prog = orig_prog;
-		goto out_imms;
-	}
 	flush_icache_range((u32)header, (u32)(ctx.target + ctx.idx));
 	if (bpf_jit_enable > 1)
 		/* there are 2 passes here */
 		bpf_jit_dump(prog->len, image_size, 2, ctx.target);
-	bpf_jit_binary_lock_ro(header);
+	if (bpf_jit_binary_lock_ro(header))
+		goto out_free;
 	prog->bpf_func = (void *)ctx.target;
 	prog->jited = 1;
 	prog->jited_len = image_size;
 		bpf_jit_prog_release_other(prog, prog == orig_prog ?
 					   tmp : orig_prog);
 	return prog;
+out_free:
+	image_ptr = NULL;
+	bpf_jit_binary_free(header);
+	prog = orig_prog;
+	goto out_imms;
 }

arch/arm64/net/bpf_jit_comp.c+2−5

 	bpf_prog_pack_free(image, size);
 }
-void arch_protect_bpf_trampoline(void *image, unsigned int size)
+int arch_protect_bpf_trampoline(void *image, unsigned int size)
-{
-}
-void arch_unprotect_bpf_trampoline(void *image, unsigned int size)
 {
+	return 0;
 }
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image,

arch/loongarch/net/bpf_jit.c+16−6

 	flush_icache_range((unsigned long)header, (unsigned long)(ctx.image + ctx.idx));
 	if (!prog->is_func || extra_pass) {
+		int err;
 		if (extra_pass && ctx.idx != jit_data->ctx.idx) {
 			pr_err_once("multi-func JIT bug %d != %d\n",
 				    ctx.idx, jit_data->ctx.idx);
-			bpf_jit_binary_free(header);
+			goto out_free;
-			prog->bpf_func = NULL;
+		}
-			prog->jited = 0;
+		err = bpf_jit_binary_lock_ro(header);
-			prog->jited_len = 0;
+		if (err) {
-			goto out_offset;
+			pr_err_once("bpf_jit_binary_lock_ro() returned %d\n",
+				    err);
+			goto out_free;
 		}
-		bpf_jit_binary_lock_ro(header);
 	} else {
 		jit_data->ctx = ctx;
 		jit_data->image = image_ptr;
 	out_offset = -1;
 	return prog;
+out_free:
+	bpf_jit_binary_free(header);
+	prog->bpf_func = NULL;
+	prog->jited = 0;
+	prog->jited_len = 0;
+	goto out_offset;
 }
 /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */

arch/mips/net/bpf_jit_comp.c+2−1

@@ -1012,7 +1012,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 1011 unmodified lines
1012	bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]);	1012	bpf_prog_fill_jited_linfo(prog, &ctx.descriptors[1]);
1013		1013
1014	/* Set as read-only exec and flush instruction cache */	1014	/* Set as read-only exec and flush instruction cache */
1015	bpf_jit_binary_lock_ro(header);	1015	if (bpf_jit_binary_lock_ro(header))
		1016	goto out_err;
1016	flush_icache_range((unsigned long)header,	1017	flush_icache_range((unsigned long)header,
1017	(unsigned long)&ctx.target[ctx.jit_index]);	1018	(unsigned long)&ctx.target[ctx.jit_index]);
1018		1019

arch/parisc/net/bpf_jit_core.c+7−1

 	bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
 	if (!prog->is_func || extra_pass) {
-		bpf_jit_binary_lock_ro(jit_data->header);
+		if (bpf_jit_binary_lock_ro(jit_data->header)) {
+			bpf_jit_binary_free(jit_data->header);
+			prog->bpf_func = NULL;
+			prog->jited = 0;
+			prog->jited_len = 0;
+			goto out_offset;
+		}
 		prologue_len = ctx->epilogue_offset - ctx->body_len;
 		for (i = 0; i < prog->len; i++)
 			ctx->offset[i] += prologue_len;

arch/s390/net/bpf_jit_comp.c+5−1

 		print_fn_code(jit.prg_buf, jit.size_prg);
 	}
 	if (!fp->is_func || extra_pass) {
-		bpf_jit_binary_lock_ro(header);
+		if (bpf_jit_binary_lock_ro(header)) {
+			bpf_jit_binary_free(header);
+			fp = orig_fp;
+			goto free_addrs;
+		}
 	} else {
 		jit_data->header = header;
 		jit_data->ctx = jit;

arch/sparc/net/bpf_jit_comp_64.c+5−1

 	bpf_flush_icache(header, (u8 *)header + header->size);
 	if (!prog->is_func || extra_pass) {
-		bpf_jit_binary_lock_ro(header);
+		if (bpf_jit_binary_lock_ro(header)) {
+			bpf_jit_binary_free(header);
+			prog = orig_prog;
+			goto out_off;
+		}
 	} else {
 		jit_data->ctx = ctx;
 		jit_data->image = image_ptr;

arch/x86/net/bpf_jit_comp.c+3−7

 			break;
 		case BPF_ALU64 | BPF_MOV | BPF_X:
-			if (insn->off == BPF_ADDR_SPACE_CAST &&
+			if (insn_is_cast_user(insn)) {
-			    insn->imm == 1U << 16) {
 				if (dst_reg != src_reg)
 					/* 32-bit mov */
 					emit_mov_reg(&prog, false, dst_reg, src_reg);
 	bpf_prog_pack_free(image, size);
 }
-void arch_protect_bpf_trampoline(void *image, unsigned int size)
+int arch_protect_bpf_trampoline(void *image, unsigned int size)
-{
-}
-void arch_unprotect_bpf_trampoline(void *image, unsigned int size)
 {
+	return 0;
 }
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,

arch/x86/net/bpf_jit_comp32.c+1−2

@@ -2600,8 +2600,7 @@ out_image: · 2599 unmodified lines
2600	if (bpf_jit_enable > 1)	2600	if (bpf_jit_enable > 1)
2601	bpf_jit_dump(prog->len, proglen, pass + 1, image);	2601	bpf_jit_dump(prog->len, proglen, pass + 1, image);
2602		2602
2603	if (image) {	2603	if (image && !bpf_jit_binary_lock_ro(header)) {
2604	bpf_jit_binary_lock_ro(header);
2605	prog->bpf_func = (void *)image;	2604	prog->bpf_func = (void *)image;
2606	prog->jited = 1;	2605	prog->jited = 1;
2607	prog->jited_len = proglen;	2606	prog->jited_len = proglen;

include/linux/bpf.h+7−2

 				void *func_addr);
 void *arch_alloc_bpf_trampoline(unsigned int size);
 void arch_free_bpf_trampoline(void *image, unsigned int size);
-void arch_protect_bpf_trampoline(void *image, unsigned int size);
+int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size);
-void arch_unprotect_bpf_trampoline(void *image, unsigned int size);
 int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,
 			     struct bpf_tramp_links *tlinks, void *func_addr);
 	struct bpf_prog *tgt_prog;
 };
+struct bpf_raw_tp_link {
+	struct bpf_link link;
+	struct bpf_raw_event_map *btp;
+	u64 cookie;
+};
 struct bpf_link_primer {
 	struct bpf_link *link;
 	struct file *file;

include/linux/filter.h+16−4

 	return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1;
 }
+/* addr_space_cast from as(0) to as(1) is for converting bpf arena pointers
+ * to pointers in user vma.
+ */
+static inline bool insn_is_cast_user(const struct bpf_insn *insn)
+{
+	return insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) &&
+			      insn->off == BPF_ADDR_SPACE_CAST &&
+			      insn->imm == 1U << 16;
+}
 /* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
 #define BPF_LD_IMM64(DST, IMM)					\
 	BPF_LD_IMM64_RAW(DST, 0, IMM)
 #define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0]))
-static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
+static inline int __must_check bpf_prog_lock_ro(struct bpf_prog *fp)
 {
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
 	if (!fp->jited) {
 		set_vm_flush_reset_perms(fp);
-		set_memory_ro((unsigned long)fp, fp->pages);
+		return set_memory_ro((unsigned long)fp, fp->pages);
 	}
 #endif
+	return 0;
 }
-static inline void bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
+static inline int __must_check
+bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr)
 {
 	set_vm_flush_reset_perms(hdr);
-	set_memory_rox((unsigned long)hdr, hdr->size >> PAGE_SHIFT);
+	return set_memory_rox((unsigned long)hdr, hdr->size >> PAGE_SHIFT);
 }
 int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);

include/linux/trace_events.h+20−16

@@ -765,8 +765,11 @@ unsigned int trace_call_bpf(struct trace_event_call call, void ctx); · 764 unmodified lines
765	int perf_event_attach_bpf_prog(struct perf_event event, struct bpf_prog prog, u64 bpf_cookie);	765	int perf_event_attach_bpf_prog(struct perf_event event, struct bpf_prog prog, u64 bpf_cookie);
766	void perf_event_detach_bpf_prog(struct perf_event *event);	766	void perf_event_detach_bpf_prog(struct perf_event *event);
767	int perf_event_query_prog_array(struct perf_event event, void __user info);	767	int perf_event_query_prog_array(struct perf_event event, void __user info);
768	int bpf_probe_register(struct bpf_raw_event_map btp, struct bpf_prog prog);	768
769	int bpf_probe_unregister(struct bpf_raw_event_map btp, struct bpf_prog prog);	769	struct bpf_raw_tp_link;
		770	int bpf_probe_register(struct bpf_raw_event_map btp, struct bpf_raw_tp_link link);
		771	int bpf_probe_unregister(struct bpf_raw_event_map btp, struct bpf_raw_tp_link link);
		772
770	struct bpf_raw_event_map bpf_get_raw_tracepoint(const char name);	773	struct bpf_raw_event_map bpf_get_raw_tracepoint(const char name);
771	void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);	774	void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp);
772	int bpf_get_perf_event_info(const struct perf_event event, u32 prog_id,	775	int bpf_get_perf_event_info(const struct perf_event event, u32 prog_id,
@@ -794,11 +797,12 @@ perf_event_query_prog_array(struct perf_event event, void __user info) · 21 unmodified lines
794	{	797	{
795	return -EOPNOTSUPP;	798	return -EOPNOTSUPP;
796	}	799	}
797	static inline int bpf_probe_register(struct bpf_raw_event_map btp, struct bpf_prog p)	800	struct bpf_raw_tp_link;
		801	static inline int bpf_probe_register(struct bpf_raw_event_map btp, struct bpf_raw_tp_link link)
798	{	802	{
799	return -EOPNOTSUPP;	803	return -EOPNOTSUPP;
800	}	804	}
801	static inline int bpf_probe_unregister(struct bpf_raw_event_map btp, struct bpf_prog p)	805	static inline int bpf_probe_unregister(struct bpf_raw_event_map btp, struct bpf_raw_tp_link link)
802	{	806	{
803	return -EOPNOTSUPP;	807	return -EOPNOTSUPP;
804	}	808	}
@@ -909,31 +913,31 @@ void perf_trace_buf_alloc(int size, struct pt_regs regs, int rctxp); · 104 unmodified lines
909	int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog, u64 bpf_cookie);	913	int perf_event_set_bpf_prog(struct perf_event event, struct bpf_prog prog, u64 bpf_cookie);
910	void perf_event_free_bpf_prog(struct perf_event *event);	914	void perf_event_free_bpf_prog(struct perf_event *event);
911		915
912	void bpf_trace_run1(struct bpf_prog *prog, u64 arg1);	916	void bpf_trace_run1(struct bpf_raw_tp_link *link, u64 arg1);
913	void bpf_trace_run2(struct bpf_prog *prog, u64 arg1, u64 arg2);	917	void bpf_trace_run2(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2);
914	void bpf_trace_run3(struct bpf_prog *prog, u64 arg1, u64 arg2,	918	void bpf_trace_run3(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
915	u64 arg3);	919	u64 arg3);
916	void bpf_trace_run4(struct bpf_prog *prog, u64 arg1, u64 arg2,	920	void bpf_trace_run4(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
917	u64 arg3, u64 arg4);	921	u64 arg3, u64 arg4);
918	void bpf_trace_run5(struct bpf_prog *prog, u64 arg1, u64 arg2,	922	void bpf_trace_run5(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
919	u64 arg3, u64 arg4, u64 arg5);	923	u64 arg3, u64 arg4, u64 arg5);
920	void bpf_trace_run6(struct bpf_prog *prog, u64 arg1, u64 arg2,	924	void bpf_trace_run6(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
921	u64 arg3, u64 arg4, u64 arg5, u64 arg6);	925	u64 arg3, u64 arg4, u64 arg5, u64 arg6);
922	void bpf_trace_run7(struct bpf_prog *prog, u64 arg1, u64 arg2,	926	void bpf_trace_run7(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
923	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7);	927	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7);
924	void bpf_trace_run8(struct bpf_prog *prog, u64 arg1, u64 arg2,	928	void bpf_trace_run8(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
925	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,	929	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
926	u64 arg8);	930	u64 arg8);
927	void bpf_trace_run9(struct bpf_prog *prog, u64 arg1, u64 arg2,	931	void bpf_trace_run9(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
928	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,	932	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
929	u64 arg8, u64 arg9);	933	u64 arg8, u64 arg9);
930	void bpf_trace_run10(struct bpf_prog *prog, u64 arg1, u64 arg2,	934	void bpf_trace_run10(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
931	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,	935	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
932	u64 arg8, u64 arg9, u64 arg10);	936	u64 arg8, u64 arg9, u64 arg10);
933	void bpf_trace_run11(struct bpf_prog *prog, u64 arg1, u64 arg2,	937	void bpf_trace_run11(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
934	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,	938	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
935	u64 arg8, u64 arg9, u64 arg10, u64 arg11);	939	u64 arg8, u64 arg9, u64 arg10, u64 arg11);
936	void bpf_trace_run12(struct bpf_prog *prog, u64 arg1, u64 arg2,	940	void bpf_trace_run12(struct bpf_raw_tp_link *link, u64 arg1, u64 arg2,
937	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,	941	u64 arg3, u64 arg4, u64 arg5, u64 arg6, u64 arg7,
938	u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12);	942	u64 arg8, u64 arg9, u64 arg10, u64 arg11, u64 arg12);
939	void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,	943	void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx,

include/trace/bpf_probe.h+1−2

@@ -46,8 +46,7 @@ · 45 unmodified lines
46	static notrace void \	46	static notrace void \
47	__bpf_trace_##call(void *__data, proto) \	47	__bpf_trace_##call(void *__data, proto) \
48	{ \	48	{ \
49	struct bpf_prog *prog = __data; \	49	CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(__data, CAST_TO_U64(args)); \
50	CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(prog, CAST_TO_U64(args)); \
51	}	50	}
52		51
53	#undef DECLARE_EVENT_CLASS	52	#undef DECLARE_EVENT_CLASS

include/uapi/linux/bpf.h+4−2

 	} query;
 	struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
-		__u64 name;
+		__u64		name;
-		__u32 prog_fd;
+		__u32		prog_fd;
+		__u32		:32;
+		__aligned_u64	cookie;
 	} raw_tracepoint;
 	struct { /* anonymous struct for BPF_BTF_LOAD */

kernel/bpf/bpf_struct_ops.c+6−4

 			cur_image = image;
 			trampoline_start = 0;
 		}
-		if (err < 0)
-			goto reset_unlock;
 		*(void **)(kdata + moff) = image + trampoline_start + cfi_get_offset();
 		if (err)
 			goto reset_unlock;
 	}
-	for (i = 0; i < st_map->image_pages_cnt; i++)
+	for (i = 0; i < st_map->image_pages_cnt; i++) {
-		arch_protect_bpf_trampoline(st_map->image_pages[i], PAGE_SIZE);
+		err = arch_protect_bpf_trampoline(st_map->image_pages[i],
+						  PAGE_SIZE);
+		if (err)
+			goto reset_unlock;
+	}
 	if (st_map->map.map_flags & BPF_F_LINK) {
 		err = 0;

kernel/bpf/cgroup.c+0−2

 	switch (func_id) {
 	case BPF_FUNC_get_current_uid_gid:
 		return &bpf_get_current_uid_gid_proto;
-	case BPF_FUNC_get_current_pid_tgid:
-		return &bpf_get_current_pid_tgid_proto;
 	case BPF_FUNC_get_current_comm:
 		return &bpf_get_current_comm_proto;
 #ifdef CONFIG_CGROUP_NET_CLASSID

kernel/bpf/core.c+24−8

 static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)
 {
 	struct bpf_prog_pack *pack;
+	int err;
 	pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)),
 		       GFP_KERNEL);
 	if (!pack)
 		return NULL;
 	pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE);
-	if (!pack->ptr) {
+	if (!pack->ptr)
-		kfree(pack);
+		goto out;
-		return NULL;
-	}
 	bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
 	bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
-	list_add_tail(&pack->list, &pack_list);
 	set_vm_flush_reset_perms(pack->ptr);
-	set_memory_rox((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
+	err = set_memory_rox((unsigned long)pack->ptr,
+			     BPF_PROG_PACK_SIZE / PAGE_SIZE);
+	if (err)
+		goto out;
+	list_add_tail(&pack->list, &pack_list);
 	return pack;
+out:
+	bpf_jit_free_exec(pack->ptr);
+	kfree(pack);
+	return NULL;
 }
 void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
 		size = round_up(size, PAGE_SIZE);
 		ptr = bpf_jit_alloc_exec(size);
 		if (ptr) {
+			int err;
 			bpf_fill_ill_insns(ptr, size);
 			set_vm_flush_reset_perms(ptr);
-			set_memory_rox((unsigned long)ptr, size / PAGE_SIZE);
+			err = set_memory_rox((unsigned long)ptr,
+					     size / PAGE_SIZE);
+			if (err) {
+				bpf_jit_free_exec(ptr);
+				ptr = NULL;
+			}
 		}
 		goto out;
 	}
 	}
 finalize:
-	bpf_prog_lock_ro(fp);
+	*err = bpf_prog_lock_ro(fp);
+	if (*err)
+		return fp;
 	/* The tail call compatibility check can only be done at
 	 * this late stage as we need to determine, if we deal

kernel/bpf/helpers.c+4−0

 		return &bpf_strtol_proto;
 	case BPF_FUNC_strtoul:
 		return &bpf_strtoul_proto;
+	case BPF_FUNC_get_current_pid_tgid:
+		return &bpf_get_current_pid_tgid_proto;
+	case BPF_FUNC_get_ns_current_pid_tgid:
+		return &bpf_get_ns_current_pid_tgid_proto;
 	default:
 		break;
 	}

kernel/bpf/lpm_trie.c+13−5

 }
 /**
- * longest_prefix_match() - determine the longest prefix
+ * __longest_prefix_match() - determine the longest prefix
  * @trie:	The trie to get internal sizes from
  * @node:	The node to operate on
  * @key:	The key to compare to @node
  *
  * Determine the longest prefix of @node that matches the bits in @key.
  */
-static size_t longest_prefix_match(const struct lpm_trie *trie,
+static __always_inline
-				   const struct lpm_trie_node *node,
+size_t __longest_prefix_match(const struct lpm_trie *trie,
-				   const struct bpf_lpm_trie_key_u8 *key)
+			      const struct lpm_trie_node *node,
+			      const struct bpf_lpm_trie_key_u8 *key)
 {
 	u32 limit = min(node->prefixlen, key->prefixlen);
 	u32 prefixlen = 0, i = 0;
 	return prefixlen;
 }
+static size_t longest_prefix_match(const struct lpm_trie *trie,
+				   const struct lpm_trie_node *node,
+				   const struct bpf_lpm_trie_key_u8 *key)
+{
+	return __longest_prefix_match(trie, node, key);
+}
 /* Called from syscall or from eBPF program */
 static void *trie_lookup_elem(struct bpf_map *map, void *_key)
 {
 		 * If it's the maximum possible prefix for this trie, we have
 		 * an exact match and can return it directly.
 		 */
-		matchlen = longest_prefix_match(trie, node, key);
+		matchlen = __longest_prefix_match(trie, node, key);
 		if (matchlen == trie->max_prefixlen) {
 			found = node;
 			break;

kernel/bpf/syscall.c+11−11

 	return err;
 }
-struct bpf_raw_tp_link {
-	struct bpf_link link;
-	struct bpf_raw_event_map *btp;
-};
 static void bpf_raw_tp_link_release(struct bpf_link *link)
 {
 	struct bpf_raw_tp_link *raw_tp =
 		container_of(link, struct bpf_raw_tp_link, link);
-	bpf_probe_unregister(raw_tp->btp, raw_tp->link.prog);
+	bpf_probe_unregister(raw_tp->btp, raw_tp);
 	bpf_put_raw_tracepoint(raw_tp->btp);
 }
 #endif /* CONFIG_PERF_EVENTS */
 static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
-				  const char __user *user_tp_name)
+				  const char __user *user_tp_name, u64 cookie)
 {
 	struct bpf_link_primer link_primer;
 	struct bpf_raw_tp_link *link;
 	bpf_link_init(&link->link, BPF_LINK_TYPE_RAW_TRACEPOINT,
 		      &bpf_raw_tp_link_lops, prog);
 	link->btp = btp;
+	link->cookie = cookie;
 	err = bpf_link_prime(&link->link, &link_primer);
 	if (err) {
 		goto out_put_btp;
 	}
-	err = bpf_probe_register(link->btp, prog);
+	err = bpf_probe_register(link->btp, link);
 	if (err) {
 		bpf_link_cleanup(&link_primer);
 		goto out_put_btp;
 	return err;
 }
-#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.prog_fd
+#define BPF_RAW_TRACEPOINT_OPEN_LAST_FIELD raw_tracepoint.cookie
 static int bpf_raw_tracepoint_open(const union bpf_attr *attr)
 {
 	struct bpf_prog *prog;
+	void __user *tp_name;
+	__u64 cookie;
 	int fd;
 	if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
 	if (IS_ERR(prog))
 		return PTR_ERR(prog);
-	fd = bpf_raw_tp_link_attach(prog, u64_to_user_ptr(attr->raw_tracepoint.name));
+	tp_name = u64_to_user_ptr(attr->raw_tracepoint.name);
+	cookie = attr->raw_tracepoint.cookie;
+	fd = bpf_raw_tp_link_attach(prog, tp_name, cookie);
 	if (fd < 0)
 		bpf_prog_put(prog);
 	return fd;
 			goto out;
 		}
 		if (prog->expected_attach_type == BPF_TRACE_RAW_TP)
-			ret = bpf_raw_tp_link_attach(prog, NULL);
+			ret = bpf_raw_tp_link_attach(prog, NULL, attr->link_create.tracing.cookie);
 		else if (prog->expected_attach_type == BPF_TRACE_ITER)
 			ret = bpf_iter_link_attach(attr, uattr, prog);
 		else if (prog->expected_attach_type == BPF_LSM_CGROUP)

kernel/bpf/trampoline.c+5−10

 	if (err < 0)
 		goto out_free;
-	arch_protect_bpf_trampoline(im->image, im->size);
+	err = arch_protect_bpf_trampoline(im->image, im->size);
+	if (err)
+		goto out_free;
 	WARN_ON(tr->cur_image && total == 0);
 	if (tr->cur_image)
 	bpf_jit_free_exec(image);
 }
-void __weak arch_protect_bpf_trampoline(void *image, unsigned int size)
+int __weak arch_protect_bpf_trampoline(void *image, unsigned int size)
-{
-	WARN_ON_ONCE(size > PAGE_SIZE);
-	set_memory_rox((long)image, 1);
-}
-void __weak arch_unprotect_bpf_trampoline(void *image, unsigned int size)
 {
 	WARN_ON_ONCE(size > PAGE_SIZE);
-	set_memory_nx((long)image, 1);
+	return set_memory_rox((long)image, 1);
-	set_memory_rw((long)image, 1);
 }
 int __weak arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags,

kernel/bpf/verifier.c+20−11

 	struct tnum t;
 	u64 val;
-again:
+	/* In case of GE/GT/SGE/JST, reuse LE/LT/SLE/SLT logic from below */
+	switch (opcode) {
+	case BPF_JGE:
+	case BPF_JGT:
+	case BPF_JSGE:
+	case BPF_JSGT:
+		opcode = flip_opcode(opcode);
+		swap(reg1, reg2);
+		break;
+	default:
+		break;
+	}
 	switch (opcode) {
 	case BPF_JEQ:
 		if (is_jmp32) {
 			reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value);
 		}
 		break;
-	case BPF_JGE:
-	case BPF_JGT:
-	case BPF_JSGE:
-	case BPF_JSGT:
-		/* just reuse LE/LT logic above */
-		opcode = flip_opcode(opcode);
-		swap(reg1, reg2);
-		goto again;
 	default:
 		return;
 	}
 		if (bpf_prog_calc_tag(func[i]))
 			goto out_free;
 		func[i]->is_func = 1;
+		func[i]->sleepable = prog->sleepable;
 		func[i]->aux->func_idx = i;
 		/* Below members will be freed only at prog->aux */
 		func[i]->aux->btf = prog->aux->btf;
 	 * bpf_prog_load will add the kallsyms for the main program.
 	 */
 	for (i = 1; i < env->subprog_cnt; i++) {
-		bpf_prog_lock_ro(func[i]);
+		err = bpf_prog_lock_ro(func[i]);
-		bpf_prog_kallsyms_add(func[i]);
+		if (err)
+			goto out_free;
 	}
+	for (i = 1; i < env->subprog_cnt; i++)
+		bpf_prog_kallsyms_add(func[i]);
 	/* Last step: make now unused interpreter insns from main
 	 * prog consistent for later dump requests, so they can
 	 * later look the same as if they were interpreted only.

kernel/trace/bpf_trace.c+32−20

 {
 	u32 instr;
-	/* Being extra safe in here in case entry ip is on the page-edge. */
+	/* We want to be extra safe in case entry ip is on the page edge,
-	if (get_kernel_nofault(instr, (u32 *) fentry_ip - 1))
+	 * but otherwise we need to avoid get_kernel_nofault()'s overhead.
-		return fentry_ip;
+	 */
+	if ((fentry_ip & ~PAGE_MASK) < ENDBR_INSN_SIZE) {
+		if (get_kernel_nofault(instr, (u32 *)(fentry_ip - ENDBR_INSN_SIZE)))
+			return fentry_ip;
+	} else {
+		instr = *(u32 *)(fentry_ip - ENDBR_INSN_SIZE);
+	}
 	if (is_endbr(instr))
 		fentry_ip -= ENDBR_INSN_SIZE;
 	return fentry_ip;
 		return &bpf_ktime_get_boot_ns_proto;
 	case BPF_FUNC_tail_call:
 		return &bpf_tail_call_proto;
-	case BPF_FUNC_get_current_pid_tgid:
-		return &bpf_get_current_pid_tgid_proto;
 	case BPF_FUNC_get_current_task:
 		return &bpf_get_current_task_proto;
 	case BPF_FUNC_get_current_task_btf:
 		return &bpf_send_signal_thread_proto;
 	case BPF_FUNC_perf_event_read_value:
 		return &bpf_perf_event_read_value_proto;
-	case BPF_FUNC_get_ns_current_pid_tgid:
-		return &bpf_get_ns_current_pid_tgid_proto;
 	case BPF_FUNC_ringbuf_output:
 		return &bpf_ringbuf_output_proto;
 	case BPF_FUNC_ringbuf_reserve:
 		return &bpf_get_stackid_proto_raw_tp;
 	case BPF_FUNC_get_stack:
 		return &bpf_get_stack_proto_raw_tp;
+	case BPF_FUNC_get_attach_cookie:
+		return &bpf_get_attach_cookie_proto_tracing;
 	default:
 		return bpf_tracing_func_proto(func_id, prog);
 	}
 	case BPF_FUNC_get_func_arg_cnt:
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_func_arg_cnt_proto : NULL;
 	case BPF_FUNC_get_attach_cookie:
+		if (prog->type == BPF_PROG_TYPE_TRACING &&
+		    prog->expected_attach_type == BPF_TRACE_RAW_TP)
+			return &bpf_get_attach_cookie_proto_tracing;
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto_tracing : NULL;
 	default:
 		fn = raw_tp_prog_func_proto(func_id, prog);
 }
 static __always_inline
-void __bpf_trace_run(struct bpf_prog *prog, u64 *args)
+void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
 {
+	struct bpf_prog *prog = link->link.prog;
+	struct bpf_run_ctx *old_run_ctx;
+	struct bpf_trace_run_ctx run_ctx;
 	cant_sleep();
 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
 		bpf_prog_inc_misses_counter(prog);
 		goto out;
 	}
+	run_ctx.bpf_cookie = link->cookie;
+	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
 	rcu_read_lock();
 	(void) bpf_prog_run(prog, args);
 	rcu_read_unlock();
+	bpf_reset_run_ctx(old_run_ctx);
 out:
 	this_cpu_dec(*(prog->active));
 }
 #define __SEQ_0_11	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
 #define BPF_TRACE_DEFN_x(x)						\
-	void bpf_trace_run##x(struct bpf_prog *prog,			\
+	void bpf_trace_run##x(struct bpf_raw_tp_link *link,		\
 			      REPEAT(x, SARG, __DL_COM, __SEQ_0_11))	\
 	{								\
 		u64 args[x];						\
 		REPEAT(x, COPY, __DL_SEM, __SEQ_0_11);			\
-		__bpf_trace_run(prog, args);				\
+		__bpf_trace_run(link, args);				\
 	}								\
 	EXPORT_SYMBOL_GPL(bpf_trace_run##x)
 BPF_TRACE_DEFN_x(1);
 BPF_TRACE_DEFN_x(11);
 BPF_TRACE_DEFN_x(12);
-static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
+int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
 {
 	struct tracepoint *tp = btp->tp;
+	struct bpf_prog *prog = link->link.prog;
 	/*
 	 * check that program doesn't access arguments beyond what's
 	if (prog->aux->max_tp_access > btp->writable_size)
 		return -EINVAL;
-	return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func,
+	return tracepoint_probe_register_may_exist(tp, (void *)btp->bpf_func, link);
-						   prog);
-}
-int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
-{
-	return __bpf_probe_register(btp, prog);
 }
-int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
+int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_raw_tp_link *link)
 {
-	return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog);
+	return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, link);
 }
 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,

net/bpf/bpf_dummy_struct_ops.c+3−1

@@ -133,7 +133,9 @@ int bpf_struct_ops_test_run(struct bpf_prog prog, const union bpf_attr kattr, · 132 unmodified lines
133	if (err < 0)	133	if (err < 0)
134	goto out;	134	goto out;
135		135
136	arch_protect_bpf_trampoline(image, PAGE_SIZE);	136	err = arch_protect_bpf_trampoline(image, PAGE_SIZE);
		137	if (err)
		138	goto out;
137	prog_ret = dummy_ops_call_op(image, args);	139	prog_ret = dummy_ops_call_op(image, args);
138		140
139	err = dummy_ops_copy_args(args);	141	err = dummy_ops_copy_args(args);

net/core/filter.c+0−2

 		return &bpf_event_output_data_proto;
 	case BPF_FUNC_get_current_uid_gid:
 		return &bpf_get_current_uid_gid_proto;
-	case BPF_FUNC_get_current_pid_tgid:
-		return &bpf_get_current_pid_tgid_proto;
 	case BPF_FUNC_sk_storage_get:
 		return &bpf_sk_storage_get_proto;
 	case BPF_FUNC_sk_storage_delete:

tools/bpf/bpftool/Makefile+6−8

 LDFLAGS += $(EXTRA_LDFLAGS)
 endif
+HOST_CFLAGS := $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),\
+		$(subst $(CLANG_CROSS_FLAGS),,$(CFLAGS)))
+HOST_LDFLAGS := $(LDFLAGS)
 INSTALL ?= install
 RM ?= rm -f
   SRCS := $(filter-out jit_disasm.c,$(SRCS))
 endif
-HOST_CFLAGS = $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),\
-		$(subst $(CLANG_CROSS_FLAGS),,$(CFLAGS)))
 BPFTOOL_BOOTSTRAP := $(BOOTSTRAP_OUTPUT)bpftool
-BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o xlated_dumper.o btf_dumper.o disasm.o)
+BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o)
 $(BOOTSTRAP_OBJS): $(LIBBPF_BOOTSTRAP)
 OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o
 CFLAGS += $(if $(BUILD_BPF_SKELS),,-DBPFTOOL_WITHOUT_SKELETONS)
-$(BOOTSTRAP_OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
-	$(QUIET_CC)$(HOSTCC) $(HOST_CFLAGS) -c -MMD $< -o $@
 $(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
 	$(QUIET_CC)$(CC) $(CFLAGS) -c -MMD $< -o $@
 $(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP)
-	$(QUIET_LINK)$(HOSTCC) $(HOST_CFLAGS) $(LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@
+	$(QUIET_LINK)$(HOSTCC) $(HOST_CFLAGS) $(HOST_LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@
 $(OUTPUT)bpftool: $(OBJS) $(LIBBPF)
 	$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(OBJS) $(LIBS) -o $@

tools/bpf/bpftool/gen.c+2−1

 			continue;
 		codegen("\
 			\n\
-				obj->struct_ops.%1$s = bpf_map__initial_value(obj->maps.%1$s, NULL);\n\
+				obj->struct_ops.%1$s = (typeof(obj->struct_ops.%1$s))\n\
+					bpf_map__initial_value(obj->maps.%1$s, NULL);\n\
 			\n\
 			", ident);
 	}

tools/bpf/bpftool/pids.c+12−7

@@ -101,7 +101,6 @@ int build_obj_refs_table(struct hashmap **map, enum bpf_obj_type type) · 100 unmodified lines
101	char buf[4096 / sizeof(e) sizeof(*e)];	101	char buf[4096 / sizeof(e) sizeof(*e)];
102	struct pid_iter_bpf *skel;	102	struct pid_iter_bpf *skel;
103	int err, ret, fd = -1, i;	103	int err, ret, fd = -1, i;
104	libbpf_print_fn_t default_print;
105		104
106	*map = hashmap__new(hash_fn_for_key_as_id, equal_fn_for_key_as_id, NULL);	105	*map = hashmap__new(hash_fn_for_key_as_id, equal_fn_for_key_as_id, NULL);
107	if (IS_ERR(*map)) {	106	if (IS_ERR(*map)) {
@@ -118,12 +117,18 @@ int build_obj_refs_table(struct hashmap **map, enum bpf_obj_type type) · 10 unmodified lines
118		117
119	skel->rodata->obj_type = type;	118	skel->rodata->obj_type = type;
120		119
121	/* we don't want output polluted with libbpf errors if bpf_iter is not	120	if (!verifier_logs) {
122	* supported	121	libbpf_print_fn_t default_print;
123	*/	122
124	default_print = libbpf_set_print(libbpf_print_none);	123	/* Unless debug information is on, we don't want the output to
125	err = pid_iter_bpf__load(skel);	124	* be polluted with libbpf errors if bpf_iter is not supported.
126	libbpf_set_print(default_print);	125	*/
		126	default_print = libbpf_set_print(libbpf_print_none);
		127	err = pid_iter_bpf__load(skel);
		128	libbpf_set_print(default_print);
		129	} else {
		130	err = pid_iter_bpf__load(skel);
		131	}
127	if (err) {	132	if (err) {
128	/* too bad, kernel doesn't support BPF iterators yet */	133	/* too bad, kernel doesn't support BPF iterators yet */
129	err = 0;	134	err = 0;

tools/bpf/bpftool/skeleton/pid_iter.bpf.c+2−2

@@ -102,8 +102,8 @@ int iter(struct bpf_iter__task_file *ctx) · 101 unmodified lines
102	BPF_LINK_TYPE_PERF_EVENT___local)) {	102	BPF_LINK_TYPE_PERF_EVENT___local)) {
103	struct bpf_link link = (struct bpf_link ) file->private_data;	103	struct bpf_link link = (struct bpf_link ) file->private_data;
104		104
105	if (link->type == bpf_core_enum_value(enum bpf_link_type___local,	105	if (BPF_CORE_READ(link, type) == bpf_core_enum_value(enum bpf_link_type___local,
106	BPF_LINK_TYPE_PERF_EVENT___local)) {	106	BPF_LINK_TYPE_PERF_EVENT___local)) {
107	e.has_bpf_cookie = true;	107	e.has_bpf_cookie = true;
108	e.bpf_cookie = get_bpf_cookie(link);	108	e.bpf_cookie = get_bpf_cookie(link);
109	}	109	}

tools/include/linux/compiler.h+4−0

 #define noinline
 #endif
+#ifndef __nocf_check
+#define __nocf_check __attribute__((nocf_check))
+#endif
 /* Are two types/vars the same type (ignoring qualifiers)? */
 #ifndef __same_type
 # define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))

tools/include/uapi/linux/bpf.h+4−2

 	} query;
 	struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
-		__u64 name;
+		__u64		name;
-		__u32 prog_fd;
+		__u32		prog_fd;
+		__u32		:32;
+		__aligned_u64	cookie;
 	} raw_tracepoint;
 	struct { /* anonymous struct for BPF_BTF_LOAD */

tools/lib/bpf/bpf.c+14−2

 		if (!OPTS_ZEROED(opts, uprobe_multi))
 			return libbpf_err(-EINVAL);
 		break;
+	case BPF_TRACE_RAW_TP:
 	case BPF_TRACE_FENTRY:
 	case BPF_TRACE_FEXIT:
 	case BPF_MODIFY_RETURN:
 	return bpf_obj_get_info_by_fd(link_fd, info, info_len);
 }
-int bpf_raw_tracepoint_open(const char *name, int prog_fd)
+int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts)
 {
 	const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint);
 	union bpf_attr attr;
 	int fd;
+	if (!OPTS_VALID(opts, bpf_raw_tp_opts))
+		return libbpf_err(-EINVAL);
 	memset(&attr, 0, attr_sz);
-	attr.raw_tracepoint.name = ptr_to_u64(name);
 	attr.raw_tracepoint.prog_fd = prog_fd;
+	attr.raw_tracepoint.name = ptr_to_u64(OPTS_GET(opts, tp_name, NULL));
+	attr.raw_tracepoint.cookie = OPTS_GET(opts, cookie, 0);
 	fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
+int bpf_raw_tracepoint_open(const char *name, int prog_fd)
+{
+	LIBBPF_OPTS(bpf_raw_tp_opts, opts, .tp_name = name);
+	return bpf_raw_tracepoint_open_opts(prog_fd, &opts);
+}
 int bpf_btf_load(const void *btf_data, size_t btf_size, struct bpf_btf_load_opts *opts)
 {
 	const size_t attr_sz = offsetofend(union bpf_attr, btf_token_fd);

tools/lib/bpf/bpf.h+9−0

@@ -617,6 +617,15 @@ LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, · 616 unmodified lines
617	__u32 query_flags, __u32 *attach_flags,	617	__u32 query_flags, __u32 *attach_flags,
618	__u32 prog_ids, __u32 prog_cnt);	618	__u32 prog_ids, __u32 prog_cnt);
619		619
		620	struct bpf_raw_tp_opts {
		621	size_t sz; /* size of this struct for forward/backward compatibility */
		622	const char *tp_name;
		623	__u64 cookie;
		624	size_t :0;
		625	};
		626	#define bpf_raw_tp_opts__last_field cookie
		627
		628	LIBBPF_API int bpf_raw_tracepoint_open_opts(int prog_fd, struct bpf_raw_tp_opts *opts);
620	LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd);	629	LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd);
621	LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf,	630	LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf,
622	__u32 buf_len, __u32 prog_id, __u32 *fd_type,	631	__u32 buf_len, __u32 prog_id, __u32 *fd_type,

tools/lib/bpf/libbpf.c+92−19

 		const char *mname;
 		mname = btf__name_by_offset(btf, member->name_off);
+		moff = member->offset / 8;
+		mdata = data + moff;
+		msize = btf__resolve_size(btf, member->type);
+		if (msize < 0) {
+			pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n",
+				map->name, mname);
+			return msize;
+		}
 		kern_member = find_member_by_name(kern_btf, kern_type, mname);
 		if (!kern_member) {
+			/* Skip all zeros or null fields if they are not
+			 * presented in the kernel BTF.
+			 */
+			if (libbpf_is_mem_zeroed(mdata, msize)) {
+				pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n",
+					map->name, mname);
+				continue;
+			}
 			pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
 				map->name, mname);
 			return -ENOTSUP;
 			return -ENOTSUP;
 		}
-		moff = member->offset / 8;
 		kern_moff = kern_member->offset / 8;
-		mdata = data + moff;
 		kern_mdata = kern_data + kern_moff;
 		mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
 			continue;
 		}
-		msize = btf__resolve_size(btf, mtype_id);
 		kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
-		if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
+		if (kern_msize < 0 || msize != kern_msize) {
 			pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
 				map->name, mname, (ssize_t)msize,
 				(ssize_t)kern_msize);
 	char *cp, errmsg[STRERR_BUFSIZE];
 	size_t log_buf_size = 0;
 	char *log_buf = NULL, *tmp;
-	int btf_fd, ret, err;
 	bool own_log_buf = true;
 	__u32 log_level = prog->log_level;
+	int ret, err;
 	if (prog->type == BPF_PROG_TYPE_UNSPEC) {
 		/*
 	load_attr.prog_ifindex = prog->prog_ifindex;
 	/* specify func_info/line_info only if kernel supports them */
-	btf_fd = btf__fd(obj->btf);
+	if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
-	if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
+		load_attr.prog_btf_fd = btf__fd(obj->btf);
-		load_attr.prog_btf_fd = btf_fd;
 		load_attr.func_info = prog->func_info;
 		load_attr.func_info_rec_size = prog->func_info_rec_size;
 		load_attr.func_info_cnt = prog->func_info_cnt;
 		return libbpf_err(-EINVAL);
 	}
+	if (map->fd < 0) {
+		pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);
+		return libbpf_err(-EINVAL);
+	}
 	if (map->pin_path) {
 		if (path && strcmp(path, map->pin_path)) {
 			pr_warn("map '%s' already has pin path '%s' different from '%s'\n",
 	SEC_DEF("sockops",		SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT),
 	SEC_DEF("sk_skb/stream_parser",	SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT),
 	SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT),
+	SEC_DEF("sk_skb/verdict",	SK_SKB, BPF_SK_SKB_VERDICT, SEC_ATTACHABLE_OPT),
 	SEC_DEF("sk_skb",		SK_SKB, 0, SEC_NONE),
 	SEC_DEF("sk_msg",		SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT),
 	SEC_DEF("lirc_mode2",		LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT),
 		return -EINVAL;
 	}
+	if (map->fd < 0) {
+		pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
+		return -EINVAL;
+	}
 	if (!check_value_sz)
 		return 0;
 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
 {
 	int ret;
+	int prog_fd = bpf_program__fd(prog);
-	ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
+	if (prog_fd < 0) {
+		pr_warn("prog '%s': can't use BPF program without FD (was it loaded?)\n",
+			prog->name);
+		return libbpf_err(-EINVAL);
+	}
+	ret = bpf_link_update(bpf_link__fd(link), prog_fd, NULL);
 	return libbpf_err_errno(ret);
 }
 	}
 	prog_fd = bpf_program__fd(prog);
 	if (prog_fd < 0) {
-		pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
+		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
 			prog->name);
 		return libbpf_err_ptr(-EINVAL);
 	}
 	if (!OPTS_VALID(opts, bpf_kprobe_multi_opts))
 		return libbpf_err_ptr(-EINVAL);
+	prog_fd = bpf_program__fd(prog);
+	if (prog_fd < 0) {
+		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
+			prog->name);
+		return libbpf_err_ptr(-EINVAL);
+	}
 	syms    = OPTS_GET(opts, syms, false);
 	addrs   = OPTS_GET(opts, addrs, false);
 	cnt     = OPTS_GET(opts, cnt, false);
 	}
 	link->detach = &bpf_link__detach_fd;
-	prog_fd = bpf_program__fd(prog);
 	link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts);
 	if (link_fd < 0) {
 		err = -errno;
 	if (!OPTS_VALID(opts, bpf_uprobe_multi_opts))
 		return libbpf_err_ptr(-EINVAL);
+	prog_fd = bpf_program__fd(prog);
+	if (prog_fd < 0) {
+		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
+			prog->name);
+		return libbpf_err_ptr(-EINVAL);
+	}
 	syms = OPTS_GET(opts, syms, NULL);
 	offsets = OPTS_GET(opts, offsets, NULL);
 	ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL);
 	}
 	link->detach = &bpf_link__detach_fd;
-	prog_fd = bpf_program__fd(prog);
 	link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts);
 	if (link_fd < 0) {
 		err = -errno;
 		return libbpf_err_ptr(-EINVAL);
 	if (bpf_program__fd(prog) < 0) {
-		pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
+		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
 			prog->name);
 		return libbpf_err_ptr(-EINVAL);
 	}
 	return libbpf_get_error(*link);
 }
-struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
+struct bpf_link *
-						    const char *tp_name)
+bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
+					const char *tp_name,
+					struct bpf_raw_tracepoint_opts *opts)
 {
+	LIBBPF_OPTS(bpf_raw_tp_opts, raw_opts);
 	char errmsg[STRERR_BUFSIZE];
 	struct bpf_link *link;
 	int prog_fd, pfd;
+	if (!OPTS_VALID(opts, bpf_raw_tracepoint_opts))
+		return libbpf_err_ptr(-EINVAL);
 	prog_fd = bpf_program__fd(prog);
 	if (prog_fd < 0) {
 		pr_warn("prog '%s': can't attach before loaded\n", prog->name);
 		return libbpf_err_ptr(-ENOMEM);
 	link->detach = &bpf_link__detach_fd;
-	pfd = bpf_raw_tracepoint_open(tp_name, prog_fd);
+	raw_opts.tp_name = tp_name;
+	raw_opts.cookie = OPTS_GET(opts, cookie, 0);
+	pfd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_opts);
 	if (pfd < 0) {
 		pfd = -errno;
 		free(link);
 	return link;
 }
+struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
+						    const char *tp_name)
+{
+	return bpf_program__attach_raw_tracepoint_opts(prog, tp_name, NULL);
+}
 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
 {
 	static const char *const prefixes[] = {
 	if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
 		return libbpf_err_ptr(-EOPNOTSUPP);
+	if (bpf_program__fd(prog) < 0) {
+		pr_warn("prog '%s': can't attach BPF program without FD (was it loaded?)\n",
+			prog->name);
+		return libbpf_err_ptr(-EINVAL);
+	}
 	err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
 	if (err)
 		return libbpf_err_ptr(err);
 	__u32 zero = 0;
 	int err, fd;
-	if (!bpf_map__is_struct_ops(map) || map->fd == -1)
+	if (!bpf_map__is_struct_ops(map))
+		return libbpf_err_ptr(-EINVAL);
+	if (map->fd < 0) {
+		pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);
 		return libbpf_err_ptr(-EINVAL);
+	}
 	link = calloc(1, sizeof(*link));
 	if (!link)
 	__u32 zero = 0;
 	int err;
-	if (!bpf_map__is_struct_ops(map) || !map_is_created(map))
+	if (!bpf_map__is_struct_ops(map))
 		return -EINVAL;
+	if (map->fd < 0) {
+		pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
+		return -EINVAL;
+	}
 	st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
 	/* Ensure the type of a link is correct */
 	if (st_ops_link->map_fd < 0)

tools/lib/bpf/libbpf.h+11−0

@@ -760,9 +760,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, · 759 unmodified lines
760	const char *tp_name,	760	const char *tp_name,
761	const struct bpf_tracepoint_opts *opts);	761	const struct bpf_tracepoint_opts *opts);
762		762
		763	struct bpf_raw_tracepoint_opts {
		764	size_t sz; /* size of this struct for forward/backward compatibility */
		765	__u64 cookie;
		766	size_t :0;
		767	};
		768	#define bpf_raw_tracepoint_opts__last_field cookie
		769
763	LIBBPF_API struct bpf_link *	770	LIBBPF_API struct bpf_link *
764	bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,	771	bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
765	const char *tp_name);	772	const char *tp_name);
		773	LIBBPF_API struct bpf_link *
		774	bpf_program__attach_raw_tracepoint_opts(const struct bpf_program *prog,
		775	const char *tp_name,
		776	struct bpf_raw_tracepoint_opts *opts);
766		777
767	struct bpf_trace_opts {	778	struct bpf_trace_opts {
768	/* size of this struct, for forward/backward compatibility */	779	/* size of this struct, for forward/backward compatibility */

tools/lib/bpf/libbpf.map+2−0

@@ -410,6 +410,8 @@ LIBBPF_1.3.0 { · 409 unmodified lines
410		410
411	LIBBPF_1.4.0 {	411	LIBBPF_1.4.0 {
412	global:	412	global:
		413	bpf_program__attach_raw_tracepoint_opts;
		414	bpf_raw_tracepoint_open_opts;
413	bpf_token_create;	415	bpf_token_create;
414	btf__new_split;	416	btf__new_split;
415	btf_ext__raw_data;	417	btf_ext__raw_data;

tools/testing/selftests/bpf/benchs/bench_local_storage_create.c+1−1

@@ -186,7 +186,7 @@ static void task_producer(void input) · 185 unmodified lines
186		186
187	for (i = 0; i < batch_sz; i++) {	187	for (i = 0; i < batch_sz; i++) {
188	if (!pthd_results[i])	188	if (!pthd_results[i])
189	pthread_join(pthds[i], NULL);;	189	pthread_join(pthds[i], NULL);
190	}	190	}
191	}	191	}
192		192

tools/testing/selftests/bpf/benchs/bench_trigger.c+40−8

 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2020 Facebook */
+#define _GNU_SOURCE
+#include <unistd.h>
 #include "bench.h"
 #include "trigger_bench.skel.h"
 #include "trace_helpers.h"
+/* adjust slot shift in inc_hits() if changing */
+#define MAX_BUCKETS 256
+#pragma GCC diagnostic ignored "-Wattributes"
 /* BPF triggering benchmarks */
 static struct trigger_ctx {
 	struct trigger_bench *skel;
 } ctx;
-static struct counter base_hits;
+static struct counter base_hits[MAX_BUCKETS];
+static __always_inline void inc_counter(struct counter *counters)
+{
+	static __thread int tid = 0;
+	unsigned slot;
+	if (unlikely(tid == 0))
+		tid = syscall(SYS_gettid);
+	/* multiplicative hashing, it's fast */
+	slot = 2654435769U * tid;
+	slot >>= 24;
+	atomic_inc(&base_hits[slot].value); /* use highest byte as an index */
+}
+static long sum_and_reset_counters(struct counter *counters)
+{
+	int i;
+	long sum = 0;
+	for (i = 0; i < MAX_BUCKETS; i++)
+		sum += atomic_swap(&counters[i].value, 0);
+	return sum;
+}
 static void trigger_validate(void)
 {
 {
 	while (true) {
 		(void)syscall(__NR_getpgid);
-		atomic_inc(&base_hits.value);
+		inc_counter(base_hits);
 	}
 	return NULL;
 }
 static void trigger_base_measure(struct bench_res *res)
 {
-	res->hits = atomic_swap(&base_hits.value, 0);
+	res->hits = sum_and_reset_counters(base_hits);
 }
 static void *trigger_producer(void *input)
 static void trigger_measure(struct bench_res *res)
 {
-	res->hits = atomic_swap(&ctx.skel->bss->hits, 0);
+	res->hits = sum_and_reset_counters(ctx.skel->bss->hits);
 }
 static void setup_ctx(void)
  * GCC doesn't generate stack setup preample for these functions due to them
  * having no input arguments and doing nothing in the body.
  */
-__weak void uprobe_target_nop(void)
+__nocf_check __weak void uprobe_target_nop(void)
 {
 	asm volatile ("nop");
 }
 {
 }
-__weak int uprobe_target_push(void)
+__nocf_check __weak int uprobe_target_push(void)
 {
 	/* overhead of function call is negligible compared to uprobe
 	 * triggering, so this shouldn't affect benchmark results much
 	return 1;
 }
-__weak void uprobe_target_ret(void)
+__nocf_check __weak void uprobe_target_ret(void)
 {
 	asm volatile ("");
 }
 {
 	while (true) {
 		uprobe_target_nop();
-		atomic_inc(&base_hits.value);
+		inc_counter(base_hits);
 	}
 	return NULL;
 }

tools/testing/selftests/bpf/prog_tests/bpf_cookie.c+113−1

@@ -573,6 +573,115 @@ cleanup: · 572 unmodified lines
573	close(lsm_fd);	573	close(lsm_fd);
574	}	574	}
575		575
		576	static void tp_btf_subtest(struct test_bpf_cookie *skel)
		577	{
		578	__u64 cookie;
		579	int prog_fd, link_fd = -1;
		580	struct bpf_link *link = NULL;
		581	LIBBPF_OPTS(bpf_link_create_opts, link_opts);
		582	LIBBPF_OPTS(bpf_raw_tp_opts, raw_tp_opts);
		583	LIBBPF_OPTS(bpf_trace_opts, trace_opts);
		584
		585	/* There are three different ways to attach tp_btf (BTF-aware raw
		586	* tracepoint) programs. Let's test all of them.
		587	*/
		588	prog_fd = bpf_program__fd(skel->progs.handle_tp_btf);
		589
		590	/* low-level BPF_RAW_TRACEPOINT_OPEN command wrapper */
		591	skel->bss->tp_btf_res = 0;
		592
		593	raw_tp_opts.cookie = cookie = 0x11000000000000L;
		594	link_fd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_tp_opts);
		595	if (!ASSERT_GE(link_fd, 0, "bpf_raw_tracepoint_open_opts"))
		596	goto cleanup;
		597
		598	usleep(1); /* trigger */
		599	close(link_fd); /* detach */
		600	link_fd = -1;
		601
		602	ASSERT_EQ(skel->bss->tp_btf_res, cookie, "raw_tp_open_res");
		603
		604	/* low-level generic bpf_link_create() API */
		605	skel->bss->tp_btf_res = 0;
		606
		607	link_opts.tracing.cookie = cookie = 0x22000000000000L;
		608	link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_RAW_TP, &link_opts);
		609	if (!ASSERT_GE(link_fd, 0, "bpf_link_create"))
		610	goto cleanup;
		611
		612	usleep(1); /* trigger */
		613	close(link_fd); /* detach */
		614	link_fd = -1;
		615
		616	ASSERT_EQ(skel->bss->tp_btf_res, cookie, "link_create_res");
		617
		618	/* high-level bpf_link-based bpf_program__attach_trace_opts() API */
		619	skel->bss->tp_btf_res = 0;
		620
		621	trace_opts.cookie = cookie = 0x33000000000000L;
		622	link = bpf_program__attach_trace_opts(skel->progs.handle_tp_btf, &trace_opts);
		623	if (!ASSERT_OK_PTR(link, "attach_trace_opts"))
		624	goto cleanup;
		625
		626	usleep(1); /* trigger */
		627	bpf_link__destroy(link); /* detach */
		628	link = NULL;
		629
		630	ASSERT_EQ(skel->bss->tp_btf_res, cookie, "attach_trace_opts_res");
		631
		632	cleanup:
		633	if (link_fd >= 0)
		634	close(link_fd);
		635	bpf_link__destroy(link);
		636	}
		637
		638	static void raw_tp_subtest(struct test_bpf_cookie *skel)
		639	{
		640	__u64 cookie;
		641	int prog_fd, link_fd = -1;
		642	struct bpf_link *link = NULL;
		643	LIBBPF_OPTS(bpf_raw_tp_opts, raw_tp_opts);
		644	LIBBPF_OPTS(bpf_raw_tracepoint_opts, opts);
		645
		646	/* There are two different ways to attach raw_tp programs */
		647	prog_fd = bpf_program__fd(skel->progs.handle_raw_tp);
		648
		649	/* low-level BPF_RAW_TRACEPOINT_OPEN command wrapper */
		650	skel->bss->raw_tp_res = 0;
		651
		652	raw_tp_opts.tp_name = "sys_enter";
		653	raw_tp_opts.cookie = cookie = 0x55000000000000L;
		654	link_fd = bpf_raw_tracepoint_open_opts(prog_fd, &raw_tp_opts);
		655	if (!ASSERT_GE(link_fd, 0, "bpf_raw_tracepoint_open_opts"))
		656	goto cleanup;
		657
		658	usleep(1); /* trigger */
		659	close(link_fd); /* detach */
		660	link_fd = -1;
		661
		662	ASSERT_EQ(skel->bss->raw_tp_res, cookie, "raw_tp_open_res");
		663
		664	/* high-level bpf_link-based bpf_program__attach_raw_tracepoint_opts() API */
		665	skel->bss->raw_tp_res = 0;
		666
		667	opts.cookie = cookie = 0x66000000000000L;
		668	link = bpf_program__attach_raw_tracepoint_opts(skel->progs.handle_raw_tp,
		669	"sys_enter", &opts);
		670	if (!ASSERT_OK_PTR(link, "attach_raw_tp_opts"))
		671	goto cleanup;
		672
		673	usleep(1); /* trigger */
		674	bpf_link__destroy(link); /* detach */
		675	link = NULL;
		676
		677	ASSERT_EQ(skel->bss->raw_tp_res, cookie, "attach_raw_tp_opts_res");
		678
		679	cleanup:
		680	if (link_fd >= 0)
		681	close(link_fd);
		682	bpf_link__destroy(link);
		683	}
		684
576	void test_bpf_cookie(void)	685	void test_bpf_cookie(void)
577	{	686	{
578	struct test_bpf_cookie *skel;	687	struct test_bpf_cookie *skel;
@@ -601,6 +710,9 @@ void test_bpf_cookie(void) · 22 unmodified lines
601	tracing_subtest(skel);	710	tracing_subtest(skel);
602	if (test__start_subtest("lsm"))	711	if (test__start_subtest("lsm"))
603	lsm_subtest(skel);	712	lsm_subtest(skel);
604		713	if (test__start_subtest("tp_btf"))
		714	tp_btf_subtest(skel);
		715	if (test__start_subtest("raw_tp"))
		716	raw_tp_subtest(skel);
605	test_bpf_cookie__destroy(skel);	717	test_bpf_cookie__destroy(skel);
606	}	718	}

tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+1−12

 	WRITE_ONCE(stop, 0);
-	lfd = socket(AF_INET6, SOCK_STREAM, 0);
+	lfd = start_server(AF_INET6, SOCK_STREAM, NULL, 0, 0);
 	if (!ASSERT_NEQ(lfd, -1, "socket"))
 		return;
 	    settimeo(lfd, 0) || settimeo(fd, 0))
 		goto done;
-	/* bind, listen and start server thread to accept */
-	sa6.sin6_family = AF_INET6;
-	sa6.sin6_addr = in6addr_loopback;
-	err = bind(lfd, (struct sockaddr *)&sa6, addrlen);
-	if (!ASSERT_NEQ(err, -1, "bind"))
-		goto done;
 	err = getsockname(lfd, (struct sockaddr *)&sa6, &addrlen);
 	if (!ASSERT_NEQ(err, -1, "getsockname"))
 		goto done;
-	err = listen(lfd, 1);
-	if (!ASSERT_NEQ(err, -1, "listen"))
-		goto done;
 	if (sk_stg_map) {
 		err = bpf_map_update_elem(bpf_map__fd(sk_stg_map), &fd,
 					  &expected_stg, BPF_NOEXIST);

tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c+1−25

@@ -25,7 +25,7 @@ static void test_lookup_update(void) · 24 unmodified lines
25	int map1_fd, map2_fd, map3_fd, map4_fd, map5_fd, map1_id, map2_id;	25	int map1_fd, map2_fd, map3_fd, map4_fd, map5_fd, map1_id, map2_id;
26	int outer_arr_fd, outer_hash_fd, outer_arr_dyn_fd;	26	int outer_arr_fd, outer_hash_fd, outer_arr_dyn_fd;
27	struct test_btf_map_in_map *skel;	27	struct test_btf_map_in_map *skel;
28	int err, key = 0, val, i, fd;	28	int err, key = 0, val, i;
29		29
30	skel = test_btf_map_in_map__open_and_load();	30	skel = test_btf_map_in_map__open_and_load();
31	if (CHECK(!skel, "skel_open", "failed to open&load skeleton\n"))	31	if (CHECK(!skel, "skel_open", "failed to open&load skeleton\n"))
@@ -102,30 +102,6 @@ static void test_lookup_update(void) · 70 unmodified lines
102	CHECK(map1_id == 0, "map1_id", "failed to get ID 1\n");	102	CHECK(map1_id == 0, "map1_id", "failed to get ID 1\n");
103	CHECK(map2_id == 0, "map2_id", "failed to get ID 2\n");	103	CHECK(map2_id == 0, "map2_id", "failed to get ID 2\n");
104		104
105	test_btf_map_in_map__destroy(skel);
106	skel = NULL;
107
108	/* we need to either wait for or force synchronize_rcu(), before
109	* checking for "still exists" condition, otherwise map could still be
110	* resolvable by ID, causing false positives.
111	*
112	* Older kernels (5.8 and earlier) freed map only after two
113	* synchronize_rcu()s, so trigger two, to be entirely sure.
114	*/
115	CHECK(kern_sync_rcu(), "sync_rcu", "failed\n");
116	CHECK(kern_sync_rcu(), "sync_rcu", "failed\n");
117
118	fd = bpf_map_get_fd_by_id(map1_id);
119	if (CHECK(fd >= 0, "map1_leak", "inner_map1 leaked!\n")) {
120	close(fd);
121	goto cleanup;
122	}
123	fd = bpf_map_get_fd_by_id(map2_id);
124	if (CHECK(fd >= 0, "map2_leak", "inner_map2 leaked!\n")) {
125	close(fd);
126	goto cleanup;
127	}
128
129	cleanup:	105	cleanup:
130	test_btf_map_in_map__destroy(skel);	106	test_btf_map_in_map__destroy(skel);
131	}	107	}

tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c+183−31

@@ -12,77 +12,229 @@ · 11 unmodified lines
12	#include <sys/wait.h>	12	#include <sys/wait.h>
13	#include <sys/mount.h>	13	#include <sys/mount.h>
14	#include <sys/fcntl.h>	14	#include <sys/fcntl.h>
		15	#include "network_helpers.h"
15		16
16	#define STACK_SIZE (1024 * 1024)	17	#define STACK_SIZE (1024 * 1024)
17	static char child_stack[STACK_SIZE];	18	static char child_stack[STACK_SIZE];
18		19
19	static int test_current_pid_tgid(void *args)	20	static int get_pid_tgid(pid_t pid, pid_t tgid,
		21	struct test_ns_current_pid_tgid__bss *bss)
20	{	22	{
21	struct test_ns_current_pid_tgid__bss *bss;
22	struct test_ns_current_pid_tgid *skel;
23	int err = -1, duration = 0;
24	pid_t tgid, pid;
25	struct stat st;	23	struct stat st;
		24	int err;
26		25
27	skel = test_ns_current_pid_tgid__open_and_load();	26	*pid = syscall(SYS_gettid);
28	if (CHECK(!skel, "skel_open_load", "failed to load skeleton\n"))	27	*tgid = getpid();
29	goto cleanup;
30
31	pid = syscall(SYS_gettid);
32	tgid = getpid();
33		28
34	err = stat("/proc/self/ns/pid", &st);	29	err = stat("/proc/self/ns/pid", &st);
35	if (CHECK(err, "stat", "failed /proc/self/ns/pid: %d\n", err))	30	if (!ASSERT_OK(err, "stat /proc/self/ns/pid"))
36	goto cleanup;	31	return err;
37		32
38	bss = skel->bss;
39	bss->dev = st.st_dev;	33	bss->dev = st.st_dev;
40	bss->ino = st.st_ino;	34	bss->ino = st.st_ino;
41	bss->user_pid = 0;	35	bss->user_pid = 0;
42	bss->user_tgid = 0;	36	bss->user_tgid = 0;
		37	return 0;
		38	}
		39
		40	static int test_current_pid_tgid_tp(void *args)
		41	{
		42	struct test_ns_current_pid_tgid__bss *bss;
		43	struct test_ns_current_pid_tgid *skel;
		44	int ret = -1, err;
		45	pid_t tgid, pid;
		46
		47	skel = test_ns_current_pid_tgid__open();
		48	if (!ASSERT_OK_PTR(skel, "test_ns_current_pid_tgid__open"))
		49	return ret;
		50
		51	bpf_program__set_autoload(skel->progs.tp_handler, true);
		52
		53	err = test_ns_current_pid_tgid__load(skel);
		54	if (!ASSERT_OK(err, "test_ns_current_pid_tgid__load"))
		55	goto cleanup;
		56
		57	bss = skel->bss;
		58	if (get_pid_tgid(&pid, &tgid, bss))
		59	goto cleanup;
43		60
44	err = test_ns_current_pid_tgid__attach(skel);	61	err = test_ns_current_pid_tgid__attach(skel);
45	if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))	62	if (!ASSERT_OK(err, "test_ns_current_pid_tgid__attach"))
46	goto cleanup;	63	goto cleanup;
47		64
48	/* trigger tracepoint */	65	/* trigger tracepoint */
49	usleep(1);	66	usleep(1);
50	ASSERT_EQ(bss->user_pid, pid, "pid");	67	if (!ASSERT_EQ(bss->user_pid, pid, "pid"))
51	ASSERT_EQ(bss->user_tgid, tgid, "tgid");	68	goto cleanup;
52	err = 0;	69	if (!ASSERT_EQ(bss->user_tgid, tgid, "tgid"))
		70	goto cleanup;
		71	ret = 0;
		72
		73	cleanup:
		74	test_ns_current_pid_tgid__destroy(skel);
		75	return ret;
		76	}
		77
		78	static int test_current_pid_tgid_cgrp(void *args)
		79	{
		80	struct test_ns_current_pid_tgid__bss *bss;
		81	struct test_ns_current_pid_tgid *skel;
		82	int server_fd = -1, ret = -1, err;
		83	int cgroup_fd = (int )args;
		84	pid_t tgid, pid;
		85
		86	skel = test_ns_current_pid_tgid__open();
		87	if (!ASSERT_OK_PTR(skel, "test_ns_current_pid_tgid__open"))
		88	return ret;
		89
		90	bpf_program__set_autoload(skel->progs.cgroup_bind4, true);
		91
		92	err = test_ns_current_pid_tgid__load(skel);
		93	if (!ASSERT_OK(err, "test_ns_current_pid_tgid__load"))
		94	goto cleanup;
		95
		96	bss = skel->bss;
		97	if (get_pid_tgid(&pid, &tgid, bss))
		98	goto cleanup;
		99
		100	skel->links.cgroup_bind4 = bpf_program__attach_cgroup(
		101	skel->progs.cgroup_bind4, cgroup_fd);
		102	if (!ASSERT_OK_PTR(skel->links.cgroup_bind4, "bpf_program__attach_cgroup"))
		103	goto cleanup;
		104
		105	server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
		106	if (!ASSERT_GE(server_fd, 0, "start_server"))
		107	goto cleanup;
		108
		109	if (!ASSERT_EQ(bss->user_pid, pid, "pid"))
		110	goto cleanup;
		111	if (!ASSERT_EQ(bss->user_tgid, tgid, "tgid"))
		112	goto cleanup;
		113	ret = 0;
53		114
54	cleanup:	115	cleanup:
55	test_ns_current_pid_tgid__destroy(skel);	116	if (server_fd >= 0)
		117	close(server_fd);
		118	test_ns_current_pid_tgid__destroy(skel);
		119	return ret;
		120	}
		121
		122	static int test_current_pid_tgid_sk_msg(void *args)
		123	{
		124	int verdict, map, server_fd = -1, client_fd = -1;
		125	struct test_ns_current_pid_tgid__bss *bss;
		126	static const char send_msg[] = "message";
		127	struct test_ns_current_pid_tgid *skel;
		128	int ret = -1, err, key = 0;
		129	pid_t tgid, pid;
		130
		131	skel = test_ns_current_pid_tgid__open();
		132	if (!ASSERT_OK_PTR(skel, "test_ns_current_pid_tgid__open"))
		133	return ret;
		134
		135	bpf_program__set_autoload(skel->progs.sk_msg, true);
		136
		137	err = test_ns_current_pid_tgid__load(skel);
		138	if (!ASSERT_OK(err, "test_ns_current_pid_tgid__load"))
		139	goto cleanup;
		140
		141	bss = skel->bss;
		142	if (get_pid_tgid(&pid, &tgid, skel->bss))
		143	goto cleanup;
		144
		145	verdict = bpf_program__fd(skel->progs.sk_msg);
		146	map = bpf_map__fd(skel->maps.sock_map);
		147	err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
		148	if (!ASSERT_OK(err, "prog_attach"))
		149	goto cleanup;
		150
		151	server_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
		152	if (!ASSERT_GE(server_fd, 0, "start_server"))
		153	goto cleanup;
56		154
57	return err;	155	client_fd = connect_to_fd(server_fd, 0);
		156	if (!ASSERT_GE(client_fd, 0, "connect_to_fd"))
		157	goto cleanup;
		158
		159	err = bpf_map_update_elem(map, &key, &client_fd, BPF_ANY);
		160	if (!ASSERT_OK(err, "bpf_map_update_elem"))
		161	goto cleanup;
		162
		163	err = send(client_fd, send_msg, sizeof(send_msg), 0);
		164	if (!ASSERT_EQ(err, sizeof(send_msg), "send(msg)"))
		165	goto cleanup;
		166
		167	if (!ASSERT_EQ(bss->user_pid, pid, "pid"))
		168	goto cleanup;
		169	if (!ASSERT_EQ(bss->user_tgid, tgid, "tgid"))
		170	goto cleanup;
		171	ret = 0;
		172
		173	cleanup:
		174	if (server_fd >= 0)
		175	close(server_fd);
		176	if (client_fd >= 0)
		177	close(client_fd);
		178	test_ns_current_pid_tgid__destroy(skel);
		179	return ret;
58	}	180	}
59		181
60	static void test_ns_current_pid_tgid_new_ns(void)	182	static void test_ns_current_pid_tgid_new_ns(int (fn)(void ), void *arg)
61	{	183	{
62	int wstatus, duration = 0;	184	int wstatus;
63	pid_t cpid;	185	pid_t cpid;
64		186
65	/* Create a process in a new namespace, this process	187	/* Create a process in a new namespace, this process
66	* will be the init process of this new namespace hence will be pid 1.	188	* will be the init process of this new namespace hence will be pid 1.
67	*/	189	*/
68	cpid = clone(test_current_pid_tgid, child_stack + STACK_SIZE,	190	cpid = clone(fn, child_stack + STACK_SIZE,
69	CLONE_NEWPID \| SIGCHLD, NULL);	191	CLONE_NEWPID \| SIGCHLD, arg);
70		192
71	if (CHECK(cpid == -1, "clone", "%s\n", strerror(errno)))	193	if (!ASSERT_NEQ(cpid, -1, "clone"))
72	return;	194	return;
73		195
74	if (CHECK(waitpid(cpid, &wstatus, 0) == -1, "waitpid", "%s\n", strerror(errno)))	196	if (!ASSERT_NEQ(waitpid(cpid, &wstatus, 0), -1, "waitpid"))
75	return;	197	return;
76		198
77	if (CHECK(WEXITSTATUS(wstatus) != 0, "newns_pidtgid", "failed"))	199	if (!ASSERT_OK(WEXITSTATUS(wstatus), "newns_pidtgid"))
78	return;	200	return;
79	}	201	}
80		202
		203	static void test_in_netns(int (fn)(void ), void *arg)
		204	{
		205	struct nstoken *nstoken = NULL;
		206
		207	SYS(cleanup, "ip netns add ns_current_pid_tgid");
		208	SYS(cleanup, "ip -net ns_current_pid_tgid link set dev lo up");
		209
		210	nstoken = open_netns("ns_current_pid_tgid");
		211	if (!ASSERT_OK_PTR(nstoken, "open_netns"))
		212	goto cleanup;
		213
		214	test_ns_current_pid_tgid_new_ns(fn, arg);
		215
		216	cleanup:
		217	if (nstoken)
		218	close_netns(nstoken);
		219	SYS_NOFAIL("ip netns del ns_current_pid_tgid");
		220	}
		221
81	/* TODO: use a different tracepoint */	222	/* TODO: use a different tracepoint */
82	void serial_test_ns_current_pid_tgid(void)	223	void serial_test_ns_current_pid_tgid(void)
83	{	224	{
84	if (test__start_subtest("ns_current_pid_tgid_root_ns"))	225	if (test__start_subtest("root_ns_tp"))
85	test_current_pid_tgid(NULL);	226	test_current_pid_tgid_tp(NULL);
86	if (test__start_subtest("ns_current_pid_tgid_new_ns"))	227	if (test__start_subtest("new_ns_tp"))
87	test_ns_current_pid_tgid_new_ns();	228	test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_tp, NULL);
		229	if (test__start_subtest("new_ns_cgrp")) {
		230	int cgroup_fd = -1;
		231
		232	cgroup_fd = test__join_cgroup("/sock_addr");
		233	if (ASSERT_GE(cgroup_fd, 0, "join_cgroup")) {
		234	test_in_netns(test_current_pid_tgid_cgrp, &cgroup_fd);
		235	close(cgroup_fd);
		236	}
		237	}
		238	if (test__start_subtest("new_ns_sk_msg"))
		239	test_in_netns(test_current_pid_tgid_sk_msg, NULL);
88	}	240	}

tools/testing/selftests/bpf/prog_tests/test_struct_ops_module.c+47−0

@@ -93,9 +93,56 @@ cleanup: · 92 unmodified lines
93	struct_ops_module__destroy(skel);	93	struct_ops_module__destroy(skel);
94	}	94	}
95		95
		96	static void test_struct_ops_not_zeroed(void)
		97	{
		98	struct struct_ops_module *skel;
		99	int err;
		100
		101	/* zeroed is 0, and zeroed_op is null */
		102	skel = struct_ops_module__open();
		103	if (!ASSERT_OK_PTR(skel, "struct_ops_module_open"))
		104	return;
		105
		106	err = struct_ops_module__load(skel);
		107	ASSERT_OK(err, "struct_ops_module_load");
		108
		109	struct_ops_module__destroy(skel);
		110
		111	/* zeroed is not 0 */
		112	skel = struct_ops_module__open();
		113	if (!ASSERT_OK_PTR(skel, "struct_ops_module_open_not_zeroed"))
		114	return;
		115
		116	/* libbpf should reject the testmod_zeroed since struct
		117	* bpf_testmod_ops in the kernel has no "zeroed" field and the
		118	* value of "zeroed" is non-zero.
		119	*/
		120	skel->struct_ops.testmod_zeroed->zeroed = 0xdeadbeef;
		121	err = struct_ops_module__load(skel);
		122	ASSERT_ERR(err, "struct_ops_module_load_not_zeroed");
		123
		124	struct_ops_module__destroy(skel);
		125
		126	/* zeroed_op is not null */
		127	skel = struct_ops_module__open();
		128	if (!ASSERT_OK_PTR(skel, "struct_ops_module_open_not_zeroed_op"))
		129	return;
		130
		131	/* libbpf should reject the testmod_zeroed since the value of its
		132	* "zeroed_op" is not null.
		133	*/
		134	skel->struct_ops.testmod_zeroed->zeroed_op = skel->progs.test_3;
		135	err = struct_ops_module__load(skel);
		136	ASSERT_ERR(err, "struct_ops_module_load_not_zeroed_op");
		137
		138	struct_ops_module__destroy(skel);
		139	}
		140
96	void serial_test_struct_ops_module(void)	141	void serial_test_struct_ops_module(void)
97	{	142	{
98	if (test__start_subtest("test_struct_ops_load"))	143	if (test__start_subtest("test_struct_ops_load"))
99	test_struct_ops_load();	144	test_struct_ops_load();
		145	if (test__start_subtest("test_struct_ops_not_zeroed"))
		146	test_struct_ops_not_zeroed();
100	}	147	}
101		148

tools/testing/selftests/bpf/progs/iters.c+1−1

@@ -673,7 +673,7 @@ static __noinline void fill(struct bpf_iter_num it, int arr, __u32 n, int mul) · 672 unmodified lines
673		673
674	static __noinline int sum(struct bpf_iter_num it, int arr, __u32 n)	674	static __noinline int sum(struct bpf_iter_num it, int arr, __u32 n)
675	{	675	{
676	int *t, i, sum = 0;;	676	int *t, i, sum = 0;
677		677
678	while ((t = bpf_iter_num_next(it))) {	678	while ((t = bpf_iter_num_next(it))) {
679	i = *t;	679	i = *t;

tools/testing/selftests/bpf/progs/struct_ops_module.c+15−1

@@ -23,7 +23,7 @@ void BPF_PROG(test_2, int a, int b) · 22 unmodified lines
23	test_2_result = a + b;	23	test_2_result = a + b;
24	}	24	}
25		25
26	SEC("struct_ops/test_3")	26	SEC("?struct_ops/test_3")
27	int BPF_PROG(test_3, int a, int b)	27	int BPF_PROG(test_3, int a, int b)
28	{	28	{
29	test_2_result = a + b + 3;	29	test_2_result = a + b + 3;
@@ -54,3 +54,17 @@ struct bpf_testmod_ops___v2 testmod_2 = { · 24 unmodified lines
54	.test_1 = (void *)test_1,	54	.test_1 = (void *)test_1,
55	.test_2 = (void *)test_2_v2,	55	.test_2 = (void *)test_2_v2,
56	};	56	};
		57
		58	struct bpf_testmod_ops___zeroed {
		59	int (*test_1)(void);
		60	void (*test_2)(int a, int b);
		61	int (test_maybe_null)(int dummy, struct task_struct task);
		62	void (*zeroed_op)(int a, int b);
		63	int zeroed;
		64	};
		65
		66	SEC(".struct_ops.link")
		67	struct bpf_testmod_ops___zeroed testmod_zeroed = {
		68	.test_1 = (void *)test_1,
		69	.test_2 = (void *)test_2_v2,
		70	};

tools/testing/selftests/bpf/progs/test_bpf_cookie.c+16−0

@@ -15,6 +15,8 @@ __u64 uprobe_res; · 14 unmodified lines
15	__u64 uretprobe_res;	15	__u64 uretprobe_res;
16	__u64 tp_res;	16	__u64 tp_res;
17	__u64 pe_res;	17	__u64 pe_res;
		18	__u64 raw_tp_res;
		19	__u64 tp_btf_res;
18	__u64 fentry_res;	20	__u64 fentry_res;
19	__u64 fexit_res;	21	__u64 fexit_res;
20	__u64 fmod_ret_res;	22	__u64 fmod_ret_res;
@@ -87,6 +89,20 @@ int handle_pe(struct pt_regs *ctx) · 66 unmodified lines
87	return 0;	89	return 0;
88	}	90	}
89		91
		92	SEC("raw_tp/sys_enter")
		93	int handle_raw_tp(void *ctx)
		94	{
		95	update(ctx, &raw_tp_res);
		96	return 0;
		97	}
		98
		99	SEC("tp_btf/sys_enter")
		100	int handle_tp_btf(void *ctx)
		101	{
		102	update(ctx, &tp_btf_res);
		103	return 0;
		104	}
		105
90	SEC("fentry/bpf_fentry_test1")	106	SEC("fentry/bpf_fentry_test1")
91	int BPF_PROG(fentry_test1, int a)	107	int BPF_PROG(fentry_test1, int a)
92	{	108	{

tools/testing/selftests/bpf/progs/test_ns_current_pid_tgid.c+28−3

@@ -5,23 +5,48 @@ · 4 unmodified lines
5	#include <stdint.h>	5	#include <stdint.h>
6	#include <bpf/bpf_helpers.h>	6	#include <bpf/bpf_helpers.h>
7		7
		8	struct {
		9	__uint(type, BPF_MAP_TYPE_SOCKMAP);
		10	__uint(max_entries, 2);
		11	__type(key, __u32);
		12	__type(value, __u32);
		13	} sock_map SEC(".maps");
		14
8	__u64 user_pid = 0;	15	__u64 user_pid = 0;
9	__u64 user_tgid = 0;	16	__u64 user_tgid = 0;
10	__u64 dev = 0;	17	__u64 dev = 0;
11	__u64 ino = 0;	18	__u64 ino = 0;
12		19
13	SEC("tracepoint/syscalls/sys_enter_nanosleep")	20	static void get_pid_tgid(void)
14	int handler(const void *ctx)
15	{	21	{
16	struct bpf_pidns_info nsdata;	22	struct bpf_pidns_info nsdata;
17		23
18	if (bpf_get_ns_current_pid_tgid(dev, ino, &nsdata, sizeof(struct bpf_pidns_info)))	24	if (bpf_get_ns_current_pid_tgid(dev, ino, &nsdata, sizeof(struct bpf_pidns_info)))
19	return 0;	25	return;
20		26
21	user_pid = nsdata.pid;	27	user_pid = nsdata.pid;
22	user_tgid = nsdata.tgid;	28	user_tgid = nsdata.tgid;
		29	}
23		30
		31	SEC("?tracepoint/syscalls/sys_enter_nanosleep")
		32	int tp_handler(const void *ctx)
		33	{
		34	get_pid_tgid();
24	return 0;	35	return 0;
25	}	36	}
26		37
		38	SEC("?cgroup/bind4")
		39	int cgroup_bind4(struct bpf_sock_addr *ctx)
		40	{
		41	get_pid_tgid();
		42	return 1;
		43	}
		44
		45	SEC("?sk_msg")
		46	int sk_msg(struct sk_msg_md *msg)
		47	{
		48	get_pid_tgid();
		49	return SK_PASS;
		50	}
		51
27	char _license[] SEC("license") = "GPL";	52	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/trigger_bench.c+27−12

 char _license[] SEC("license") = "GPL";
-long hits = 0;
+#define CPU_MASK 255
+#define MAX_CPUS (CPU_MASK + 1) /* should match MAX_BUCKETS in benchs/bench_trigger.c */
+/* matches struct counter in bench.h */
+struct counter {
+	long value;
+} __attribute__((aligned(128)));
+struct counter hits[MAX_CPUS];
+static __always_inline void inc_counter(void)
+{
+	int cpu = bpf_get_smp_processor_id();
+	__sync_add_and_fetch(&hits[cpu & CPU_MASK].value, 1);
+}
 SEC("tp/syscalls/sys_enter_getpgid")
 int bench_trigger_tp(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 int BPF_PROG(bench_trigger_raw_tp, struct pt_regs *regs, long id)
 {
 	if (id == __NR_getpgid)
-		__sync_add_and_fetch(&hits, 1);
+		inc_counter();
 	return 0;
 }
 SEC("kprobe/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_kprobe(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 SEC("kretprobe/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_kretprobe(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 SEC("kprobe.multi/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_kprobe_multi(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 SEC("kretprobe.multi/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_kretprobe_multi(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 SEC("fentry/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_fentry(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 SEC("fexit/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_fexit(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 SEC("fentry.s/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_fentry_sleep(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }
 SEC("fmod_ret/" SYS_PREFIX "sys_getpgid")
 int bench_trigger_fmodret(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return -22;
 }
 SEC("uprobe")
 int bench_trigger_uprobe(void *ctx)
 {
-	__sync_add_and_fetch(&hits, 1);
+	inc_counter();
 	return 0;
 }

tools/testing/selftests/bpf/test_cpp.cpp+5−0

 #include <bpf/bpf.h>
 #include <bpf/btf.h>
 #include "test_core_extern.skel.h"
+#include "struct_ops_module.skel.h"
 template <typename T>
 class Skeleton {
 {
 	struct btf_dump_opts opts = { };
 	struct test_core_extern *skel;
+	struct struct_ops_module *skel2;
 	struct btf *btf;
 	int fd;
 	skel = test_core_extern__open_and_load();
 	test_core_extern__destroy(skel);
+	skel2 = struct_ops_module__open_and_load();
+	struct_ops_module__destroy(skel2);
 	fd = bpf_enable_stats(BPF_STATS_RUN_TIME);
 	if (fd < 0)
 		std::cout << "FAILED to enable stats: " << fd << std::endl;

tools/testing/selftests/bpf/test_tc_tunnel.sh+12−1

 server_listen() {
 	ip netns exec "${ns2}" nc "${netcat_opt}" -l "${port}" > "${outfile}" &
 	server_pid=$!
-	sleep 0.2
 }
 client_connect() {
 	fi
 }
+wait_for_port() {
+	for i in $(seq 20); do
+		if ip netns exec "${ns2}" ss ${2:--4}OHntl | grep -q "$1"; then
+			return 0
+		fi
+		sleep 0.1
+	done
+	return 1
+}
 set -e
 # no arguments: automated test, run all
 # basic communication works
 echo "test basic connectivity"
 server_listen
+wait_for_port ${port} ${netcat_opt}
 client_connect
 verify_data
 	section "encap_${tuntype}_${mac}"
 echo "test bpf encap without decap (expect failure)"
 server_listen
+wait_for_port ${port} ${netcat_opt}
 ! client_connect
 if [[ "$tuntype" =~ "udp" ]]; then