lael-labs/linux

Documentation/bpf/bpf_iterators.rst+1−1

@@ -86,7 +86,7 @@ following steps: · 85 unmodified lines
86	The following are a few examples of selftest BPF iterator programs:	86	The following are a few examples of selftest BPF iterator programs:
87		87
88	* `bpf_iter_tcp4.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c>`_	88	* `bpf_iter_tcp4.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c>`_
89	* `bpf_iter_task_vma.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c>`_	89	* `bpf_iter_task_vmas.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_vmas.c>`_
90	* `bpf_iter_task_file.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c>`_	90	* `bpf_iter_task_file.c <https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/tree/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c>`_
91		91
92	Let us look at ``bpf_iter_task_file.c``, which runs in kernel space:	92	Let us look at ``bpf_iter_task_file.c``, which runs in kernel space:

Documentation/bpf/btf.rst+21−4

          * bits 24-28: kind (e.g. int, ptr, array...etc)
          * bits 29-30: unused
          * bit     31: kind_flag, currently used by
-         *             struct, union, fwd, enum and enum64.
+         *             struct, union, enum, fwd, enum64,
+         *             decl_tag and type_tag
          */
         __u32 info;
         /* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64.
 ``struct btf_type`` encoding requirement:
  * ``name_off``: offset to a non-empty string
- * ``info.kind_flag``: 0
+ * ``info.kind_flag``: 0 or 1
  * ``info.kind``: BTF_KIND_DECL_TAG
  * ``info.vlen``: 0
  * ``type``: ``struct``, ``union``, ``func``, ``var`` or ``typedef``
         __u32   component_idx;
     };
-The ``name_off`` encodes btf_decl_tag attribute string.
 The ``type`` should be ``struct``, ``union``, ``func``, ``var`` or ``typedef``.
 For ``var`` or ``typedef`` type, ``btf_decl_tag.component_idx`` must be ``-1``.
 For the other three types, if the btf_decl_tag attribute is
 a ``func`` argument, and ``btf_decl_tag.component_idx`` should be a
 valid index (starting from 0) pointing to a member or an argument.
+If ``info.kind_flag`` is 0, then this is a normal decl tag, and the
+``name_off`` encodes btf_decl_tag attribute string.
+If ``info.kind_flag`` is 1, then the decl tag represents an arbitrary
+__attribute__. In this case, ``name_off`` encodes a string
+representing the attribute-list of the attribute specifier. For
+example, for an ``__attribute__((aligned(4)))`` the string's contents
+is ``aligned(4)``.
 2.2.18 BTF_KIND_TYPE_TAG
 ~~~~~~~~~~~~~~~~~~~~~~~~
 ``struct btf_type`` encoding requirement:
  * ``name_off``: offset to a non-empty string
- * ``info.kind_flag``: 0
+ * ``info.kind_flag``: 0 or 1
  * ``info.kind``: BTF_KIND_TYPE_TAG
  * ``info.vlen``: 0
  * ``type``: the type with ``btf_type_tag`` attribute
 and finally the base type. The base type is one of
 int, ptr, array, struct, union, enum, func_proto and float types.
+Similarly to decl tags, if the ``info.kind_flag`` is 0, then this is a
+normal type tag, and the ``name_off`` encodes btf_type_tag attribute
+string.
+If ``info.kind_flag`` is 1, then the type tag represents an arbitrary
+__attribute__, and the ``name_off`` encodes a string representing the
+attribute-list of the attribute specifier.
 2.2.19 BTF_KIND_ENUM64
 ~~~~~~~~~~~~~~~~~~~~~~

Documentation/bpf/standardization/instruction-set.rst+14−6

 .. table:: Arithmetic instructions
-  =====  =====  =======  ==========================================================
+  =====  =====  =======  ===================================================================================
   name   code   offset   description
-  =====  =====  =======  ==========================================================
+  =====  =====  =======  ===================================================================================
   ADD    0x0    0        dst += src
   SUB    0x1    0        dst -= src
   MUL    0x2    0        dst \*= src
   DIV    0x3    0        dst = (src != 0) ? (dst / src) : 0
-  SDIV   0x3    1        dst = (src != 0) ? (dst s/ src) : 0
+  SDIV   0x3    1        dst = (src == 0) ? 0 : ((src == -1 && dst == LLONG_MIN) ? LLONG_MIN : (dst s/ src))
   OR     0x4    0        dst \|= src
   AND    0x5    0        dst &= src
   LSH    0x6    0        dst <<= (src & mask)
   RSH    0x7    0        dst >>= (src & mask)
   NEG    0x8    0        dst = -dst
   MOD    0x9    0        dst = (src != 0) ? (dst % src) : dst
-  SMOD   0x9    1        dst = (src != 0) ? (dst s% src) : dst
+  SMOD   0x9    1        dst = (src == 0) ? dst : ((src == -1 && dst == LLONG_MIN) ? 0: (dst s% src))
   XOR    0xa    0        dst ^= src
   MOV    0xb    0        dst = src
   MOVSX  0xb    8/16/32  dst = (s8,s16,s32)src
   ARSH   0xc    0        :term:`sign extending<Sign Extend>` dst >>= (src & mask)
   END    0xd    0        byte swap operations (see `Byte swap instructions`_ below)
-  =====  =====  =======  ==========================================================
+  =====  =====  =======  ===================================================================================
 Underflow and overflow are allowed during arithmetic operations, meaning
 the 64-bit or 32-bit value will wrap. If BPF program execution would
 result in division by zero, the destination register is instead set to zero.
+Otherwise, for ``ALU64``, if execution would result in ``LLONG_MIN``
+dividing -1, the desination register is instead set to ``LLONG_MIN``. For
+``ALU``, if execution would result in ``INT_MIN`` dividing -1, the
+desination register is instead set to ``INT_MIN``.
 If execution would result in modulo by zero, for ``ALU64`` the value of
 the destination register is unchanged whereas for ``ALU`` the upper
-32 bits of the destination register are zeroed.
+32 bits of the destination register are zeroed. Otherwise, for ``ALU64``,
+if execution would resuslt in ``LLONG_MIN`` modulo -1, the destination
+register is instead set to 0. For ``ALU``, if execution would result in
+``INT_MIN`` modulo -1, the destination register is instead set to 0.
 ``{ADD, X, ALU}``, where 'code' = ``ADD``, 'source' = ``X``, and 'class' = ``ALU``, means::

arch/arm64/include/asm/insn.h+10−2

 	AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX,
 	AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX,
 	AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX,
+	AARCH64_INSN_LDST_LOAD_ACQ,
 	AARCH64_INSN_LDST_LOAD_EX,
 	AARCH64_INSN_LDST_LOAD_ACQ_EX,
+	AARCH64_INSN_LDST_STORE_REL,
 	AARCH64_INSN_LDST_STORE_EX,
 	AARCH64_INSN_LDST_STORE_REL_EX,
 	AARCH64_INSN_LDST_SIGNED_LOAD_IMM_OFFSET,
 __AARCH64_INSN_FUNCS(ldr_lit,	0xBF000000, 0x18000000)
 __AARCH64_INSN_FUNCS(ldrsw_lit,	0xFF000000, 0x98000000)
 __AARCH64_INSN_FUNCS(exclusive,	0x3F800000, 0x08000000)
-__AARCH64_INSN_FUNCS(load_ex,	0x3F400000, 0x08400000)
+__AARCH64_INSN_FUNCS(load_acq,  0x3FDFFC00, 0x08DFFC00)
-__AARCH64_INSN_FUNCS(store_ex,	0x3F400000, 0x08000000)
+__AARCH64_INSN_FUNCS(store_rel, 0x3FDFFC00, 0x089FFC00)
+__AARCH64_INSN_FUNCS(load_ex,	0x3FC00000, 0x08400000)
+__AARCH64_INSN_FUNCS(store_ex,	0x3FC00000, 0x08000000)
 __AARCH64_INSN_FUNCS(mops,	0x3B200C00, 0x19000400)
 __AARCH64_INSN_FUNCS(stp,	0x7FC00000, 0x29000000)
 __AARCH64_INSN_FUNCS(ldp,	0x7FC00000, 0x29400000)
 				     int offset,
 				     enum aarch64_insn_variant variant,
 				     enum aarch64_insn_ldst_type type);
+u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
+					enum aarch64_insn_register base,
+					enum aarch64_insn_size_type size,
+					enum aarch64_insn_ldst_type type);
 u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
 				   enum aarch64_insn_register base,
 				   enum aarch64_insn_register state,

arch/arm64/lib/insn.c+29−0

@@ -540,6 +540,35 @@ u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1, · 539 unmodified lines
540	offset >> shift);	540	offset >> shift);
541	}	541	}
542		542
		543	u32 aarch64_insn_gen_load_acq_store_rel(enum aarch64_insn_register reg,
		544	enum aarch64_insn_register base,
		545	enum aarch64_insn_size_type size,
		546	enum aarch64_insn_ldst_type type)
		547	{
		548	u32 insn;
		549
		550	switch (type) {
		551	case AARCH64_INSN_LDST_LOAD_ACQ:
		552	insn = aarch64_insn_get_load_acq_value();
		553	break;
		554	case AARCH64_INSN_LDST_STORE_REL:
		555	insn = aarch64_insn_get_store_rel_value();
		556	break;
		557	default:
		558	pr_err("%s: unknown load-acquire/store-release encoding %d\n",
		559	__func__, type);
		560	return AARCH64_BREAK_FAULT;
		561	}
		562
		563	insn = aarch64_insn_encode_ldst_size(size, insn);
		564
		565	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
		566	reg);
		567
		568	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
		569	base);
		570	}
		571
543	u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,	572	u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
544	enum aarch64_insn_register base,	573	enum aarch64_insn_register base,
545	enum aarch64_insn_register state,	574	enum aarch64_insn_register state,

arch/arm64/net/bpf_jit.h+20−0

@@ -119,6 +119,26 @@ · 118 unmodified lines
119	aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \	119	aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \
120	AARCH64_INSN_LDST_STORE_REL_EX)	120	AARCH64_INSN_LDST_STORE_REL_EX)
121		121
		122	/* Load-acquire & store-release */
		123	#define A64_LDAR(Rt, Rn, size) \
		124	aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
		125	AARCH64_INSN_LDST_LOAD_ACQ)
		126	#define A64_STLR(Rt, Rn, size) \
		127	aarch64_insn_gen_load_acq_store_rel(Rt, Rn, AARCH64_INSN_SIZE_##size, \
		128	AARCH64_INSN_LDST_STORE_REL)
		129
		130	/* Rt = [Rn] (load acquire) */
		131	#define A64_LDARB(Wt, Xn) A64_LDAR(Wt, Xn, 8)
		132	#define A64_LDARH(Wt, Xn) A64_LDAR(Wt, Xn, 16)
		133	#define A64_LDAR32(Wt, Xn) A64_LDAR(Wt, Xn, 32)
		134	#define A64_LDAR64(Xt, Xn) A64_LDAR(Xt, Xn, 64)
		135
		136	/* [Rn] = Rt (store release) */
		137	#define A64_STLRB(Wt, Xn) A64_STLR(Wt, Xn, 8)
		138	#define A64_STLRH(Wt, Xn) A64_STLR(Wt, Xn, 16)
		139	#define A64_STLR32(Wt, Xn) A64_STLR(Wt, Xn, 32)
		140	#define A64_STLR64(Xt, Xn) A64_STLR(Xt, Xn, 64)
		141
122	/*	142	/*
123	* LSE atomics	143	* LSE atomics
124	*	144	*

arch/arm64/net/bpf_jit_comp.c+87−5

@@ -272,7 +272,7 @@ static inline void emit_a64_add_i(const bool is64, const int dst, const int src, · 271 unmodified lines
272	{	272	{
273	if (is_addsub_imm(imm)) {	273	if (is_addsub_imm(imm)) {
274	emit(A64_ADD_I(is64, dst, src, imm), ctx);	274	emit(A64_ADD_I(is64, dst, src, imm), ctx);
275	} else if (is_addsub_imm(-imm)) {	275	} else if (is_addsub_imm(-(u32)imm)) {
276	emit(A64_SUB_I(is64, dst, src, -imm), ctx);	276	emit(A64_SUB_I(is64, dst, src, -imm), ctx);
277	} else {	277	} else {
278	emit_a64_mov_i(is64, tmp, imm, ctx);	278	emit_a64_mov_i(is64, tmp, imm, ctx);
@@ -647,6 +647,81 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) · 368 unmodified lines
647	return 0;	647	return 0;
648	}	648	}
649		649
		650	static int emit_atomic_ld_st(const struct bpf_insn insn, struct jit_ctx ctx)
		651	{
		652	const s32 imm = insn->imm;
		653	const s16 off = insn->off;
		654	const u8 code = insn->code;
		655	const bool arena = BPF_MODE(code) == BPF_PROBE_ATOMIC;
		656	const u8 arena_vm_base = bpf2a64[ARENA_VM_START];
		657	const u8 dst = bpf2a64[insn->dst_reg];
		658	const u8 src = bpf2a64[insn->src_reg];
		659	const u8 tmp = bpf2a64[TMP_REG_1];
		660	u8 reg;
		661
		662	switch (imm) {
		663	case BPF_LOAD_ACQ:
		664	reg = src;
		665	break;
		666	case BPF_STORE_REL:
		667	reg = dst;
		668	break;
		669	default:
		670	pr_err_once("unknown atomic load/store op code %02x\n", imm);
		671	return -EINVAL;
		672	}
		673
		674	if (off) {
		675	emit_a64_add_i(1, tmp, reg, tmp, off, ctx);
		676	reg = tmp;
		677	}
		678	if (arena) {
		679	emit(A64_ADD(1, tmp, reg, arena_vm_base), ctx);
		680	reg = tmp;
		681	}
		682
		683	switch (imm) {
		684	case BPF_LOAD_ACQ:
		685	switch (BPF_SIZE(code)) {
		686	case BPF_B:
		687	emit(A64_LDARB(dst, reg), ctx);
		688	break;
		689	case BPF_H:
		690	emit(A64_LDARH(dst, reg), ctx);
		691	break;
		692	case BPF_W:
		693	emit(A64_LDAR32(dst, reg), ctx);
		694	break;
		695	case BPF_DW:
		696	emit(A64_LDAR64(dst, reg), ctx);
		697	break;
		698	}
		699	break;
		700	case BPF_STORE_REL:
		701	switch (BPF_SIZE(code)) {
		702	case BPF_B:
		703	emit(A64_STLRB(src, reg), ctx);
		704	break;
		705	case BPF_H:
		706	emit(A64_STLRH(src, reg), ctx);
		707	break;
		708	case BPF_W:
		709	emit(A64_STLR32(src, reg), ctx);
		710	break;
		711	case BPF_DW:
		712	emit(A64_STLR64(src, reg), ctx);
		713	break;
		714	}
		715	break;
		716	default:
		717	pr_err_once("unexpected atomic load/store op code %02x\n",
		718	imm);
		719	return -EINVAL;
		720	}
		721
		722	return 0;
		723	}
		724
650	#ifdef CONFIG_ARM64_LSE_ATOMICS	725	#ifdef CONFIG_ARM64_LSE_ATOMICS
651	static int emit_lse_atomic(const struct bpf_insn insn, struct jit_ctx ctx)	726	static int emit_lse_atomic(const struct bpf_insn insn, struct jit_ctx ctx)
652	{	727	{
@@ -1159,7 +1234,7 @@ emit_bswap_uxt: · 506 unmodified lines
1159	case BPF_ALU64 \| BPF_SUB \| BPF_K:	1234	case BPF_ALU64 \| BPF_SUB \| BPF_K:
1160	if (is_addsub_imm(imm)) {	1235	if (is_addsub_imm(imm)) {
1161	emit(A64_SUB_I(is64, dst, dst, imm), ctx);	1236	emit(A64_SUB_I(is64, dst, dst, imm), ctx);
1162	} else if (is_addsub_imm(-imm)) {	1237	} else if (is_addsub_imm(-(u32)imm)) {
1163	emit(A64_ADD_I(is64, dst, dst, -imm), ctx);	1238	emit(A64_ADD_I(is64, dst, dst, -imm), ctx);
1164	} else {	1239	} else {
1165	emit_a64_mov_i(is64, tmp, imm, ctx);	1240	emit_a64_mov_i(is64, tmp, imm, ctx);
@@ -1330,7 +1405,7 @@ emit_cond_jmp: · 164 unmodified lines
1330	case BPF_JMP32 \| BPF_JSLE \| BPF_K:	1405	case BPF_JMP32 \| BPF_JSLE \| BPF_K:
1331	if (is_addsub_imm(imm)) {	1406	if (is_addsub_imm(imm)) {
1332	emit(A64_CMP_I(is64, dst, imm), ctx);	1407	emit(A64_CMP_I(is64, dst, imm), ctx);
1333	} else if (is_addsub_imm(-imm)) {	1408	} else if (is_addsub_imm(-(u32)imm)) {
1334	emit(A64_CMN_I(is64, dst, -imm), ctx);	1409	emit(A64_CMN_I(is64, dst, -imm), ctx);
1335	} else {	1410	} else {
1336	emit_a64_mov_i(is64, tmp, imm, ctx);	1411	emit_a64_mov_i(is64, tmp, imm, ctx);
@@ -1641,11 +1716,17 @@ emit_cond_jmp: · 304 unmodified lines
1641	return ret;	1716	return ret;
1642	break;	1717	break;
1643		1718
		1719	case BPF_STX \| BPF_ATOMIC \| BPF_B:
		1720	case BPF_STX \| BPF_ATOMIC \| BPF_H:
1644	case BPF_STX \| BPF_ATOMIC \| BPF_W:	1721	case BPF_STX \| BPF_ATOMIC \| BPF_W:
1645	case BPF_STX \| BPF_ATOMIC \| BPF_DW:	1722	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
		1723	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_B:
		1724	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_H:
1646	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_W:	1725	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_W:
1647	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_DW:	1726	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_DW:
1648	if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))	1727	if (bpf_atomic_is_load_store(insn))
		1728	ret = emit_atomic_ld_st(insn, ctx);
		1729	else if (cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
1649	ret = emit_lse_atomic(insn, ctx);	1730	ret = emit_lse_atomic(insn, ctx);
1650	else	1731	else
1651	ret = emit_ll_sc_atomic(insn, ctx);	1732	ret = emit_ll_sc_atomic(insn, ctx);
@@ -2669,7 +2750,8 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) · 1017 unmodified lines
2669	switch (insn->code) {	2750	switch (insn->code) {
2670	case BPF_STX \| BPF_ATOMIC \| BPF_W:	2751	case BPF_STX \| BPF_ATOMIC \| BPF_W:
2671	case BPF_STX \| BPF_ATOMIC \| BPF_DW:	2752	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
2672	if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))	2753	if (!bpf_atomic_is_load_store(insn) &&
		2754	!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
2673	return false;	2755	return false;
2674	}	2756	}
2675	return true;	2757	return true;

arch/s390/net/bpf_jit_comp.c+10−4

 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
 {
-	/*
+	if (!in_arena)
-	 * Currently the verifier uses this function only to check which
+		return true;
-	 * atomic stores to arena are supported, and they all are.
+	switch (insn->code) {
-	 */
+	case BPF_STX | BPF_ATOMIC | BPF_B:
+	case BPF_STX | BPF_ATOMIC | BPF_H:
+	case BPF_STX | BPF_ATOMIC | BPF_W:
+	case BPF_STX | BPF_ATOMIC | BPF_DW:
+		if (bpf_atomic_is_load_store(insn))
+			return false;
+	}
 	return true;
 }

arch/x86/net/Makefile+1−1

@@ -6,5 +6,5 @@ · 5 unmodified lines
6	ifeq ($(CONFIG_X86_32),y)	6	ifeq ($(CONFIG_X86_32),y)
7	obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o	7	obj-$(CONFIG_BPF_JIT) += bpf_jit_comp32.o
8	else	8	else
9	obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o	9	obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o
10	endif	10	endif

arch/x86/net/bpf_jit_comp.c+87−13

@@ -1250,8 +1250,8 @@ static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm) · 1249 unmodified lines
1250	emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);	1250	emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);
1251	}	1251	}
1252		1252
1253	static int emit_atomic(u8 **pprog, u8 atomic_op,	1253	static int emit_atomic_rmw(u8 **pprog, u32 atomic_op,
1254	u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)	1254	u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
1255	{	1255	{
1256	u8 prog = pprog;	1256	u8 prog = pprog;
1257		1257
@@ -1291,8 +1291,9 @@ static int emit_atomic(u8 **pprog, u8 atomic_op, · 33 unmodified lines
1291	return 0;	1291	return 0;
1292	}	1292	}
1293		1293
1294	static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,	1294	static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size,
1295	u32 dst_reg, u32 src_reg, u32 index_reg, int off)	1295	u32 dst_reg, u32 src_reg, u32 index_reg,
		1296	int off)
1296	{	1297	{
1297	u8 prog = pprog;	1298	u8 prog = pprog;
1298		1299
@@ -1305,7 +1306,7 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size, · 6 unmodified lines
1305	EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg));	1306	EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg));
1306	break;	1307	break;
1307	default:	1308	default:
1308	pr_err("bpf_jit: 1 and 2 byte atomics are not supported\n");	1309	pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
1309	return -EFAULT;	1310	return -EFAULT;
1310	}	1311	}
1311		1312
@@ -1339,6 +1340,49 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size, · 27 unmodified lines
1339	return 0;	1340	return 0;
1340	}	1341	}
1341		1342
		1343	static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg,
		1344	u32 src_reg, s16 off, u8 bpf_size)
		1345	{
		1346	switch (atomic_op) {
		1347	case BPF_LOAD_ACQ:
		1348	/* dst_reg = smp_load_acquire(src_reg + off16) */
		1349	emit_ldx(pprog, bpf_size, dst_reg, src_reg, off);
		1350	break;
		1351	case BPF_STORE_REL:
		1352	/* smp_store_release(dst_reg + off16, src_reg) */
		1353	emit_stx(pprog, bpf_size, dst_reg, src_reg, off);
		1354	break;
		1355	default:
		1356	pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
		1357	atomic_op);
		1358	return -EFAULT;
		1359	}
		1360
		1361	return 0;
		1362	}
		1363
		1364	static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size,
		1365	u32 dst_reg, u32 src_reg, u32 index_reg,
		1366	int off)
		1367	{
		1368	switch (atomic_op) {
		1369	case BPF_LOAD_ACQ:
		1370	/* dst_reg = smp_load_acquire(src_reg + idx_reg + off16) */
		1371	emit_ldx_index(pprog, size, dst_reg, src_reg, index_reg, off);
		1372	break;
		1373	case BPF_STORE_REL:
		1374	/* smp_store_release(dst_reg + idx_reg + off16, src_reg) */
		1375	emit_stx_index(pprog, size, dst_reg, src_reg, index_reg, off);
		1376	break;
		1377	default:
		1378	pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
		1379	atomic_op);
		1380	return -EFAULT;
		1381	}
		1382
		1383	return 0;
		1384	}
		1385
1342	#define DONT_CLEAR 1	1386	#define DONT_CLEAR 1
1343		1387
1344	bool ex_handler_bpf(const struct exception_table_entry x, struct pt_regs regs)	1388	bool ex_handler_bpf(const struct exception_table_entry x, struct pt_regs regs)
@@ -2121,6 +2165,13 @@ populate_extable: · 776 unmodified lines
2121	}	2165	}
2122	break;	2166	break;
2123		2167
		2168	case BPF_STX \| BPF_ATOMIC \| BPF_B:
		2169	case BPF_STX \| BPF_ATOMIC \| BPF_H:
		2170	if (!bpf_atomic_is_load_store(insn)) {
		2171	pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
		2172	return -EFAULT;
		2173	}
		2174	fallthrough;
2124	case BPF_STX \| BPF_ATOMIC \| BPF_W:	2175	case BPF_STX \| BPF_ATOMIC \| BPF_W:
2125	case BPF_STX \| BPF_ATOMIC \| BPF_DW:	2176	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
2126	if (insn->imm == (BPF_AND \| BPF_FETCH) \|\|	2177	if (insn->imm == (BPF_AND \| BPF_FETCH) \|\|
@@ -2156,10 +2207,10 @@ populate_extable: · 29 unmodified lines
2156	EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)],	2207	EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)],
2157	add_2reg(0xC0, AUX_REG, real_src_reg));	2208	add_2reg(0xC0, AUX_REG, real_src_reg));
2158	/* Attempt to swap in new value */	2209	/* Attempt to swap in new value */
2159	err = emit_atomic(&prog, BPF_CMPXCHG,	2210	err = emit_atomic_rmw(&prog, BPF_CMPXCHG,
2160	real_dst_reg, AUX_REG,	2211	real_dst_reg, AUX_REG,
2161	insn->off,	2212	insn->off,
2162	BPF_SIZE(insn->code));	2213	BPF_SIZE(insn->code));
2163	if (WARN_ON(err))	2214	if (WARN_ON(err))
2164	return err;	2215	return err;
2165	/*	2216	/*
@@ -2174,17 +2225,35 @@ populate_extable: · 8 unmodified lines
2174	break;	2225	break;
2175	}	2226	}
2176		2227
2177	err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,	2228	if (bpf_atomic_is_load_store(insn))
2178	insn->off, BPF_SIZE(insn->code));	2229	err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg,
		2230	insn->off, BPF_SIZE(insn->code));
		2231	else
		2232	err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg,
		2233	insn->off, BPF_SIZE(insn->code));
2179	if (err)	2234	if (err)
2180	return err;	2235	return err;
2181	break;	2236	break;
2182		2237
		2238	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_B:
		2239	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_H:
		2240	if (!bpf_atomic_is_load_store(insn)) {
		2241	pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
		2242	return -EFAULT;
		2243	}
		2244	fallthrough;
2183	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_W:	2245	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_W:
2184	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_DW:	2246	case BPF_STX \| BPF_PROBE_ATOMIC \| BPF_DW:
2185	start_of_ldx = prog;	2247	start_of_ldx = prog;
2186	err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code),	2248
2187	dst_reg, src_reg, X86_REG_R12, insn->off);	2249	if (bpf_atomic_is_load_store(insn))
		2250	err = emit_atomic_ld_st_index(&prog, insn->imm,
		2251	BPF_SIZE(insn->code), dst_reg,
		2252	src_reg, X86_REG_R12, insn->off);
		2253	else
		2254	err = emit_atomic_rmw_index(&prog, insn->imm, BPF_SIZE(insn->code),
		2255	dst_reg, src_reg, X86_REG_R12,
		2256	insn->off);
2188	if (err)	2257	if (err)
2189	return err;	2258	return err;
2190	goto populate_extable;	2259	goto populate_extable;
@@ -3801,3 +3870,8 @@ u64 bpf_arch_uaddress_limit(void) · 1610 unmodified lines
3801	{	3870	{
3802	return 0;	3871	return 0;
3803	}	3872	}
		3873
		3874	bool bpf_jit_supports_timed_may_goto(void)
		3875	{
		3876	return true;
		3877	}

arch/x86/net/bpf_timed_may_goto.S+55−0

@@ -0,0 +1,55 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3
	4	#include <linux/export.h>
	5	#include <linux/linkage.h>
	6	#include <asm/nospec-branch.h>
	7
	8	.code64
	9	.section .text, "ax"
	10
	11	SYM_FUNC_START(arch_bpf_timed_may_goto)
	12	ANNOTATE_NOENDBR
	13
	14	/*
	15	* r10 passes us stack depth, load the pointer to count and timestamp
	16	* into r10 by adding it to BPF frame pointer.
	17	*/
	18	leaq (%rbp, %r10, 1), %r10
	19
	20	/* Setup frame. */
	21	pushq %rbp
	22	movq %rsp, %rbp
	23
	24	/* Save r0-r5. */
	25	pushq %rax
	26	pushq %rdi
	27	pushq %rsi
	28	pushq %rdx
	29	pushq %rcx
	30	pushq %r8
	31
	32	/*
	33	* r10 has the pointer to count and timestamp, pass it as first
	34	* argument.
	35	*/
	36	movq %r10, %rdi
	37
	38	/* Emit call depth accounting for call below. */
	39	CALL_DEPTH_ACCOUNT
	40	call bpf_check_timed_may_goto
	41
	42	/* BPF_REG_AX=r10 will be stored into count, so move return value to it. */
	43	movq %rax, %r10
	44
	45	/* Restore r5-r0. */
	46	popq %r8
	47	popq %rcx
	48	popq %rdx
	49	popq %rsi
	50	popq %rdi
	51	popq %rax
	52
	53	leave
	54	RET
	55	SYM_FUNC_END(arch_bpf_timed_may_goto)

fs/bpf_fs_kfuncs.c+214−11

@@ -2,10 +2,12 @@ · 1 unmodified line
2	/* Copyright (c) 2024 Google LLC. */	2	/* Copyright (c) 2024 Google LLC. */
3		3
4	#include <linux/bpf.h>	4	#include <linux/bpf.h>
		5	#include <linux/bpf_lsm.h>
5	#include <linux/btf.h>	6	#include <linux/btf.h>
6	#include <linux/btf_ids.h>	7	#include <linux/btf_ids.h>
7	#include <linux/dcache.h>	8	#include <linux/dcache.h>
8	#include <linux/fs.h>	9	#include <linux/fs.h>
		10	#include <linux/fsnotify.h>
9	#include <linux/file.h>	11	#include <linux/file.h>
10	#include <linux/mm.h>	12	#include <linux/mm.h>
11	#include <linux/xattr.h>	13	#include <linux/xattr.h>
@@ -93,6 +95,24 @@ __bpf_kfunc int bpf_path_d_path(struct path path, char buf, size_t buf__sz) · 81 unmodified lines
93	return len;	95	return len;
94	}	96	}
95		97
		98	static bool match_security_bpf_prefix(const char *name__str)
		99	{
		100	return !strncmp(name__str, XATTR_NAME_BPF_LSM, XATTR_NAME_BPF_LSM_LEN);
		101	}
		102
		103	static int bpf_xattr_read_permission(const char name, struct inode inode)
		104	{
		105	if (WARN_ON(!inode))
		106	return -EINVAL;
		107
		108	/* Allow reading xattr with user. and security.bpf. prefix */
		109	if (strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) &&
		110	!match_security_bpf_prefix(name))
		111	return -EPERM;
		112
		113	return inode_permission(&nop_mnt_idmap, inode, MAY_READ);
		114	}
		115
96	/**	116	/**
97	* bpf_get_dentry_xattr - get xattr of a dentry	117	* bpf_get_dentry_xattr - get xattr of a dentry
98	* @dentry: dentry to get xattr from	118	* @dentry: dentry to get xattr from
@@ -101,9 +121,10 @@ __bpf_kfunc int bpf_path_d_path(struct path path, char buf, size_t buf__sz) · 2 unmodified lines
101	*	121	*
102	* Get xattr name__str of dentry and store the output in value_ptr.	122	* Get xattr name__str of dentry and store the output in value_ptr.
103	*	123	*
104	* For security reasons, only name__str with prefix "user." is allowed.	124	* For security reasons, only name__str with prefixes "user." or
		125	* "security.bpf." are allowed.
105	*	126	*
106	* Return: 0 on success, a negative value on error.	127	* Return: length of the xattr value on success, a negative value on error.
107	*/	128	*/
108	__bpf_kfunc int bpf_get_dentry_xattr(struct dentry dentry, const char name__str,	129	__bpf_kfunc int bpf_get_dentry_xattr(struct dentry dentry, const char name__str,
109	struct bpf_dynptr *value_p)	130	struct bpf_dynptr *value_p)
@@ -114,18 +135,12 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry dentry, const char name__st · 4 unmodified lines
114	void *value;	135	void *value;
115	int ret;	136	int ret;
116		137
117	if (WARN_ON(!inode))
118	return -EINVAL;
119
120	if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN))
121	return -EPERM;
122
123	value_len = __bpf_dynptr_size(value_ptr);	138	value_len = __bpf_dynptr_size(value_ptr);
124	value = __bpf_dynptr_data_rw(value_ptr, value_len);	139	value = __bpf_dynptr_data_rw(value_ptr, value_len);
125	if (!value)	140	if (!value)
126	return -EINVAL;	141	return -EINVAL;
127		142
128	ret = inode_permission(&nop_mnt_idmap, inode, MAY_READ);	143	ret = bpf_xattr_read_permission(name__str, inode);
129	if (ret)	144	if (ret)
130	return ret;	145	return ret;
131	return __vfs_getxattr(dentry, inode, name__str, value, value_len);	146	return __vfs_getxattr(dentry, inode, name__str, value, value_len);
@@ -139,9 +154,10 @@ __bpf_kfunc int bpf_get_dentry_xattr(struct dentry dentry, const char name__st · 7 unmodified lines
139	*	154	*
140	* Get xattr name__str of file and store the output in value_ptr.	155	* Get xattr name__str of file and store the output in value_ptr.
141	*	156	*
142	* For security reasons, only name__str with prefix "user." is allowed.	157	* For security reasons, only name__str with prefixes "user." or
		158	* "security.bpf." are allowed.
143	*	159	*
144	* Return: 0 on success, a negative value on error.	160	* Return: length of the xattr value on success, a negative value on error.
145	*/	161	*/
146	__bpf_kfunc int bpf_get_file_xattr(struct file file, const char name__str,	162	__bpf_kfunc int bpf_get_file_xattr(struct file file, const char name__str,
147	struct bpf_dynptr *value_p)	163	struct bpf_dynptr *value_p)
@@ -154,6 +170,160 @@ __bpf_kfunc int bpf_get_file_xattr(struct file file, const char name__str, · 6 unmodified lines
154		170
155	__bpf_kfunc_end_defs();	171	__bpf_kfunc_end_defs();
156		172
		173	static int bpf_xattr_write_permission(const char name, struct inode inode)
		174	{
		175	if (WARN_ON(!inode))
		176	return -EINVAL;
		177
		178	/* Only allow setting and removing security.bpf. xattrs */
		179	if (!match_security_bpf_prefix(name))
		180	return -EPERM;
		181
		182	return inode_permission(&nop_mnt_idmap, inode, MAY_WRITE);
		183	}
		184
		185	/**
		186	* bpf_set_dentry_xattr_locked - set a xattr of a dentry
		187	* @dentry: dentry to get xattr from
		188	* @name__str: name of the xattr
		189	* @value_p: xattr value
		190	* @flags: flags to pass into filesystem operations
		191	*
		192	* Set xattr name__str of dentry to the value in value_ptr.
		193	*
		194	* For security reasons, only name__str with prefix "security.bpf."
		195	* is allowed.
		196	*
		197	* The caller already locked dentry->d_inode.
		198	*
		199	* Return: 0 on success, a negative value on error.
		200	*/
		201	int bpf_set_dentry_xattr_locked(struct dentry dentry, const char name__str,
		202	const struct bpf_dynptr *value_p, int flags)
		203	{
		204
		205	struct bpf_dynptr_kern value_ptr = (struct bpf_dynptr_kern )value_p;
		206	struct inode *inode = d_inode(dentry);
		207	const void *value;
		208	u32 value_len;
		209	int ret;
		210
		211	value_len = __bpf_dynptr_size(value_ptr);
		212	value = __bpf_dynptr_data(value_ptr, value_len);
		213	if (!value)
		214	return -EINVAL;
		215
		216	ret = bpf_xattr_write_permission(name__str, inode);
		217	if (ret)
		218	return ret;
		219
		220	ret = __vfs_setxattr(&nop_mnt_idmap, dentry, inode, name__str,
		221	value, value_len, flags);
		222	if (!ret) {
		223	fsnotify_xattr(dentry);
		224
		225	/* This xattr is set by BPF LSM, so we do not call
		226	* security_inode_post_setxattr. Otherwise, we would
		227	* risk deadlocks by calling back to the same kfunc.
		228	*
		229	* This is the same as security_inode_setsecurity().
		230	*/
		231	}
		232	return ret;
		233	}
		234
		235	/**
		236	* bpf_remove_dentry_xattr_locked - remove a xattr of a dentry
		237	* @dentry: dentry to get xattr from
		238	* @name__str: name of the xattr
		239	*
		240	* Rmove xattr name__str of dentry.
		241	*
		242	* For security reasons, only name__str with prefix "security.bpf."
		243	* is allowed.
		244	*
		245	* The caller already locked dentry->d_inode.
		246	*
		247	* Return: 0 on success, a negative value on error.
		248	*/
		249	int bpf_remove_dentry_xattr_locked(struct dentry dentry, const char name__str)
		250	{
		251	struct inode *inode = d_inode(dentry);
		252	int ret;
		253
		254	ret = bpf_xattr_write_permission(name__str, inode);
		255	if (ret)
		256	return ret;
		257
		258	ret = __vfs_removexattr(&nop_mnt_idmap, dentry, name__str);
		259	if (!ret) {
		260	fsnotify_xattr(dentry);
		261
		262	/* This xattr is removed by BPF LSM, so we do not call
		263	* security_inode_post_removexattr. Otherwise, we would
		264	* risk deadlocks by calling back to the same kfunc.
		265	*/
		266	}
		267	return ret;
		268	}
		269
		270	__bpf_kfunc_start_defs();
		271
		272	/**
		273	* bpf_set_dentry_xattr - set a xattr of a dentry
		274	* @dentry: dentry to get xattr from
		275	* @name__str: name of the xattr
		276	* @value_p: xattr value
		277	* @flags: flags to pass into filesystem operations
		278	*
		279	* Set xattr name__str of dentry to the value in value_ptr.
		280	*
		281	* For security reasons, only name__str with prefix "security.bpf."
		282	* is allowed.
		283	*
		284	* The caller has not locked dentry->d_inode.
		285	*
		286	* Return: 0 on success, a negative value on error.
		287	*/
		288	__bpf_kfunc int bpf_set_dentry_xattr(struct dentry dentry, const char name__str,
		289	const struct bpf_dynptr *value_p, int flags)
		290	{
		291	struct inode *inode = d_inode(dentry);
		292	int ret;
		293
		294	inode_lock(inode);
		295	ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags);
		296	inode_unlock(inode);
		297	return ret;
		298	}
		299
		300	/**
		301	* bpf_remove_dentry_xattr - remove a xattr of a dentry
		302	* @dentry: dentry to get xattr from
		303	* @name__str: name of the xattr
		304	*
		305	* Rmove xattr name__str of dentry.
		306	*
		307	* For security reasons, only name__str with prefix "security.bpf."
		308	* is allowed.
		309	*
		310	* The caller has not locked dentry->d_inode.
		311	*
		312	* Return: 0 on success, a negative value on error.
		313	*/
		314	__bpf_kfunc int bpf_remove_dentry_xattr(struct dentry dentry, const char name__str)
		315	{
		316	struct inode *inode = d_inode(dentry);
		317	int ret;
		318
		319	inode_lock(inode);
		320	ret = bpf_remove_dentry_xattr_locked(dentry, name__str);
		321	inode_unlock(inode);
		322	return ret;
		323	}
		324
		325	__bpf_kfunc_end_defs();
		326
157	BTF_KFUNCS_START(bpf_fs_kfunc_set_ids)	327	BTF_KFUNCS_START(bpf_fs_kfunc_set_ids)
158	BTF_ID_FLAGS(func, bpf_get_task_exe_file,	328	BTF_ID_FLAGS(func, bpf_get_task_exe_file,
159	KF_ACQUIRE \| KF_TRUSTED_ARGS \| KF_RET_NULL)	329	KF_ACQUIRE \| KF_TRUSTED_ARGS \| KF_RET_NULL)
@@ -161,6 +331,8 @@ BTF_ID_FLAGS(func, bpf_put_file, KF_RELEASE) · 1 unmodified line
161	BTF_ID_FLAGS(func, bpf_path_d_path, KF_TRUSTED_ARGS)	331	BTF_ID_FLAGS(func, bpf_path_d_path, KF_TRUSTED_ARGS)
162	BTF_ID_FLAGS(func, bpf_get_dentry_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)	332	BTF_ID_FLAGS(func, bpf_get_dentry_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
163	BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)	333	BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
		334	BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
		335	BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE \| KF_TRUSTED_ARGS)
164	BTF_KFUNCS_END(bpf_fs_kfunc_set_ids)	336	BTF_KFUNCS_END(bpf_fs_kfunc_set_ids)
165		337
166	static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)	338	static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id)
@@ -171,6 +343,37 @@ static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id) · 4 unmodified lines
171	return -EACCES;	343	return -EACCES;
172	}	344	}
173		345
		346	/* bpf_[set\|remove]_dentry_xattr.* hooks have KF_TRUSTED_ARGS and
		347	* KF_SLEEPABLE, so they are only available to sleepable hooks with
		348	* dentry arguments.
		349	*
		350	* Setting and removing xattr requires exclusive lock on dentry->d_inode.
		351	* Some hooks already locked d_inode, while some hooks have not locked
		352	* d_inode. Therefore, we need different kfuncs for different hooks.
		353	* Specifically, hooks in the following list (d_inode_locked_hooks)
		354	* should call bpf_[set\|remove]_dentry_xattr_locked; while other hooks
		355	* should call bpf_[set\|remove]_dentry_xattr.
		356	*/
		357	BTF_SET_START(d_inode_locked_hooks)
		358	BTF_ID(func, bpf_lsm_inode_post_removexattr)
		359	BTF_ID(func, bpf_lsm_inode_post_setattr)
		360	BTF_ID(func, bpf_lsm_inode_post_setxattr)
		361	BTF_ID(func, bpf_lsm_inode_removexattr)
		362	BTF_ID(func, bpf_lsm_inode_rmdir)
		363	BTF_ID(func, bpf_lsm_inode_setattr)
		364	BTF_ID(func, bpf_lsm_inode_setxattr)
		365	BTF_ID(func, bpf_lsm_inode_unlink)
		366	#ifdef CONFIG_SECURITY_PATH
		367	BTF_ID(func, bpf_lsm_path_unlink)
		368	BTF_ID(func, bpf_lsm_path_rmdir)
		369	#endif /* CONFIG_SECURITY_PATH */
		370	BTF_SET_END(d_inode_locked_hooks)
		371
		372	bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
		373	{
		374	return btf_id_set_contains(&d_inode_locked_hooks, prog->aux->attach_btf_id);
		375	}
		376
174	static const struct btf_kfunc_id_set bpf_fs_kfunc_set = {	377	static const struct btf_kfunc_id_set bpf_fs_kfunc_set = {
175	.owner = THIS_MODULE,	378	.owner = THIS_MODULE,
176	.set = &bpf_fs_kfunc_set_ids,	379	.set = &bpf_fs_kfunc_set_ids,

include/linux/bpf-cgroup.h+1−0

@@ -111,6 +111,7 @@ struct bpf_prog_list { · 110 unmodified lines
111	struct bpf_prog *prog;	111	struct bpf_prog *prog;
112	struct bpf_cgroup_link *link;	112	struct bpf_cgroup_link *link;
113	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];	113	struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE];
		114	u32 flags;
114	};	115	};
115		116
116	int cgroup_bpf_inherit(struct cgroup *cgrp);	117	int cgroup_bpf_inherit(struct cgroup *cgrp);

include/linux/bpf.h+29−2

 		struct {
 			struct btf *btf;
 			u32 btf_id;
+			u32 ref_obj_id;
 		};
 	};
 	struct bpf_verifier_log *log; /* for verbose logs */
 	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
 }
+/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
+ * atomic load or store, and false if it is a read-modify-write instruction.
+ */
+static inline bool
+bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
+{
+	switch (atomic_insn->imm) {
+	case BPF_LOAD_ACQ:
+	case BPF_STORE_REL:
+		return true;
+	default:
+		return false;
+	}
+}
 struct bpf_prog_ops {
 	int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr,
 			union bpf_attr __user *uattr);
 	enum bpf_reg_type reg_type;
 	struct btf *btf;
 	u32 btf_id;
+	u32 ref_obj_id;
+	bool refcounted;
 };
 struct btf_mod_pair {
 	u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */
 	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
 	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
+	u32 attach_st_ops_member_off;
 	u32 ctx_arg_info_size;
 	u32 max_rdonly_access;
 	u32 max_rdwr_access;
 	struct btf *attach_btf;
-	const struct bpf_ctx_arg_aux *ctx_arg_info;
+	struct bpf_ctx_arg_aux *ctx_arg_info;
 	void __percpu *priv_stack_ptr;
 	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
 	struct bpf_prog *dst_prog;
 	bool jits_use_priv_stack;
 	bool priv_stack_requested;
 	bool changes_pkt_data;
+	bool might_sleep;
 	u64 prog_array_member_cnt; /* counts how many times as member of prog_array */
 	struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */
 	struct bpf_arena *arena;
 #endif
 	struct bpf_ksym ksym;
 	const struct bpf_prog_ops *ops;
+	const struct bpf_struct_ops *st_ops;
 	struct bpf_map **used_maps;
 	struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */
 	struct btf_mod_pair *used_btfs;
 #endif
+int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
+			       const struct bpf_ctx_arg_aux *info, u32 cnt);
 #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM)
 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,
 				    int cgroup_atype);
  */
 enum {
 	BPF_MAX_LOOPS = 8 * 1024 * 1024,
+	BPF_MAX_TIMED_LOOPS = 0xffff,
 };
 #define BPF_F_ACCESS_MASK	(BPF_F_RDONLY |		\
 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void);
+const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void);
 typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src,
 					unsigned long off, unsigned long len);
 typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type,
 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info);
 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info);
-bool bpf_iter_prog_supported(struct bpf_prog *prog);
+int bpf_iter_prog_supported(struct bpf_prog *prog);
 const struct bpf_func_proto *
 bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
 int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog);

include/linux/bpf_lsm.h+18−0

@@ -48,6 +48,11 @@ void bpf_lsm_find_cgroup_shim(const struct bpf_prog prog, bpf_func_t bpf_func) · 47 unmodified lines
48		48
49	int bpf_lsm_get_retval_range(const struct bpf_prog *prog,	49	int bpf_lsm_get_retval_range(const struct bpf_prog *prog,
50	struct bpf_retval_range *range);	50	struct bpf_retval_range *range);
		51	int bpf_set_dentry_xattr_locked(struct dentry dentry, const char name__str,
		52	const struct bpf_dynptr *value_p, int flags);
		53	int bpf_remove_dentry_xattr_locked(struct dentry dentry, const char name__str);
		54	bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog);
		55
51	#else /* !CONFIG_BPF_LSM */	56	#else /* !CONFIG_BPF_LSM */
52		57
53	static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)	58	static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
@@ -86,6 +91,19 @@ static inline int bpf_lsm_get_retval_range(const struct bpf_prog *prog, · 32 unmodified lines
86	{	91	{
87	return -EOPNOTSUPP;	92	return -EOPNOTSUPP;
88	}	93	}
		94	static inline int bpf_set_dentry_xattr_locked(struct dentry dentry, const char name__str,
		95	const struct bpf_dynptr *value_p, int flags)
		96	{
		97	return -EOPNOTSUPP;
		98	}
		99	static inline int bpf_remove_dentry_xattr_locked(struct dentry dentry, const char name__str)
		100	{
		101	return -EOPNOTSUPP;
		102	}
		103	static inline bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog)
		104	{
		105	return false;
		106	}
89	#endif /* CONFIG_BPF_LSM */	107	#endif /* CONFIG_BPF_LSM */
90		108
91	#endif /* _LINUX_BPF_LSM_H */	109	#endif /* _LINUX_BPF_LSM_H */

include/linux/bpf_verifier.h+23−9

 	bool active_rcu_lock;
 	bool speculative;
-	/* If this state was ever pointed-to by other state's loop_entry field
-	 * this flag would be set to true. Used to avoid freeing such states
-	 * while they are still in use.
-	 */
-	bool used_as_loop_entry;
 	bool in_sleepable;
 	/* first and last insn idx of this verifier state */
 	u32 dfs_depth;
 	u32 callback_unroll_depth;
 	u32 may_goto_depth;
+	/* If this state was ever pointed-to by other state's loop_entry field
+	 * this flag would be set to true. Used to avoid freeing such states
+	 * while they are still in use.
+	 */
+	u32 used_as_loop_entry;
 };
 #define bpf_get_spilled_reg(slot, frame, mask)				\
 /* linked list of verifier states used to prune search */
 struct bpf_verifier_state_list {
 	struct bpf_verifier_state state;
-	struct bpf_verifier_state_list *next;
+	struct list_head node;
-	int miss_cnt, hit_cnt;
+	u32 miss_cnt;
+	u32 hit_cnt:31;
+	u32 in_free_list:1;
 };
 struct bpf_loop_inline_state {
 	 * accepts callback function as a parameter.
 	 */
 	bool calls_callback;
+	/* registers alive before this instruction. */
+	u16 live_regs_before;
 };
 #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */
 	/* true if bpf_fastcall stack region is used by functions that can't be inlined */
 	bool keep_fastcall_stack: 1;
 	bool changes_pkt_data: 1;
+	bool might_sleep: 1;
 	enum priv_stack_mode priv_stack_mode;
 	u8 arg_cnt;
 	bool test_state_freq;		/* test verifier with different pruning frequency */
 	bool test_reg_invariants;	/* fail verification on register invariants violations */
 	struct bpf_verifier_state *cur_state; /* current verifier state */
-	struct bpf_verifier_state_list **explored_states; /* search pruning optimization */
+	/* Search pruning optimization, array of list_heads for
-	struct bpf_verifier_state_list *free_list;
+	 * lists of struct bpf_verifier_state_list.
+	 */
+	struct list_head *explored_states;
+	struct list_head free_list;	/* list of struct bpf_verifier_state_list */
 	struct bpf_map *used_maps[MAX_USED_MAPS]; /* array of map's used by eBPF program */
 	struct btf_mod_pair used_btfs[MAX_USED_BTFS]; /* array of BTF's used by BPF program */
 	u32 used_map_cnt;		/* number of used maps */
 	struct {
 		int *insn_state;
 		int *insn_stack;
+		/* vector of instruction indexes sorted in post-order */
+		int *insn_postorder;
 		int cur_stack;
+		/* current position in the insn_postorder vector */
+		int cur_postorder;
 	} cfg;
 	struct backtrack_state bt;
 	struct bpf_insn_hist_entry *insn_hist;
 	u32 peak_states;
 	/* longest register parentage chain walked for liveness marking */
 	u32 longest_mark_read_walk;
+	u32 free_list_size;
+	u32 explored_states_size;
 	bpfptr_t fd_array;
 	/* bit mask to keep track of whether a register has been accessed

include/linux/btf.h+3−0

 #define KF_ITER_DESTROY (1 << 10) /* kfunc implements BPF iter destructor */
 #define KF_RCU_PROTECTED (1 << 11) /* kfunc should be protected by rcu cs when they are invoked */
 #define KF_FASTCALL     (1 << 12) /* kfunc supports bpf_fastcall protocol */
+#define KF_ARENA_RET    (1 << 13) /* kfunc returns an arena pointer */
+#define KF_ARENA_ARG1   (1 << 14) /* kfunc takes an arena pointer as its first argument */
+#define KF_ARENA_ARG2   (1 << 15) /* kfunc takes an arena pointer as its second argument */
 /*
  * Tag marking a kernel function as a kfunc. This is meant to minimize the

include/linux/filter.h+20−0

  *   BPF_XOR | BPF_FETCH      src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
  *   BPF_XCHG                 src_reg = atomic_xchg(dst_reg + off16, src_reg)
  *   BPF_CMPXCHG              r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
+ *   BPF_LOAD_ACQ             dst_reg = smp_load_acquire(src_reg + off16)
+ *   BPF_STORE_REL            smp_store_release(dst_reg + off16, src_reg)
  */
 #define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF)			\
 		.off   = 0,					\
 		.imm   = BPF_CALL_IMM(FUNC) })
+/* Kfunc call */
+#define BPF_CALL_KFUNC(OFF, IMM)				\
+	((struct bpf_insn) {					\
+		.code  = BPF_JMP | BPF_CALL,			\
+		.dst_reg = 0,					\
+		.src_reg = BPF_PSEUDO_KFUNC_CALL,		\
+		.off   = OFF,					\
+		.imm   = IMM })
 /* Raw code statement block */
 #define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM)			\
 	struct u64_stats_sync syncp;
 } __aligned(2 * sizeof(u64));
+struct bpf_timed_may_goto {
+	u64 count;
+	u64 timestamp;
+};
 struct sk_filter {
 	refcount_t	refcnt;
 	struct rcu_head	rcu;
 bool bpf_jit_supports_arena(void);
 bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena);
 bool bpf_jit_supports_private_stack(void);
+bool bpf_jit_supports_timed_may_goto(void);
 u64 bpf_arch_uaddress_limit(void);
 void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie);
+u64 arch_bpf_timed_may_goto(void);
+u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *);
 bool bpf_helper_changes_pkt_data(enum bpf_func_id func_id);
 static inline bool bpf_dump_raw_ok(const struct cred *cred)

include/linux/lsm_hook_defs.h+3−3

@@ -426,14 +426,14 @@ LSM_HOOK(void, LSM_RET_VOID, audit_rule_free, void *lsmrule) · 425 unmodified lines
426	#endif /* CONFIG_AUDIT */	426	#endif /* CONFIG_AUDIT */
427		427
428	#ifdef CONFIG_BPF_SYSCALL	428	#ifdef CONFIG_BPF_SYSCALL
429	LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size)	429	LSM_HOOK(int, 0, bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
430	LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode)	430	LSM_HOOK(int, 0, bpf_map, struct bpf_map *map, fmode_t fmode)
431	LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)	431	LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)
432	LSM_HOOK(int, 0, bpf_map_create, struct bpf_map map, union bpf_attr attr,	432	LSM_HOOK(int, 0, bpf_map_create, struct bpf_map map, union bpf_attr attr,
433	struct bpf_token *token)	433	struct bpf_token *token, bool kernel)
434	LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map)	434	LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map)
435	LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog prog, union bpf_attr attr,	435	LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog prog, union bpf_attr attr,
436	struct bpf_token *token)	436	struct bpf_token *token, bool kernel)
437	LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog)	437	LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog)
438	LSM_HOOK(int, 0, bpf_token_create, struct bpf_token token, union bpf_attr attr,	438	LSM_HOOK(int, 0, bpf_token_create, struct bpf_token token, union bpf_attr attr,
439	const struct path *path)	439	const struct path *path)

include/linux/mm.h+5−0

 extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
 		void *buf, int len, unsigned int gup_flags);
+#ifdef CONFIG_BPF_SYSCALL
+extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
+			      void *buf, int len, unsigned int gup_flags);
+#endif
 long get_user_pages_remote(struct mm_struct *mm,
 			   unsigned long start, unsigned long nr_pages,
 			   unsigned int gup_flags, struct page **pages,

include/linux/security.h+6−6

@@ -2249,14 +2249,14 @@ struct bpf_map; · 2248 unmodified lines
2249	struct bpf_prog;	2249	struct bpf_prog;
2250	struct bpf_token;	2250	struct bpf_token;
2251	#ifdef CONFIG_SECURITY	2251	#ifdef CONFIG_SECURITY
2252	extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size);	2252	extern int security_bpf(int cmd, union bpf_attr *attr, unsigned int size, bool kernel);
2253	extern int security_bpf_map(struct bpf_map *map, fmode_t fmode);	2253	extern int security_bpf_map(struct bpf_map *map, fmode_t fmode);
2254	extern int security_bpf_prog(struct bpf_prog *prog);	2254	extern int security_bpf_prog(struct bpf_prog *prog);
2255	extern int security_bpf_map_create(struct bpf_map map, union bpf_attr attr,	2255	extern int security_bpf_map_create(struct bpf_map map, union bpf_attr attr,
2256	struct bpf_token *token);	2256	struct bpf_token *token, bool kernel);
2257	extern void security_bpf_map_free(struct bpf_map *map);	2257	extern void security_bpf_map_free(struct bpf_map *map);
2258	extern int security_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,	2258	extern int security_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,
2259	struct bpf_token *token);	2259	struct bpf_token *token, bool kernel);
2260	extern void security_bpf_prog_free(struct bpf_prog *prog);	2260	extern void security_bpf_prog_free(struct bpf_prog *prog);
2261	extern int security_bpf_token_create(struct bpf_token token, union bpf_attr attr,	2261	extern int security_bpf_token_create(struct bpf_token token, union bpf_attr attr,
2262	const struct path *path);	2262	const struct path *path);
@@ -2265,7 +2265,7 @@ extern int security_bpf_token_cmd(const struct bpf_token *token, enum bpf_cmd cm · 2 unmodified lines
2265	extern int security_bpf_token_capable(const struct bpf_token *token, int cap);	2265	extern int security_bpf_token_capable(const struct bpf_token *token, int cap);
2266	#else	2266	#else
2267	static inline int security_bpf(int cmd, union bpf_attr *attr,	2267	static inline int security_bpf(int cmd, union bpf_attr *attr,
2268	unsigned int size)	2268	unsigned int size, bool kernel)
2269	{	2269	{
2270	return 0;	2270	return 0;
2271	}	2271	}
@@ -2281,7 +2281,7 @@ static inline int security_bpf_prog(struct bpf_prog *prog) · 9 unmodified lines
2281	}	2281	}
2282		2282
2283	static inline int security_bpf_map_create(struct bpf_map map, union bpf_attr attr,	2283	static inline int security_bpf_map_create(struct bpf_map map, union bpf_attr attr,
2284	struct bpf_token *token)	2284	struct bpf_token *token, bool kernel)
2285	{	2285	{
2286	return 0;	2286	return 0;
2287	}	2287	}
@@ -2290,7 +2290,7 @@ static inline void security_bpf_map_free(struct bpf_map *map) · 2 unmodified lines
2290	{ }	2290	{ }
2291		2291
2292	static inline int security_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,	2292	static inline int security_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,
2293	struct bpf_token *token)	2293	struct bpf_token *token, bool kernel)
2294	{	2294	{
2295	return 0;	2295	return 0;
2296	}	2296	}

include/uapi/linux/bpf.h+9−1

 #define BPF_XCHG	(0xe0 | BPF_FETCH)	/* atomic exchange */
 #define BPF_CMPXCHG	(0xf0 | BPF_FETCH)	/* atomic compare-and-write */
+#define BPF_LOAD_ACQ	0x100	/* load-acquire */
+#define BPF_STORE_REL	0x110	/* store-release */
 enum bpf_cond_pseudo_jmp {
 	BPF_MAY_GOTO = 0,
 };
 #define BPF_F_BEFORE		(1U << 3)
 #define BPF_F_AFTER		(1U << 4)
 #define BPF_F_ID		(1U << 5)
+#define BPF_F_PREORDER		(1U << 6)
 #define BPF_F_LINK		BPF_F_LINK /* 1 << 13 */
 /* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
 		};
 		__u32		next_id;
 		__u32		open_flags;
+		__s32		fd_by_id_token_fd;
 	};
 	struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */
 	FN(user_ringbuf_drain, 209, ##ctx)		\
 	FN(cgrp_storage_get, 210, ##ctx)		\
 	FN(cgrp_storage_delete, 211, ##ctx)		\
-	/* */
+	/* This helper list is effectively frozen. If you are trying to	\
+	 * add a new helper, you should add a kfunc instead which has	\
+	 * less stability guarantees. See Documentation/bpf/kfuncs.rst	\
+	 */
 /* backwards-compatibility macros for users of __BPF_FUNC_MAPPER that don't
  * know or care about integer value that is now passed as second argument

include/uapi/linux/btf.h+2−1

 	 * bits 24-28: kind (e.g. int, ptr, array...etc)
 	 * bits 29-30: unused
 	 * bit     31: kind_flag, currently used by
-	 *             struct, union, enum, fwd and enum64
+	 *             struct, union, enum, fwd, enum64,
+	 *             decl_tag and type_tag
 	 */
 	__u32 info;
 	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.

include/uapi/linux/xattr.h+4−0

 #define XATTR_CAPS_SUFFIX "capability"
 #define XATTR_NAME_CAPS XATTR_SECURITY_PREFIX XATTR_CAPS_SUFFIX
+#define XATTR_BPF_LSM_SUFFIX "bpf."
+#define XATTR_NAME_BPF_LSM (XATTR_SECURITY_PREFIX XATTR_BPF_LSM_SUFFIX)
+#define XATTR_NAME_BPF_LSM_LEN (sizeof(XATTR_NAME_BPF_LSM) - 1)
 #define XATTR_POSIX_ACL_ACCESS  "posix_acl_access"
 #define XATTR_NAME_POSIX_ACL_ACCESS XATTR_SYSTEM_PREFIX XATTR_POSIX_ACL_ACCESS
 #define XATTR_POSIX_ACL_DEFAULT  "posix_acl_default"

kernel/bpf/arena.c+2−2

@@ -577,8 +577,8 @@ __bpf_kfunc void bpf_arena_free_pages(void p__map, void ptr__ign, u32 page_cnt · 576 unmodified lines
577	__bpf_kfunc_end_defs();	577	__bpf_kfunc_end_defs();
578		578
579	BTF_KFUNCS_START(arena_kfuncs)	579	BTF_KFUNCS_START(arena_kfuncs)
580	BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE)	580	BTF_ID_FLAGS(func, bpf_arena_alloc_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE \| KF_ARENA_RET \| KF_ARENA_ARG2)
581	BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE)	581	BTF_ID_FLAGS(func, bpf_arena_free_pages, KF_TRUSTED_ARGS \| KF_SLEEPABLE \| KF_ARENA_ARG2)
582	BTF_KFUNCS_END(arena_kfuncs)	582	BTF_KFUNCS_END(arena_kfuncs)
583		583
584	static const struct btf_kfunc_id_set common_kfunc_set = {	584	static const struct btf_kfunc_id_set common_kfunc_set = {

kernel/bpf/bpf_cgrp_storage.c+6−5

 	   void *, value, u64, flags, gfp_t, gfp_flags)
 {
 	struct bpf_local_storage_data *sdata;
+	bool nobusy;
 	WARN_ON_ONCE(!bpf_rcu_lock_held());
 	if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
 	if (!cgroup)
 		return (unsigned long)NULL;
-	if (!bpf_cgrp_storage_trylock())
+	nobusy = bpf_cgrp_storage_trylock();
-		return (unsigned long)NULL;
-	sdata = cgroup_storage_lookup(cgroup, map, true);
+	sdata = cgroup_storage_lookup(cgroup, map, nobusy);
 	if (sdata)
 		goto unlock;
 	/* only allocate new storage, when the cgroup is refcounted */
 	if (!percpu_ref_is_dying(&cgroup->self.refcnt) &&
-	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE))
+	    (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy)
 		sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map,
 						 value, BPF_NOEXIST, false, gfp_flags);
 unlock:
-	bpf_cgrp_storage_unlock();
+	if (nobusy)
+		bpf_cgrp_storage_unlock();
 	return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
 }

kernel/bpf/bpf_iter.c+6−7

 	tinfo->btf_id = prog->aux->attach_btf_id;
 }
-bool bpf_iter_prog_supported(struct bpf_prog *prog)
+int bpf_iter_prog_supported(struct bpf_prog *prog)
 {
 	const char *attach_fname = prog->aux->attach_func_name;
 	struct bpf_iter_target_info *tinfo = NULL, *iter;
 	int prefix_len = strlen(prefix);
 	if (strncmp(attach_fname, prefix, prefix_len))
-		return false;
+		return -EINVAL;
 	mutex_lock(&targets_mutex);
 	list_for_each_entry(iter, &targets, list) {
 	}
 	mutex_unlock(&targets_mutex);
-	if (tinfo) {
+	if (!tinfo)
-		prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
+		return -EINVAL;
-		prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
-	}
-	return tinfo != NULL;
+	return bpf_prog_ctx_arg_info_init(prog, tinfo->reg_info->ctx_arg_info,
+					  tinfo->reg_info->ctx_arg_info_size);
 }
 const struct bpf_func_proto *

kernel/bpf/bpf_lsm.c+2−0

 BTF_ID(func, bpf_lsm_inode_mknod)
 BTF_ID(func, bpf_lsm_inode_need_killpriv)
 BTF_ID(func, bpf_lsm_inode_post_setxattr)
+BTF_ID(func, bpf_lsm_inode_post_removexattr)
 BTF_ID(func, bpf_lsm_inode_readlink)
+BTF_ID(func, bpf_lsm_inode_removexattr)
 BTF_ID(func, bpf_lsm_inode_rename)
 BTF_ID(func, bpf_lsm_inode_rmdir)
 BTF_ID(func, bpf_lsm_inode_setattr)

kernel/bpf/bpf_struct_ops.c+75−60

@@ -146,39 +146,7 @@ void bpf_struct_ops_image_free(void *image) · 145 unmodified lines
146	}	146	}
147		147
148	#define MAYBE_NULL_SUFFIX "__nullable"	148	#define MAYBE_NULL_SUFFIX "__nullable"
149	#define MAX_STUB_NAME 128	149	#define REFCOUNTED_SUFFIX "__ref"
150
151	/* Return the type info of a stub function, if it exists.
152	*
153	* The name of a stub function is made up of the name of the struct_ops and
154	* the name of the function pointer member, separated by "__". For example,
155	* if the struct_ops type is named "foo_ops" and the function pointer
156	* member is named "bar", the stub function name would be "foo_ops__bar".
157	*/
158	static const struct btf_type *
159	find_stub_func_proto(const struct btf btf, const char st_op_name,
160	const char *member_name)
161	{
162	char stub_func_name[MAX_STUB_NAME];
163	const struct btf_type *func_type;
164	s32 btf_id;
165	int cp;
166
167	cp = snprintf(stub_func_name, MAX_STUB_NAME, "%s__%s",
168	st_op_name, member_name);
169	if (cp >= MAX_STUB_NAME) {
170	pr_warn("Stub function name too long\n");
171	return NULL;
172	}
173	btf_id = btf_find_by_name_kind(btf, stub_func_name, BTF_KIND_FUNC);
174	if (btf_id < 0)
175	return NULL;
176	func_type = btf_type_by_id(btf, btf_id);
177	if (!func_type)
178	return NULL;
179
180	return btf_type_by_id(btf, func_type->type); /* FUNC_PROTO */
181	}
182		150
183	/* Prepare argument info for every nullable argument of a member of a	151	/* Prepare argument info for every nullable argument of a member of a
184	* struct_ops type.	152	* struct_ops type.
@@ -203,27 +171,44 @@ find_stub_func_proto(const struct btf btf, const char st_op_name, · 18 unmodified lines
203	static int prepare_arg_info(struct btf *btf,	171	static int prepare_arg_info(struct btf *btf,
204	const char *st_ops_name,	172	const char *st_ops_name,
205	const char *member_name,	173	const char *member_name,
206	const struct btf_type *func_proto,	174	const struct btf_type func_proto, void stub_func_addr,
207	struct bpf_struct_ops_arg_info *arg_info)	175	struct bpf_struct_ops_arg_info *arg_info)
208	{	176	{
209	const struct btf_type stub_func_proto, pointed_type;	177	const struct btf_type stub_func_proto, pointed_type;
		178	bool is_nullable = false, is_refcounted = false;
210	const struct btf_param stub_args, args;	179	const struct btf_param stub_args, args;
211	struct bpf_ctx_arg_aux info, info_buf;	180	struct bpf_ctx_arg_aux info, info_buf;
212	u32 nargs, arg_no, info_cnt = 0;	181	u32 nargs, arg_no, info_cnt = 0;
		182	char ksym[KSYM_SYMBOL_LEN];
		183	const char *stub_fname;
		184	const char *suffix;
		185	s32 stub_func_id;
213	u32 arg_btf_id;	186	u32 arg_btf_id;
214	int offset;	187	int offset;
215		188
216	stub_func_proto = find_stub_func_proto(btf, st_ops_name, member_name);	189	stub_fname = kallsyms_lookup((unsigned long)stub_func_addr, NULL, NULL, NULL, ksym);
217	if (!stub_func_proto)	190	if (!stub_fname) {
218	return 0;	191	pr_warn("Cannot find the stub function name for the %s in struct %s\n",
		192	member_name, st_ops_name);
		193	return -ENOENT;
		194	}
		195
		196	stub_func_id = btf_find_by_name_kind(btf, stub_fname, BTF_KIND_FUNC);
		197	if (stub_func_id < 0) {
		198	pr_warn("Cannot find the stub function %s in btf\n", stub_fname);
		199	return -ENOENT;
		200	}
		201
		202	stub_func_proto = btf_type_by_id(btf, stub_func_id);
		203	stub_func_proto = btf_type_by_id(btf, stub_func_proto->type);
219		204
220	/* Check if the number of arguments of the stub function is the same	205	/* Check if the number of arguments of the stub function is the same
221	* as the number of arguments of the function pointer.	206	* as the number of arguments of the function pointer.
222	*/	207	*/
223	nargs = btf_type_vlen(func_proto);	208	nargs = btf_type_vlen(func_proto);
224	if (nargs != btf_type_vlen(stub_func_proto)) {	209	if (nargs != btf_type_vlen(stub_func_proto)) {
225	pr_warn("the number of arguments of the stub function %s__%s does not match the number of arguments of the member %s of struct %s\n",	210	pr_warn("the number of arguments of the stub function %s does not match the number of arguments of the member %s of struct %s\n",
226	st_ops_name, member_name, member_name, st_ops_name);	211	stub_fname, member_name, st_ops_name);
227	return -EINVAL;	212	return -EINVAL;
228	}	213	}
229		214
@@ -241,10 +226,18 @@ static int prepare_arg_info(struct btf *btf, · 11 unmodified lines
241	info = info_buf;	226	info = info_buf;
242	for (arg_no = 0; arg_no < nargs; arg_no++) {	227	for (arg_no = 0; arg_no < nargs; arg_no++) {
243	/* Skip arguments that is not suffixed with	228	/* Skip arguments that is not suffixed with
244	* "__nullable".	229	* "__nullable or __ref".
245	*/	230	*/
246	if (!btf_param_match_suffix(btf, &stub_args[arg_no],	231	is_nullable = btf_param_match_suffix(btf, &stub_args[arg_no],
247	MAYBE_NULL_SUFFIX))	232	MAYBE_NULL_SUFFIX);
		233	is_refcounted = btf_param_match_suffix(btf, &stub_args[arg_no],
		234	REFCOUNTED_SUFFIX);
		235
		236	if (is_nullable)
		237	suffix = MAYBE_NULL_SUFFIX;
		238	else if (is_refcounted)
		239	suffix = REFCOUNTED_SUFFIX;
		240	else
248	continue;	241	continue;
249		242
250	/* Should be a pointer to struct */	243	/* Should be a pointer to struct */
@@ -253,30 +246,34 @@ static int prepare_arg_info(struct btf *btf, · 2 unmodified lines
253	&arg_btf_id);	246	&arg_btf_id);
254	if (!pointed_type \|\|	247	if (!pointed_type \|\|
255	!btf_type_is_struct(pointed_type)) {	248	!btf_type_is_struct(pointed_type)) {
256	pr_warn("stub function %s__%s has %s tagging to an unsupported type\n",	249	pr_warn("stub function %s has %s tagging to an unsupported type\n",
257	st_ops_name, member_name, MAYBE_NULL_SUFFIX);	250	stub_fname, suffix);
258	goto err_out;	251	goto err_out;
259	}	252	}
260		253
261	offset = btf_ctx_arg_offset(btf, func_proto, arg_no);	254	offset = btf_ctx_arg_offset(btf, func_proto, arg_no);
262	if (offset < 0) {	255	if (offset < 0) {
263	pr_warn("stub function %s__%s has an invalid trampoline ctx offset for arg#%u\n",	256	pr_warn("stub function %s has an invalid trampoline ctx offset for arg#%u\n",
264	st_ops_name, member_name, arg_no);	257	stub_fname, arg_no);
265	goto err_out;	258	goto err_out;
266	}	259	}
267		260
268	if (args[arg_no].type != stub_args[arg_no].type) {	261	if (args[arg_no].type != stub_args[arg_no].type) {
269	pr_warn("arg#%u type in stub function %s__%s does not match with its original func_proto\n",	262	pr_warn("arg#%u type in stub function %s does not match with its original func_proto\n",
270	arg_no, st_ops_name, member_name);	263	arg_no, stub_fname);
271	goto err_out;	264	goto err_out;
272	}	265	}
273		266
274	/* Fill the information of the new argument */	267	/* Fill the information of the new argument */
275	info->reg_type =
276	PTR_TRUSTED \| PTR_TO_BTF_ID \| PTR_MAYBE_NULL;
277	info->btf_id = arg_btf_id;	268	info->btf_id = arg_btf_id;
278	info->btf = btf;	269	info->btf = btf;
279	info->offset = offset;	270	info->offset = offset;
		271	if (is_nullable) {
		272	info->reg_type = PTR_TRUSTED \| PTR_TO_BTF_ID \| PTR_MAYBE_NULL;
		273	} else if (is_refcounted) {
		274	info->reg_type = PTR_TRUSTED \| PTR_TO_BTF_ID;
		275	info->refcounted = true;
		276	}
280		277
281	info++;	278	info++;
282	info_cnt++;	279	info_cnt++;
@@ -324,6 +321,13 @@ static bool is_module_member(const struct btf *btf, u32 id) · 41 unmodified lines
324	return !strcmp(btf_name_by_offset(btf, t->name_off), "module");	321	return !strcmp(btf_name_by_offset(btf, t->name_off), "module");
325	}	322	}
326		323
		324	int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
		325	{
		326	void func_ptr = (void **)(st_ops->cfi_stubs + moff);
		327
		328	return func_ptr ? 0 : -ENOTSUPP;
		329	}
		330
327	int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,	331	int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc,
328	struct btf *btf,	332	struct btf *btf,
329	struct bpf_verifier_log *log)	333	struct bpf_verifier_log *log)
@@ -386,8 +390,11 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, · 56 unmodified lines
386	st_ops_desc->value_type = btf_type_by_id(btf, value_id);	390	st_ops_desc->value_type = btf_type_by_id(btf, value_id);
387		391
388	for_each_member(i, t, member) {	392	for_each_member(i, t, member) {
389	const struct btf_type *func_proto;	393	const struct btf_type func_proto, ret_type;
		394	void **stub_func_addr;
		395	u32 moff;
390		396
		397	moff = __btf_member_bit_offset(t, member) / 8;
391	mname = btf_name_by_offset(btf, member->name_off);	398	mname = btf_name_by_offset(btf, member->name_off);
392	if (!*mname) {	399	if (!*mname) {
393	pr_warn("anon member in struct %s is not supported\n",	400	pr_warn("anon member in struct %s is not supported\n",
@@ -413,9 +420,23 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, · 19 unmodified lines
413	func_proto = btf_type_resolve_func_ptr(btf,	420	func_proto = btf_type_resolve_func_ptr(btf,
414	member->type,	421	member->type,
415	NULL);	422	NULL);
416	if (!func_proto)	423
		424	/* The member is not a function pointer or
		425	* the function pointer is not supported.
		426	*/
		427	if (!func_proto \|\| bpf_struct_ops_supported(st_ops, moff))
417	continue;	428	continue;
418		429
		430	if (func_proto->type) {
		431	ret_type = btf_type_resolve_ptr(btf, func_proto->type, NULL);
		432	if (ret_type && !__btf_type_is_struct(ret_type)) {
		433	pr_warn("func ptr %s in struct %s returns non-struct pointer, which is not supported\n",
		434	mname, st_ops->name);
		435	err = -EOPNOTSUPP;
		436	goto errout;
		437	}
		438	}
		439
419	if (btf_distill_func_proto(log, btf,	440	if (btf_distill_func_proto(log, btf,
420	func_proto, mname,	441	func_proto, mname,
421	&st_ops->func_models[i])) {	442	&st_ops->func_models[i])) {
@@ -425,8 +446,9 @@ int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, · 3 unmodified lines
425	goto errout;	446	goto errout;
426	}	447	}
427		448
		449	stub_func_addr = (void *)(st_ops->cfi_stubs + moff);
428	err = prepare_arg_info(btf, st_ops->name, mname,	450	err = prepare_arg_info(btf, st_ops->name, mname,
429	func_proto,	451	func_proto, stub_func_addr,
430	arg_info + i);	452	arg_info + i);
431	if (err)	453	if (err)
432	goto errout;	454	goto errout;
@@ -1152,13 +1174,6 @@ void bpf_struct_ops_put(const void *kdata) · 719 unmodified lines
1152	bpf_map_put(&st_map->map);	1174	bpf_map_put(&st_map->map);
1153	}	1175	}
1154		1176
1155	int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff)
1156	{
1157	void func_ptr = (void **)(st_ops->cfi_stubs + moff);
1158
1159	return func_ptr ? 0 : -ENOTSUPP;
1160	}
1161
1162	static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)	1177	static bool bpf_struct_ops_valid_to_reg(struct bpf_map *map)
1163	{	1178	{
1164	struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;	1179	struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;

kernel/bpf/btf.c+113−14

@@ -606,6 +606,7 @@ s32 bpf_find_btf_id(const char name, u32 kind, struct btf *btf_p) · 605 unmodified lines
606	spin_unlock_bh(&btf_idr_lock);	606	spin_unlock_bh(&btf_idr_lock);
607	return ret;	607	return ret;
608	}	608	}
		609	EXPORT_SYMBOL_GPL(bpf_find_btf_id);
609		610
610	const struct btf_type btf_type_skip_modifiers(const struct btf btf,	611	const struct btf_type btf_type_skip_modifiers(const struct btf btf,
611	u32 id, u32 *res_id)	612	u32 id, u32 *res_id)
@@ -2575,7 +2576,7 @@ static int btf_ref_type_check_meta(struct btf_verifier_env *env, · 1963 unmodified lines
2575	return -EINVAL;	2576	return -EINVAL;
2576	}	2577	}
2577		2578
2578	if (btf_type_kflag(t)) {	2579	if (btf_type_kflag(t) && !btf_type_is_type_tag(t)) {
2579	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");	2580	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
2580	return -EINVAL;	2581	return -EINVAL;
2581	}	2582	}
@@ -3332,6 +3333,8 @@ static int btf_find_kptr(const struct btf btf, const struct btf_type t, · 750 unmodified lines
3332	u32 off, int sz, struct btf_field_info *info, u32 field_mask)	3333	u32 off, int sz, struct btf_field_info *info, u32 field_mask)
3333	{	3334	{
3334	enum btf_field_type type;	3335	enum btf_field_type type;
		3336	const char *tag_value;
		3337	bool is_type_tag;
3335	u32 res_id;	3338	u32 res_id;
3336		3339
3337	/* Permit modifiers on the pointer itself */	3340	/* Permit modifiers on the pointer itself */
@@ -3341,19 +3344,20 @@ static int btf_find_kptr(const struct btf btf, const struct btf_type t, · 3 unmodified lines
3341	if (!btf_type_is_ptr(t))	3344	if (!btf_type_is_ptr(t))
3342	return BTF_FIELD_IGNORE;	3345	return BTF_FIELD_IGNORE;
3343	t = btf_type_by_id(btf, t->type);	3346	t = btf_type_by_id(btf, t->type);
3344		3347	is_type_tag = btf_type_is_type_tag(t) && !btf_type_kflag(t);
3345	if (!btf_type_is_type_tag(t))	3348	if (!is_type_tag)
3346	return BTF_FIELD_IGNORE;	3349	return BTF_FIELD_IGNORE;
3347	/* Reject extra tags */	3350	/* Reject extra tags */
3348	if (btf_type_is_type_tag(btf_type_by_id(btf, t->type)))	3351	if (btf_type_is_type_tag(btf_type_by_id(btf, t->type)))
3349	return -EINVAL;	3352	return -EINVAL;
3350	if (!strcmp("kptr_untrusted", __btf_name_by_offset(btf, t->name_off)))	3353	tag_value = __btf_name_by_offset(btf, t->name_off);
		3354	if (!strcmp("kptr_untrusted", tag_value))
3351	type = BPF_KPTR_UNREF;	3355	type = BPF_KPTR_UNREF;
3352	else if (!strcmp("kptr", __btf_name_by_offset(btf, t->name_off)))	3356	else if (!strcmp("kptr", tag_value))
3353	type = BPF_KPTR_REF;	3357	type = BPF_KPTR_REF;
3354	else if (!strcmp("percpu_kptr", __btf_name_by_offset(btf, t->name_off)))	3358	else if (!strcmp("percpu_kptr", tag_value))
3355	type = BPF_KPTR_PERCPU;	3359	type = BPF_KPTR_PERCPU;
3356	else if (!strcmp("uptr", __btf_name_by_offset(btf, t->name_off)))	3360	else if (!strcmp("uptr", tag_value))
3357	type = BPF_UPTR;	3361	type = BPF_UPTR;
3358	else	3362	else
3359	return -EINVAL;	3363	return -EINVAL;
@@ -4944,11 +4948,6 @@ static s32 btf_decl_tag_check_meta(struct btf_verifier_env *env, · 1584 unmodified lines
4944	return -EINVAL;	4948	return -EINVAL;
4945	}	4949	}
4946		4950
4947	if (btf_type_kflag(t)) {
4948	btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
4949	return -EINVAL;
4950	}
4951
4952	component_idx = btf_type_decl_tag(t)->component_idx;	4951	component_idx = btf_type_decl_tag(t)->component_idx;
4953	if (component_idx < -1) {	4952	if (component_idx < -1) {
4954	btf_verifier_log_type(env, t, "Invalid component_idx");	4953	btf_verifier_log_type(env, t, "Invalid component_idx");
@@ -6507,6 +6506,8 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { · 1552 unmodified lines
6507	/* rxrpc */	6506	/* rxrpc */
6508	{ "rxrpc_recvdata", 0x1 },	6507	{ "rxrpc_recvdata", 0x1 },
6509	{ "rxrpc_resend", 0x10 },	6508	{ "rxrpc_resend", 0x10 },
		6509	{ "rxrpc_tq", 0x10 },
		6510	{ "rxrpc_client", 0x1 },
6510	/* skb */	6511	/* skb */
6511	{"kfree_skb", 0x1000},	6512	{"kfree_skb", 0x1000},
6512	/* sunrpc */	6513	/* sunrpc */
@@ -6529,6 +6530,103 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { · 16 unmodified lines
6529	{ "mr_integ_alloc", 0x2000 },	6530	{ "mr_integ_alloc", 0x2000 },
6530	/* bpf_testmod */	6531	/* bpf_testmod */
6531	{ "bpf_testmod_test_read", 0x0 },	6532	{ "bpf_testmod_test_read", 0x0 },
		6533	/* amdgpu */
		6534	{ "amdgpu_vm_bo_map", 0x1 },
		6535	{ "amdgpu_vm_bo_unmap", 0x1 },
		6536	/* netfs */
		6537	{ "netfs_folioq", 0x1 },
		6538	/* xfs from xfs_defer_pending_class */
		6539	{ "xfs_defer_create_intent", 0x1 },
		6540	{ "xfs_defer_cancel_list", 0x1 },
		6541	{ "xfs_defer_pending_finish", 0x1 },
		6542	{ "xfs_defer_pending_abort", 0x1 },
		6543	{ "xfs_defer_relog_intent", 0x1 },
		6544	{ "xfs_defer_isolate_paused", 0x1 },
		6545	{ "xfs_defer_item_pause", 0x1 },
		6546	{ "xfs_defer_item_unpause", 0x1 },
		6547	/* xfs from xfs_defer_pending_item_class */
		6548	{ "xfs_defer_add_item", 0x1 },
		6549	{ "xfs_defer_cancel_item", 0x1 },
		6550	{ "xfs_defer_finish_item", 0x1 },
		6551	/* xfs from xfs_icwalk_class */
		6552	{ "xfs_ioc_free_eofblocks", 0x10 },
		6553	{ "xfs_blockgc_free_space", 0x10 },
		6554	/* xfs from xfs_btree_cur_class */
		6555	{ "xfs_btree_updkeys", 0x100 },
		6556	{ "xfs_btree_overlapped_query_range", 0x100 },
		6557	/* xfs from xfs_imap_class*/
		6558	{ "xfs_map_blocks_found", 0x10000 },
		6559	{ "xfs_map_blocks_alloc", 0x10000 },
		6560	{ "xfs_iomap_alloc", 0x1000 },
		6561	{ "xfs_iomap_found", 0x1000 },
		6562	/* xfs from xfs_fs_class */
		6563	{ "xfs_inodegc_flush", 0x1 },
		6564	{ "xfs_inodegc_push", 0x1 },
		6565	{ "xfs_inodegc_start", 0x1 },
		6566	{ "xfs_inodegc_stop", 0x1 },
		6567	{ "xfs_inodegc_queue", 0x1 },
		6568	{ "xfs_inodegc_throttle", 0x1 },
		6569	{ "xfs_fs_sync_fs", 0x1 },
		6570	{ "xfs_blockgc_start", 0x1 },
		6571	{ "xfs_blockgc_stop", 0x1 },
		6572	{ "xfs_blockgc_worker", 0x1 },
		6573	{ "xfs_blockgc_flush_all", 0x1 },
		6574	/* xfs_scrub */
		6575	{ "xchk_nlinks_live_update", 0x10 },
		6576	/* xfs_scrub from xchk_metapath_class */
		6577	{ "xchk_metapath_lookup", 0x100 },
		6578	/* nfsd */
		6579	{ "nfsd_dirent", 0x1 },
		6580	{ "nfsd_file_acquire", 0x1001 },
		6581	{ "nfsd_file_insert_err", 0x1 },
		6582	{ "nfsd_file_cons_err", 0x1 },
		6583	/* nfs4 */
		6584	{ "nfs4_setup_sequence", 0x1 },
		6585	{ "pnfs_update_layout", 0x10000 },
		6586	{ "nfs4_inode_callback_event", 0x200 },
		6587	{ "nfs4_inode_stateid_callback_event", 0x200 },
		6588	/* nfs from pnfs_layout_event */
		6589	{ "pnfs_mds_fallback_pg_init_read", 0x10000 },
		6590	{ "pnfs_mds_fallback_pg_init_write", 0x10000 },
		6591	{ "pnfs_mds_fallback_pg_get_mirror_count", 0x10000 },
		6592	{ "pnfs_mds_fallback_read_done", 0x10000 },
		6593	{ "pnfs_mds_fallback_write_done", 0x10000 },
		6594	{ "pnfs_mds_fallback_read_pagelist", 0x10000 },
		6595	{ "pnfs_mds_fallback_write_pagelist", 0x10000 },
		6596	/* coda */
		6597	{ "coda_dec_pic_run", 0x10 },
		6598	{ "coda_dec_pic_done", 0x10 },
		6599	/* cfg80211 */
		6600	{ "cfg80211_scan_done", 0x11 },
		6601	{ "rdev_set_coalesce", 0x10 },
		6602	{ "cfg80211_report_wowlan_wakeup", 0x100 },
		6603	{ "cfg80211_inform_bss_frame", 0x100 },
		6604	{ "cfg80211_michael_mic_failure", 0x10000 },
		6605	/* cfg80211 from wiphy_work_event */
		6606	{ "wiphy_work_queue", 0x10 },
		6607	{ "wiphy_work_run", 0x10 },
		6608	{ "wiphy_work_cancel", 0x10 },
		6609	{ "wiphy_work_flush", 0x10 },
		6610	/* hugetlbfs */
		6611	{ "hugetlbfs_alloc_inode", 0x10 },
		6612	/* spufs */
		6613	{ "spufs_context", 0x10 },
		6614	/* kvm_hv */
		6615	{ "kvm_page_fault_enter", 0x100 },
		6616	/* dpu */
		6617	{ "dpu_crtc_setup_mixer", 0x100 },
		6618	/* binder */
		6619	{ "binder_transaction", 0x100 },
		6620	/* bcachefs */
		6621	{ "btree_path_free", 0x100 },
		6622	/* hfi1_tx */
		6623	{ "hfi1_sdma_progress", 0x1000 },
		6624	/* iptfs */
		6625	{ "iptfs_ingress_postq_event", 0x1000 },
		6626	/* neigh */
		6627	{ "neigh_update", 0x10 },
		6628	/* snd_firewire_lib */
		6629	{ "amdtp_packet", 0x100 },
6532	};	6630	};
6533		6631
6534	bool btf_ctx_access(int off, int size, enum bpf_access_type type,	6632	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
@@ -6679,6 +6777,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 144 unmodified lines
6679	info->reg_type = ctx_arg_info->reg_type;	6777	info->reg_type = ctx_arg_info->reg_type;
6680	info->btf = ctx_arg_info->btf ? : btf_vmlinux;	6778	info->btf = ctx_arg_info->btf ? : btf_vmlinux;
6681	info->btf_id = ctx_arg_info->btf_id;	6779	info->btf_id = ctx_arg_info->btf_id;
		6780	info->ref_obj_id = ctx_arg_info->ref_obj_id;
6682	return true;	6781	return true;
6683	}	6782	}
6684	}	6783	}
@@ -6745,7 +6844,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 60 unmodified lines
6745	info->btf_id = t->type;	6844	info->btf_id = t->type;
6746	t = btf_type_by_id(btf, t->type);	6845	t = btf_type_by_id(btf, t->type);
6747		6846
6748	if (btf_type_is_type_tag(t)) {	6847	if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) {
6749	tag_value = __btf_name_by_offset(btf, t->name_off);	6848	tag_value = __btf_name_by_offset(btf, t->name_off);
6750	if (strcmp(tag_value, "user") == 0)	6849	if (strcmp(tag_value, "user") == 0)
6751	info->reg_type \|= MEM_USER;	6850	info->reg_type \|= MEM_USER;
@@ -7004,7 +7103,7 @@ error: · 252 unmodified lines
7004		7103
7005	/* check type tag */	7104	/* check type tag */
7006	t = btf_type_by_id(btf, mtype->type);	7105	t = btf_type_by_id(btf, mtype->type);
7007	if (btf_type_is_type_tag(t)) {	7106	if (btf_type_is_type_tag(t) && !btf_type_kflag(t)) {
7008	tag_value = __btf_name_by_offset(btf, t->name_off);	7107	tag_value = __btf_name_by_offset(btf, t->name_off);
7009	/* check __user tag */	7108	/* check __user tag */
7010	if (strcmp(tag_value, "user") == 0)	7109	if (strcmp(tag_value, "user") == 0)

kernel/bpf/cgroup.c+25−8

 /* count number of elements in the list.
  * it's slow but the list cannot be long
  */
-static u32 prog_list_length(struct hlist_head *head)
+static u32 prog_list_length(struct hlist_head *head, int *preorder_cnt)
 {
 	struct bpf_prog_list *pl;
 	u32 cnt = 0;
 	hlist_for_each_entry(pl, head, node) {
 		if (!prog_list_prog(pl))
 			continue;
+		if (preorder_cnt && (pl->flags & BPF_F_PREORDER))
+			(*preorder_cnt)++;
 		cnt++;
 	}
 	return cnt;
 		if (flags & BPF_F_ALLOW_MULTI)
 			return true;
-		cnt = prog_list_length(&p->bpf.progs[atype]);
+		cnt = prog_list_length(&p->bpf.progs[atype], NULL);
 		WARN_ON_ONCE(cnt > 1);
 		if (cnt == 1)
 			return !!(flags & BPF_F_ALLOW_OVERRIDE);
 	struct bpf_prog_array *progs;
 	struct bpf_prog_list *pl;
 	struct cgroup *p = cgrp;
-	int cnt = 0;
+	int i, j, cnt = 0, preorder_cnt = 0, fstart, bstart, init_bstart;
 	/* count number of effective programs by walking parents */
 	do {
 		if (cnt == 0 || (p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
-			cnt += prog_list_length(&p->bpf.progs[atype]);
+			cnt += prog_list_length(&p->bpf.progs[atype], &preorder_cnt);
 		p = cgroup_parent(p);
 	} while (p);
 	/* populate the array with effective progs */
 	cnt = 0;
 	p = cgrp;
+	fstart = preorder_cnt;
+	bstart = preorder_cnt - 1;
 	do {
 		if (cnt > 0 && !(p->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
 			continue;
+		init_bstart = bstart;
 		hlist_for_each_entry(pl, &p->bpf.progs[atype], node) {
 			if (!prog_list_prog(pl))
 				continue;
-			item = &progs->items[cnt];
+			if (pl->flags & BPF_F_PREORDER) {
+				item = &progs->items[bstart];
+				bstart--;
+			} else {
+				item = &progs->items[fstart];
+				fstart++;
+			}
 			item->prog = prog_list_prog(pl);
 			bpf_cgroup_storages_assign(item->cgroup_storage,
 						   pl->storage);
 			cnt++;
 		}
+		/* reverse pre-ordering progs at this cgroup level */
+		for (i = bstart + 1, j = init_bstart; i < j; i++, j--)
+			swap(progs->items[i], progs->items[j]);
 	} while ((p = cgroup_parent(p)));
 	*array = progs;
 		 */
 		return -EPERM;
-	if (prog_list_length(progs) >= BPF_CGROUP_MAX_PROGS)
+	if (prog_list_length(progs, NULL) >= BPF_CGROUP_MAX_PROGS)
 		return -E2BIG;
 	pl = find_attach_entry(progs, prog, link, replace_prog,
 	pl->prog = prog;
 	pl->link = link;
+	pl->flags = flags;
 	bpf_cgroup_storages_assign(pl->storage, storage);
 	cgrp->bpf.flags[atype] = saved_flags;
 							      lockdep_is_held(&cgroup_mutex));
 			total_cnt += bpf_prog_array_length(effective);
 		} else {
-			total_cnt += prog_list_length(&cgrp->bpf.progs[atype]);
+			total_cnt += prog_list_length(&cgrp->bpf.progs[atype], NULL);
 		}
 	}
 			u32 id;
 			progs = &cgrp->bpf.progs[atype];
-			cnt = min_t(int, prog_list_length(progs), total_cnt);
+			cnt = min_t(int, prog_list_length(progs, NULL), total_cnt);
 			i = 0;
 			hlist_for_each_entry(pl, progs, node) {
 				prog = prog_list_prog(pl);

kernel/bpf/core.c+107−10

@@ -1663,14 +1663,17 @@ EXPORT_SYMBOL_GPL(__bpf_call_base); · 1662 unmodified lines
1663	INSN_3(JMP, JSET, K), \	1663	INSN_3(JMP, JSET, K), \
1664	INSN_2(JMP, JA), \	1664	INSN_2(JMP, JA), \
1665	INSN_2(JMP32, JA), \	1665	INSN_2(JMP32, JA), \
		1666	/* Atomic operations. */ \
		1667	INSN_3(STX, ATOMIC, B), \
		1668	INSN_3(STX, ATOMIC, H), \
		1669	INSN_3(STX, ATOMIC, W), \
		1670	INSN_3(STX, ATOMIC, DW), \
1666	/* Store instructions. */ \	1671	/* Store instructions. */ \
1667	/* Register based. */ \	1672	/* Register based. */ \
1668	INSN_3(STX, MEM, B), \	1673	INSN_3(STX, MEM, B), \
1669	INSN_3(STX, MEM, H), \	1674	INSN_3(STX, MEM, H), \
1670	INSN_3(STX, MEM, W), \	1675	INSN_3(STX, MEM, W), \
1671	INSN_3(STX, MEM, DW), \	1676	INSN_3(STX, MEM, DW), \
1672	INSN_3(STX, ATOMIC, W), \
1673	INSN_3(STX, ATOMIC, DW), \
1674	/* Immediate based. */ \	1677	/* Immediate based. */ \
1675	INSN_3(ST, MEM, B), \	1678	INSN_3(ST, MEM, B), \
1676	INSN_3(ST, MEM, H), \	1679	INSN_3(ST, MEM, H), \
@@ -2152,24 +2155,33 @@ out: · 475 unmodified lines
2152	if (BPF_SIZE(insn->code) == BPF_W) \	2155	if (BPF_SIZE(insn->code) == BPF_W) \
2153	atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \	2156	atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \
2154	(DST + insn->off)); \	2157	(DST + insn->off)); \
2155	else \	2158	else if (BPF_SIZE(insn->code) == BPF_DW) \
2156	atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \	2159	atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \
2157	(DST + insn->off)); \	2160	(DST + insn->off)); \
		2161	else \
		2162	goto default_label; \
2158	break; \	2163	break; \
2159	case BOP \| BPF_FETCH: \	2164	case BOP \| BPF_FETCH: \
2160	if (BPF_SIZE(insn->code) == BPF_W) \	2165	if (BPF_SIZE(insn->code) == BPF_W) \
2161	SRC = (u32) atomic_fetch_##KOP( \	2166	SRC = (u32) atomic_fetch_##KOP( \
2162	(u32) SRC, \	2167	(u32) SRC, \
2163	(atomic_t *)(unsigned long) (DST + insn->off)); \	2168	(atomic_t *)(unsigned long) (DST + insn->off)); \
2164	else \	2169	else if (BPF_SIZE(insn->code) == BPF_DW) \
2165	SRC = (u64) atomic64_fetch_##KOP( \	2170	SRC = (u64) atomic64_fetch_##KOP( \
2166	(u64) SRC, \	2171	(u64) SRC, \
2167	(atomic64_t *)(unsigned long) (DST + insn->off)); \	2172	(atomic64_t *)(unsigned long) (DST + insn->off)); \
		2173	else \
		2174	goto default_label; \
2168	break;	2175	break;
2169		2176
2170	STX_ATOMIC_DW:	2177	STX_ATOMIC_DW:
2171	STX_ATOMIC_W:	2178	STX_ATOMIC_W:
		2179	STX_ATOMIC_H:
		2180	STX_ATOMIC_B:
2172	switch (IMM) {	2181	switch (IMM) {
		2182	/* Atomic read-modify-write instructions support only W and DW
		2183	* size modifiers.
		2184	*/
2173	ATOMIC_ALU_OP(BPF_ADD, add)	2185	ATOMIC_ALU_OP(BPF_ADD, add)
2174	ATOMIC_ALU_OP(BPF_AND, and)	2186	ATOMIC_ALU_OP(BPF_AND, and)
2175	ATOMIC_ALU_OP(BPF_OR, or)	2187	ATOMIC_ALU_OP(BPF_OR, or)
@@ -2181,20 +2193,63 @@ out: · 5 unmodified lines
2181	SRC = (u32) atomic_xchg(	2193	SRC = (u32) atomic_xchg(
2182	(atomic_t *)(unsigned long) (DST + insn->off),	2194	(atomic_t *)(unsigned long) (DST + insn->off),
2183	(u32) SRC);	2195	(u32) SRC);
2184	else	2196	else if (BPF_SIZE(insn->code) == BPF_DW)
2185	SRC = (u64) atomic64_xchg(	2197	SRC = (u64) atomic64_xchg(
2186	(atomic64_t *)(unsigned long) (DST + insn->off),	2198	(atomic64_t *)(unsigned long) (DST + insn->off),
2187	(u64) SRC);	2199	(u64) SRC);
		2200	else
		2201	goto default_label;
2188	break;	2202	break;
2189	case BPF_CMPXCHG:	2203	case BPF_CMPXCHG:
2190	if (BPF_SIZE(insn->code) == BPF_W)	2204	if (BPF_SIZE(insn->code) == BPF_W)
2191	BPF_R0 = (u32) atomic_cmpxchg(	2205	BPF_R0 = (u32) atomic_cmpxchg(
2192	(atomic_t *)(unsigned long) (DST + insn->off),	2206	(atomic_t *)(unsigned long) (DST + insn->off),
2193	(u32) BPF_R0, (u32) SRC);	2207	(u32) BPF_R0, (u32) SRC);
2194	else	2208	else if (BPF_SIZE(insn->code) == BPF_DW)
2195	BPF_R0 = (u64) atomic64_cmpxchg(	2209	BPF_R0 = (u64) atomic64_cmpxchg(
2196	(atomic64_t *)(unsigned long) (DST + insn->off),	2210	(atomic64_t *)(unsigned long) (DST + insn->off),
2197	(u64) BPF_R0, (u64) SRC);	2211	(u64) BPF_R0, (u64) SRC);
		2212	else
		2213	goto default_label;
		2214	break;
		2215	/* Atomic load and store instructions support all size
		2216	* modifiers.
		2217	*/
		2218	case BPF_LOAD_ACQ:
		2219	switch (BPF_SIZE(insn->code)) {
		2220	#define LOAD_ACQUIRE(SIZEOP, SIZE) \
		2221	case BPF_##SIZEOP: \
		2222	DST = (SIZE)smp_load_acquire( \
		2223	(SIZE *)(unsigned long)(SRC + insn->off)); \
		2224	break;
		2225	LOAD_ACQUIRE(B, u8)
		2226	LOAD_ACQUIRE(H, u16)
		2227	LOAD_ACQUIRE(W, u32)
		2228	#ifdef CONFIG_64BIT
		2229	LOAD_ACQUIRE(DW, u64)
		2230	#endif
		2231	#undef LOAD_ACQUIRE
		2232	default:
		2233	goto default_label;
		2234	}
		2235	break;
		2236	case BPF_STORE_REL:
		2237	switch (BPF_SIZE(insn->code)) {
		2238	#define STORE_RELEASE(SIZEOP, SIZE) \
		2239	case BPF_##SIZEOP: \
		2240	smp_store_release( \
		2241	(SIZE *)(unsigned long)(DST + insn->off), (SIZE)SRC); \
		2242	break;
		2243	STORE_RELEASE(B, u8)
		2244	STORE_RELEASE(H, u16)
		2245	STORE_RELEASE(W, u32)
		2246	#ifdef CONFIG_64BIT
		2247	STORE_RELEASE(DW, u64)
		2248	#endif
		2249	#undef STORE_RELEASE
		2250	default:
		2251	goto default_label;
		2252	}
2198	break;	2253	break;
2199		2254
2200	default:	2255	default:
@@ -2290,17 +2345,18 @@ void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) · 89 unmodified lines
2290	insn->code = BPF_JMP \| BPF_CALL_ARGS;	2345	insn->code = BPF_JMP \| BPF_CALL_ARGS;
2291	}	2346	}
2292	#endif	2347	#endif
2293	#else	2348	#endif
		2349
2294	static unsigned int __bpf_prog_ret0_warn(const void *ctx,	2350	static unsigned int __bpf_prog_ret0_warn(const void *ctx,
2295	const struct bpf_insn *insn)	2351	const struct bpf_insn *insn)
2296	{	2352	{
2297	/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON	2353	/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON
2298	* is not working properly, so warn about it!	2354	* is not working properly, or interpreter is being used when
		2355	* prog->jit_requested is not 0, so warn about it!
2299	*/	2356	*/
2300	WARN_ON_ONCE(1);	2357	WARN_ON_ONCE(1);
2301	return 0;	2358	return 0;
2302	}	2359	}
2303	#endif
2304		2360
2305	bool bpf_prog_map_compatible(struct bpf_map *map,	2361	bool bpf_prog_map_compatible(struct bpf_map *map,
2306	const struct bpf_prog *fp)	2362	const struct bpf_prog *fp)
@@ -2380,8 +2436,18 @@ static void bpf_prog_select_func(struct bpf_prog *fp) · 73 unmodified lines
2380	{	2436	{
2381	#ifndef CONFIG_BPF_JIT_ALWAYS_ON	2437	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
2382	u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);	2438	u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);
		2439	u32 idx = (round_up(stack_depth, 32) / 32) - 1;
2383		2440
2384	fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1];	2441	/* may_goto may cause stack size > 512, leading to idx out-of-bounds.
		2442	* But for non-JITed programs, we don't need bpf_func, so no bounds
		2443	* check needed.
		2444	*/
		2445	if (!fp->jit_requested &&
		2446	!WARN_ON_ONCE(idx >= ARRAY_SIZE(interpreters))) {
		2447	fp->bpf_func = interpreters[idx];
		2448	} else {
		2449	fp->bpf_func = __bpf_prog_ret0_warn;
		2450	}
2385	#else	2451	#else
2386	fp->bpf_func = __bpf_prog_ret0_warn;	2452	fp->bpf_func = __bpf_prog_ret0_warn;
2387	#endif	2453	#endif
@@ -2906,6 +2972,11 @@ const struct bpf_func_proto * __weak bpf_get_trace_vprintk_proto(void) · 518 unmodified lines
2906	return NULL;	2972	return NULL;
2907	}	2973	}
2908		2974
		2975	const struct bpf_func_proto * __weak bpf_get_perf_event_read_value_proto(void)
		2976	{
		2977	return NULL;
		2978	}
		2979
2909	u64 __weak	2980	u64 __weak
2910	bpf_event_output(struct bpf_map map, u64 flags, void meta, u64 meta_size,	2981	bpf_event_output(struct bpf_map map, u64 flags, void meta, u64 meta_size,
2911	void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)	2982	void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
@@ -3058,6 +3129,32 @@ void __weak arch_bpf_stack_walk(bool (consume_fn)(void cookie, u64 ip, u64 sp, · 146 unmodified lines
3058	{	3129	{
3059	}	3130	}
3060		3131
		3132	bool __weak bpf_jit_supports_timed_may_goto(void)
		3133	{
		3134	return false;
		3135	}
		3136
		3137	u64 __weak arch_bpf_timed_may_goto(void)
		3138	{
		3139	return 0;
		3140	}
		3141
		3142	u64 bpf_check_timed_may_goto(struct bpf_timed_may_goto *p)
		3143	{
		3144	u64 time = ktime_get_mono_fast_ns();
		3145
		3146	/* Populate the timestamp for this stack frame, and refresh count. */
		3147	if (!p->timestamp) {
		3148	p->timestamp = time;
		3149	return BPF_MAX_TIMED_LOOPS;
		3150	}
		3151	/* Check if we've exhausted our time slice, and zero count. */
		3152	if (time - p->timestamp >= (NSEC_PER_SEC / 4))
		3153	return 0;
		3154	/* Refresh the count for the stack frame. */
		3155	return BPF_MAX_TIMED_LOOPS;
		3156	}
		3157
3061	/* for configs without MMU or 32-bit */	3158	/* for configs without MMU or 32-bit */
3062	__weak const struct bpf_map_ops arena_map_ops;	3159	__weak const struct bpf_map_ops arena_map_ops;
3063	__weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)	3160	__weak u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena)

kernel/bpf/cpumask.c+53−0

@@ -45,6 +45,10 @@ __bpf_kfunc_start_defs(); · 44 unmodified lines
45	*	45	*
46	* bpf_cpumask_create() allocates memory using the BPF memory allocator, and	46	* bpf_cpumask_create() allocates memory using the BPF memory allocator, and
47	* will not block. It may return NULL if no memory is available.	47	* will not block. It may return NULL if no memory is available.
		48	*
		49	* Return:
		50	* * A pointer to a new struct bpf_cpumask instance on success.
		51	* * NULL if the BPF memory allocator is out of memory.
48	*/	52	*/
49	__bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void)	53	__bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void)
50	{	54	{
@@ -71,6 +75,10 @@ __bpf_kfunc struct bpf_cpumask *bpf_cpumask_create(void) · 20 unmodified lines
71	* Acquires a reference to a BPF cpumask. The cpumask returned by this function	75	* Acquires a reference to a BPF cpumask. The cpumask returned by this function
72	* must either be embedded in a map as a kptr, or freed with	76	* must either be embedded in a map as a kptr, or freed with
73	* bpf_cpumask_release().	77	* bpf_cpumask_release().
		78	*
		79	* Return:
		80	* * The struct bpf_cpumask pointer passed to the function.
		81	*
74	*/	82	*/
75	__bpf_kfunc struct bpf_cpumask bpf_cpumask_acquire(struct bpf_cpumask cpumask)	83	__bpf_kfunc struct bpf_cpumask bpf_cpumask_acquire(struct bpf_cpumask cpumask)
76	{	84	{
@@ -106,6 +114,9 @@ CFI_NOSEAL(bpf_cpumask_release_dtor); · 29 unmodified lines
106	*	114	*
107	* Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask	115	* Find the index of the first nonzero bit of the cpumask. A struct bpf_cpumask
108	* pointer may be safely passed to this function.	116	* pointer may be safely passed to this function.
		117	*
		118	* Return:
		119	* * The index of the first nonzero bit in the struct cpumask.
109	*/	120	*/
110	__bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask)	121	__bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask)
111	{	122	{
@@ -119,6 +130,9 @@ __bpf_kfunc u32 bpf_cpumask_first(const struct cpumask *cpumask) · 7 unmodified lines
119	*	130	*
120	* Find the index of the first unset bit of the cpumask. A struct bpf_cpumask	131	* Find the index of the first unset bit of the cpumask. A struct bpf_cpumask
121	* pointer may be safely passed to this function.	132	* pointer may be safely passed to this function.
		133	*
		134	* Return:
		135	* * The index of the first zero bit in the struct cpumask.
122	*/	136	*/
123	__bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask)	137	__bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask)
124	{	138	{
@@ -133,6 +147,9 @@ __bpf_kfunc u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) · 8 unmodified lines
133	*	147	*
134	* Find the index of the first nonzero bit of the AND of two cpumasks.	148	* Find the index of the first nonzero bit of the AND of two cpumasks.
135	* struct bpf_cpumask pointers may be safely passed to @src1 and @src2.	149	* struct bpf_cpumask pointers may be safely passed to @src1 and @src2.
		150	*
		151	* Return:
		152	* * The index of the first bit that is nonzero in both cpumask instances.
136	*/	153	*/
137	__bpf_kfunc u32 bpf_cpumask_first_and(const struct cpumask *src1,	154	__bpf_kfunc u32 bpf_cpumask_first_and(const struct cpumask *src1,
138	const struct cpumask *src2)	155	const struct cpumask *src2)
@@ -414,12 +431,47 @@ __bpf_kfunc u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1, · 275 unmodified lines
414	* @cpumask: The cpumask being queried.	431	* @cpumask: The cpumask being queried.
415	*	432	*
416	* Count the number of set bits in the given cpumask.	433	* Count the number of set bits in the given cpumask.
		434	*
		435	* Return:
		436	* * The number of bits set in the mask.
417	*/	437	*/
418	__bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask)	438	__bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask)
419	{	439	{
420	return cpumask_weight(cpumask);	440	return cpumask_weight(cpumask);
421	}	441	}
422		442
		443	/**
		444	* bpf_cpumask_populate() - Populate the CPU mask from the contents of
		445	* a BPF memory region.
		446	*
		447	* @cpumask: The cpumask being populated.
		448	* @src: The BPF memory holding the bit pattern.
		449	* @src__sz: Length of the BPF memory region in bytes.
		450	*
		451	* Return:
		452	* * 0 if the struct cpumask * instance was populated successfully.
		453	* * -EACCES if the memory region is too small to populate the cpumask.
		454	* * -EINVAL if the memory region is not aligned to the size of a long
		455	* and the architecture does not support efficient unaligned accesses.
		456	*/
		457	__bpf_kfunc int bpf_cpumask_populate(struct cpumask cpumask, void src, size_t src__sz)
		458	{
		459	unsigned long source = (unsigned long)src;
		460
		461	/* The memory region must be large enough to populate the entire CPU mask. */
		462	if (src__sz < bitmap_size(nr_cpu_ids))
		463	return -EACCES;
		464
		465	/* If avoiding unaligned accesses, the input region must be aligned to the nearest long. */
		466	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
		467	!IS_ALIGNED(source, sizeof(long)))
		468	return -EINVAL;
		469
		470	bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids);
		471
		472	return 0;
		473	}
		474
423	__bpf_kfunc_end_defs();	475	__bpf_kfunc_end_defs();
424		476
425	BTF_KFUNCS_START(cpumask_kfunc_btf_ids)	477	BTF_KFUNCS_START(cpumask_kfunc_btf_ids)
@@ -448,6 +500,7 @@ BTF_ID_FLAGS(func, bpf_cpumask_copy, KF_RCU) · 22 unmodified lines
448	BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU)	500	BTF_ID_FLAGS(func, bpf_cpumask_any_distribute, KF_RCU)
449	BTF_ID_FLAGS(func, bpf_cpumask_any_and_distribute, KF_RCU)	501	BTF_ID_FLAGS(func, bpf_cpumask_any_and_distribute, KF_RCU)
450	BTF_ID_FLAGS(func, bpf_cpumask_weight, KF_RCU)	502	BTF_ID_FLAGS(func, bpf_cpumask_weight, KF_RCU)
		503	BTF_ID_FLAGS(func, bpf_cpumask_populate, KF_RCU)
451	BTF_KFUNCS_END(cpumask_kfunc_btf_ids)	504	BTF_KFUNCS_END(cpumask_kfunc_btf_ids)
452		505
453	static const struct btf_kfunc_id_set cpumask_kfunc_set = {	506	static const struct btf_kfunc_id_set cpumask_kfunc_set = {

kernel/bpf/disasm.c+14−2

 				insn->dst_reg, class == BPF_ALU ? 'w' : 'r',
 				insn->dst_reg);
 		} else if (is_addr_space_cast(insn)) {
-			verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %d, %d)\n",
+			verbose(cbs->private_data, "(%02x) r%d = addr_space_cast(r%d, %u, %u)\n",
 				insn->code, insn->dst_reg,
 				insn->src_reg, ((u32)insn->imm) >> 16, (u16)insn->imm);
 		} else if (is_mov_percpu_addr(insn)) {
 				BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
 				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
 				insn->dst_reg, insn->off, insn->src_reg);
+		} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
+			   insn->imm == BPF_LOAD_ACQ) {
+			verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n",
+				insn->code, insn->dst_reg,
+				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
+				insn->src_reg, insn->off);
+		} else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
+			   insn->imm == BPF_STORE_REL) {
+			verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n",
+				insn->code,
+				bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
+				insn->dst_reg, insn->off, insn->src_reg);
 		} else {
 			verbose(cbs->private_data, "BUG_%02x\n", insn->code);
 		}
 				insn->code, class == BPF_JMP32 ? 'w' : 'r',
 				insn->dst_reg,
 				bpf_jmp_string[BPF_OP(insn->code) >> 4],
-				insn->imm, insn->off);
+				(u32)insn->imm, insn->off);
 		}
 	} else {
 		verbose(cbs->private_data, "(%02x) %s\n",

kernel/bpf/hashtab.c+6−3

 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size,
 				     void __percpu *pptr)
 {
-	*(void __percpu **)(l->key + key_size) = pptr;
+	*(void __percpu **)(l->key + roundup(key_size, 8)) = pptr;
 }
 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size)
 {
-	return *(void __percpu **)(l->key + key_size);
+	return *(void __percpu **)(l->key + roundup(key_size, 8));
 }
 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l)
 static void check_and_free_fields(struct bpf_htab *htab,
 				  struct htab_elem *elem)
 {
+	if (IS_ERR_OR_NULL(htab->map.record))
+		return;
 	if (htab_is_percpu(htab)) {
 		void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);
 		int cpu;
 	*insn++ = BPF_EMIT_CALL(__htab_map_lookup_elem);
 	*insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 3);
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_0,
-				offsetof(struct htab_elem, key) + map->key_size);
+				offsetof(struct htab_elem, key) + roundup(map->key_size, 8));
 	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_0, 0);
 	*insn++ = BPF_MOV64_PERCPU_REG(BPF_REG_0, BPF_REG_0);

kernel/bpf/helpers.c+119−4

@@ -1758,8 +1758,8 @@ static const struct bpf_func_proto bpf_dynptr_from_mem_proto = { · 1757 unmodified lines
1758	.arg4_type = ARG_PTR_TO_DYNPTR \| DYNPTR_TYPE_LOCAL \| MEM_UNINIT \| MEM_WRITE,	1758	.arg4_type = ARG_PTR_TO_DYNPTR \| DYNPTR_TYPE_LOCAL \| MEM_UNINIT \| MEM_WRITE,
1759	};	1759	};
1760		1760
1761	BPF_CALL_5(bpf_dynptr_read, void , dst, u32, len, const struct bpf_dynptr_kern , src,	1761	static int __bpf_dynptr_read(void dst, u32 len, const struct bpf_dynptr_kern src,
1762	u32, offset, u64, flags)	1762	u32 offset, u64 flags)
1763	{	1763	{
1764	enum bpf_dynptr_type type;	1764	enum bpf_dynptr_type type;
1765	int err;	1765	int err;
@@ -1792,6 +1792,12 @@ BPF_CALL_5(bpf_dynptr_read, void *, dst, u32, len, const struct bpf_dynptr_kern · 26 unmodified lines
1792	}	1792	}
1793	}	1793	}
1794		1794
		1795	BPF_CALL_5(bpf_dynptr_read, void , dst, u32, len, const struct bpf_dynptr_kern , src,
		1796	u32, offset, u64, flags)
		1797	{
		1798	return __bpf_dynptr_read(dst, len, src, offset, flags);
		1799	}
		1800
1795	static const struct bpf_func_proto bpf_dynptr_read_proto = {	1801	static const struct bpf_func_proto bpf_dynptr_read_proto = {
1796	.func = bpf_dynptr_read,	1802	.func = bpf_dynptr_read,
1797	.gpl_only = false,	1803	.gpl_only = false,
@@ -1803,8 +1809,8 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = { · 5 unmodified lines
1803	.arg5_type = ARG_ANYTHING,	1809	.arg5_type = ARG_ANYTHING,
1804	};	1810	};
1805		1811
1806	BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern , dst, u32, offset, void , src,	1812	static int __bpf_dynptr_write(const struct bpf_dynptr_kern dst, u32 offset, void src,
1807	u32, len, u64, flags)	1813	u32 len, u64 flags)
1808	{	1814	{
1809	enum bpf_dynptr_type type;	1815	enum bpf_dynptr_type type;
1810	int err;	1816	int err;
@@ -1842,6 +1848,12 @@ BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern *, dst, u32, offset, v · 31 unmodified lines
1842	}	1848	}
1843	}	1849	}
1844		1850
		1851	BPF_CALL_5(bpf_dynptr_write, const struct bpf_dynptr_kern , dst, u32, offset, void , src,
		1852	u32, len, u64, flags)
		1853	{
		1854	return __bpf_dynptr_write(dst, offset, src, len, flags);
		1855	}
		1856
1845	static const struct bpf_func_proto bpf_dynptr_write_proto = {	1857	static const struct bpf_func_proto bpf_dynptr_write_proto = {
1846	.func = bpf_dynptr_write,	1858	.func = bpf_dynptr_write,
1847	.gpl_only = false,	1859	.gpl_only = false,
@@ -2043,6 +2055,8 @@ bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 195 unmodified lines
2043	return &bpf_task_pt_regs_proto;	2055	return &bpf_task_pt_regs_proto;
2044	case BPF_FUNC_trace_vprintk:	2056	case BPF_FUNC_trace_vprintk:
2045	return bpf_get_trace_vprintk_proto();	2057	return bpf_get_trace_vprintk_proto();
		2058	case BPF_FUNC_perf_event_read_value:
		2059	return bpf_get_perf_event_read_value_proto();
2046	default:	2060	default:
2047	return NULL;	2061	return NULL;
2048	}	2062	}
@@ -2757,6 +2771,61 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p, · 708 unmodified lines
2757	return 0;	2771	return 0;
2758	}	2772	}
2759		2773
		2774	/**
		2775	* bpf_dynptr_copy() - Copy data from one dynptr to another.
		2776	* @dst_ptr: Destination dynptr - where data should be copied to
		2777	* @dst_off: Offset into the destination dynptr
		2778	* @src_ptr: Source dynptr - where data should be copied from
		2779	* @src_off: Offset into the source dynptr
		2780	* @size: Length of the data to copy from source to destination
		2781	*
		2782	* Copies data from source dynptr to destination dynptr.
		2783	* Returns 0 on success; negative error, otherwise.
		2784	*/
		2785	__bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u32 dst_off,
		2786	struct bpf_dynptr *src_ptr, u32 src_off, u32 size)
		2787	{
		2788	struct bpf_dynptr_kern dst = (struct bpf_dynptr_kern )dst_ptr;
		2789	struct bpf_dynptr_kern src = (struct bpf_dynptr_kern )src_ptr;
		2790	void src_slice, dst_slice;
		2791	char buf[256];
		2792	u32 off;
		2793
		2794	src_slice = bpf_dynptr_slice(src_ptr, src_off, NULL, size);
		2795	dst_slice = bpf_dynptr_slice_rdwr(dst_ptr, dst_off, NULL, size);
		2796
		2797	if (src_slice && dst_slice) {
		2798	memmove(dst_slice, src_slice, size);
		2799	return 0;
		2800	}
		2801
		2802	if (src_slice)
		2803	return __bpf_dynptr_write(dst, dst_off, src_slice, size, 0);
		2804
		2805	if (dst_slice)
		2806	return __bpf_dynptr_read(dst_slice, size, src, src_off, 0);
		2807
		2808	if (bpf_dynptr_check_off_len(dst, dst_off, size) \|\|
		2809	bpf_dynptr_check_off_len(src, src_off, size))
		2810	return -E2BIG;
		2811
		2812	off = 0;
		2813	while (off < size) {
		2814	u32 chunk_sz = min_t(u32, sizeof(buf), size - off);
		2815	int err;
		2816
		2817	err = __bpf_dynptr_read(buf, chunk_sz, src, src_off + off, 0);
		2818	if (err)
		2819	return err;
		2820	err = __bpf_dynptr_write(dst, dst_off + off, buf, chunk_sz, 0);
		2821	if (err)
		2822	return err;
		2823
		2824	off += chunk_sz;
		2825	}
		2826	return 0;
		2827	}
		2828
2760	__bpf_kfunc void bpf_cast_to_kern_ctx(void obj)	2829	__bpf_kfunc void bpf_cast_to_kern_ctx(void obj)
2761	{	2830	{
2762	return obj;	2831	return obj;
@@ -3066,6 +3135,50 @@ __bpf_kfunc int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void __user · 303 unmodified lines
3066	return ret + 1;	3135	return ret + 1;
3067	}	3136	}
3068		3137
		3138	/**
		3139	* bpf_copy_from_user_task_str() - Copy a string from an task's address space
		3140	* @dst: Destination address, in kernel space. This buffer must be
		3141	* at least @dst__sz bytes long.
		3142	* @dst__sz: Maximum number of bytes to copy, includes the trailing NUL.
		3143	* @unsafe_ptr__ign: Source address in the task's address space.
		3144	* @tsk: The task whose address space will be used
		3145	* @flags: The only supported flag is BPF_F_PAD_ZEROS
		3146	*
		3147	* Copies a NUL terminated string from a task's address space to @dst__sz
		3148	* buffer. If user string is too long this will still ensure zero termination
		3149	* in the @dst__sz buffer unless buffer size is 0.
		3150	*
		3151	* If BPF_F_PAD_ZEROS flag is set, memset the tail of @dst__sz to 0 on success
		3152	* and memset all of @dst__sz on failure.
		3153	*
		3154	* Return: The number of copied bytes on success including the NUL terminator.
		3155	* A negative error code on failure.
		3156	*/
		3157	__bpf_kfunc int bpf_copy_from_user_task_str(void *dst, u32 dst__sz,
		3158	const void __user *unsafe_ptr__ign,
		3159	struct task_struct *tsk, u64 flags)
		3160	{
		3161	int ret;
		3162
		3163	if (unlikely(flags & ~BPF_F_PAD_ZEROS))
		3164	return -EINVAL;
		3165
		3166	if (unlikely(dst__sz == 0))
		3167	return 0;
		3168
		3169	ret = copy_remote_vm_str(tsk, (unsigned long)unsafe_ptr__ign, dst, dst__sz, 0);
		3170	if (ret < 0) {
		3171	if (flags & BPF_F_PAD_ZEROS)
		3172	memset(dst, 0, dst__sz);
		3173	return ret;
		3174	}
		3175
		3176	if (flags & BPF_F_PAD_ZEROS)
		3177	memset(dst + ret, 0, dst__sz - ret);
		3178
		3179	return ret + 1;
		3180	}
		3181
3069	/* Keep unsinged long in prototype so that kfunc is usable when emitted to	3182	/* Keep unsinged long in prototype so that kfunc is usable when emitted to
3070	* vmlinux.h in BPF programs directly, but note that while in BPF prog, the	3183	* vmlinux.h in BPF programs directly, but note that while in BPF prog, the
3071	* unsigned long always points to 8-byte region on stack, the kernel may only	3184	* unsigned long always points to 8-byte region on stack, the kernel may only
@@ -3161,6 +3274,7 @@ BTF_ID_FLAGS(func, bpf_dynptr_is_null) · 89 unmodified lines
3161	BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly)	3274	BTF_ID_FLAGS(func, bpf_dynptr_is_rdonly)
3162	BTF_ID_FLAGS(func, bpf_dynptr_size)	3275	BTF_ID_FLAGS(func, bpf_dynptr_size)
3163	BTF_ID_FLAGS(func, bpf_dynptr_clone)	3276	BTF_ID_FLAGS(func, bpf_dynptr_clone)
		3277	BTF_ID_FLAGS(func, bpf_dynptr_copy)
3164	#ifdef CONFIG_NET	3278	#ifdef CONFIG_NET
3165	BTF_ID_FLAGS(func, bpf_modify_return_test_tp)	3279	BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
3166	#endif	3280	#endif
@@ -3173,6 +3287,7 @@ BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) · 6 unmodified lines
3173	BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT \| KF_RET_NULL)	3287	BTF_ID_FLAGS(func, bpf_iter_bits_next, KF_ITER_NEXT \| KF_RET_NULL)
3174	BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)	3288	BTF_ID_FLAGS(func, bpf_iter_bits_destroy, KF_ITER_DESTROY)
3175	BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)	3289	BTF_ID_FLAGS(func, bpf_copy_from_user_str, KF_SLEEPABLE)
		3290	BTF_ID_FLAGS(func, bpf_copy_from_user_task_str, KF_SLEEPABLE)
3176	BTF_ID_FLAGS(func, bpf_get_kmem_cache)	3291	BTF_ID_FLAGS(func, bpf_get_kmem_cache)
3177	BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW \| KF_SLEEPABLE)	3292	BTF_ID_FLAGS(func, bpf_iter_kmem_cache_new, KF_ITER_NEW \| KF_SLEEPABLE)
3178	BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT \| KF_RET_NULL \| KF_SLEEPABLE)	3293	BTF_ID_FLAGS(func, bpf_iter_kmem_cache_next, KF_ITER_NEXT \| KF_RET_NULL \| KF_SLEEPABLE)

kernel/bpf/preload/bpf_preload_kern.c+1−0

@@ -90,3 +90,4 @@ static void __exit fini(void) · 89 unmodified lines
90	late_initcall(load);	90	late_initcall(load);
91	module_exit(fini);	91	module_exit(fini);
92	MODULE_LICENSE("GPL");	92	MODULE_LICENSE("GPL");
		93	MODULE_DESCRIPTION("Embedded BPF programs for introspection in bpffs");

kernel/bpf/syscall.c+33−15

 #define BPF_MAP_CREATE_LAST_FIELD map_token_fd
 /* called via syscall */
-static int map_create(union bpf_attr *attr)
+static int map_create(union bpf_attr *attr, bool kernel)
 {
 	const struct bpf_map_ops *ops;
 	struct bpf_token *token = NULL;
 			attr->btf_vmlinux_value_type_id;
 	}
-	err = security_bpf_map_create(map, attr, token);
+	err = security_bpf_map_create(map, attr, token, kernel);
 	if (err)
 		goto free_map_sec;
 struct bpf_map *bpf_map_inc_not_zero(struct bpf_map *map)
 {
-	spin_lock_bh(&map_idr_lock);
+	lockdep_assert(rcu_read_lock_held());
-	map = __bpf_map_inc_not_zero(map, false);
+	return __bpf_map_inc_not_zero(map, false);
-	spin_unlock_bh(&map_idr_lock);
-	return map;
 }
 EXPORT_SYMBOL_GPL(bpf_map_inc_not_zero);
 	kvfree(prog->aux->jited_linfo);
 	kvfree(prog->aux->linfo);
 	kfree(prog->aux->kfunc_tab);
+	kfree(prog->aux->ctx_arg_info);
 	if (prog->aux->attach_btf)
 		btf_put(prog->aux->attach_btf);
 	if (err < 0)
 		goto free_prog;
-	err = security_bpf_prog_load(prog, attr, token);
+	err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel);
 	if (err)
 		goto free_prog_sec;
 #define BPF_F_ATTACH_MASK_BASE	\
 	(BPF_F_ALLOW_OVERRIDE |	\
 	 BPF_F_ALLOW_MULTI |	\
-	 BPF_F_REPLACE)
+	 BPF_F_REPLACE |	\
+	 BPF_F_PREORDER)
 #define BPF_F_ATTACH_MASK_MPROG	\
 	(BPF_F_REPLACE |	\
 	info.recursion_misses = stats.misses;
 	info.verified_insns = prog->aux->verified_insns;
+	if (prog->aux->btf)
+		info.btf_id = btf_obj_id(prog->aux->btf);
 	if (!bpf_capable()) {
 		info.jited_prog_len = 0;
 		}
 	}
-	if (prog->aux->btf)
-		info.btf_id = btf_obj_id(prog->aux->btf);
 	info.attach_btf_id = prog->aux->attach_btf_id;
 	if (attach_btf)
 		info.attach_btf_obj_id = btf_obj_id(attach_btf);
 	return btf_new_fd(attr, uattr, uattr_size);
 }
-#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD btf_id
+#define BPF_BTF_GET_FD_BY_ID_LAST_FIELD fd_by_id_token_fd
 static int bpf_btf_get_fd_by_id(const union bpf_attr *attr)
 {
+	struct bpf_token *token = NULL;
 	if (CHECK_ATTR(BPF_BTF_GET_FD_BY_ID))
 		return -EINVAL;
-	if (!capable(CAP_SYS_ADMIN))
+	if (attr->open_flags & ~BPF_F_TOKEN_FD)
+		return -EINVAL;
+	if (attr->open_flags & BPF_F_TOKEN_FD) {
+		token = bpf_token_get_from_fd(attr->fd_by_id_token_fd);
+		if (IS_ERR(token))
+			return PTR_ERR(token);
+		if (!bpf_token_allow_cmd(token, BPF_BTF_GET_FD_BY_ID)) {
+			bpf_token_put(token);
+			token = NULL;
+		}
+	}
+	if (!bpf_token_capable(token, CAP_SYS_ADMIN)) {
+		bpf_token_put(token);
 		return -EPERM;
+	}
+	bpf_token_put(token);
 	return btf_get_fd_by_id(attr->btf_id);
 }
 	if (copy_from_bpfptr(&attr, uattr, size) != 0)
 		return -EFAULT;
-	err = security_bpf(cmd, &attr, size);
+	err = security_bpf(cmd, &attr, size, uattr.is_kernel);
 	if (err < 0)
 		return err;
 	switch (cmd) {
 	case BPF_MAP_CREATE:
-		err = map_create(&attr);
+		err = map_create(&attr, uattr.is_kernel);
 		break;
 	case BPF_MAP_LOOKUP_ELEM:
 		err = map_lookup_elem(&attr);

kernel/bpf/verifier.c+1043−333

@@ -579,6 +579,13 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) · 578 unmodified lines
579	insn->imm == BPF_CMPXCHG;	579	insn->imm == BPF_CMPXCHG;
580	}	580	}
581		581
		582	static bool is_atomic_load_insn(const struct bpf_insn *insn)
		583	{
		584	return BPF_CLASS(insn->code) == BPF_STX &&
		585	BPF_MODE(insn->code) == BPF_ATOMIC &&
		586	insn->imm == BPF_LOAD_ACQ;
		587	}
		588
582	static int __get_spi(s32 off)	589	static int __get_spi(s32 off)
583	{	590	{
584	return (-off - 1) / BPF_REG_SIZE;	591	return (-off - 1) / BPF_REG_SIZE;
@@ -1545,6 +1552,17 @@ static void release_reference_state(struct bpf_verifier_state *state, int idx) · 960 unmodified lines
1545	return;	1552	return;
1546	}	1553	}
1547		1554
		1555	static bool find_reference_state(struct bpf_verifier_state *state, int ptr_id)
		1556	{
		1557	int i;
		1558
		1559	for (i = 0; i < state->acquired_refs; i++)
		1560	if (state->refs[i].id == ptr_id)
		1561	return true;
		1562
		1563	return false;
		1564	}
		1565
1548	static int release_lock_state(struct bpf_verifier_state state, int type, int id, void ptr)	1566	static int release_lock_state(struct bpf_verifier_state state, int type, int id, void ptr)
1549	{	1567	{
1550	int i;	1568	int i;
@@ -1600,6 +1618,14 @@ static struct bpf_reference_state find_lock_state(struct bpf_verifier_state st · 49 unmodified lines
1600	return NULL;	1618	return NULL;
1601	}	1619	}
1602		1620
		1621	static void update_peak_states(struct bpf_verifier_env *env)
		1622	{
		1623	u32 cur_states;
		1624
		1625	cur_states = env->explored_states_size + env->free_list_size;
		1626	env->peak_states = max(env->peak_states, cur_states);
		1627	}
		1628
1603	static void free_func_state(struct bpf_func_state *state)	1629	static void free_func_state(struct bpf_func_state *state)
1604	{	1630	{
1605	if (!state)	1631	if (!state)
@@ -1622,6 +1648,50 @@ static void free_verifier_state(struct bpf_verifier_state *state, · 16 unmodified lines
1622	kfree(state);	1648	kfree(state);
1623	}	1649	}
1624		1650
		1651	/* struct bpf_verifier_state->{parent,loop_entry} refer to states
		1652	* that are in either of env->{expored_states,free_list}.
		1653	* In both cases the state is contained in struct bpf_verifier_state_list.
		1654	*/
		1655	static struct bpf_verifier_state_list state_parent_as_list(struct bpf_verifier_state st)
		1656	{
		1657	if (st->parent)
		1658	return container_of(st->parent, struct bpf_verifier_state_list, state);
		1659	return NULL;
		1660	}
		1661
		1662	static struct bpf_verifier_state_list state_loop_entry_as_list(struct bpf_verifier_state st)
		1663	{
		1664	if (st->loop_entry)
		1665	return container_of(st->loop_entry, struct bpf_verifier_state_list, state);
		1666	return NULL;
		1667	}
		1668
		1669	/* A state can be freed if it is no longer referenced:
		1670	* - is in the env->free_list;
		1671	* - has no children states;
		1672	* - is not used as loop_entry.
		1673	*
		1674	* Freeing a state can make it's loop_entry free-able.
		1675	*/
		1676	static void maybe_free_verifier_state(struct bpf_verifier_env *env,
		1677	struct bpf_verifier_state_list *sl)
		1678	{
		1679	struct bpf_verifier_state_list *loop_entry_sl;
		1680
		1681	while (sl && sl->in_free_list &&
		1682	sl->state.branches == 0 &&
		1683	sl->state.used_as_loop_entry == 0) {
		1684	loop_entry_sl = state_loop_entry_as_list(&sl->state);
		1685	if (loop_entry_sl)
		1686	loop_entry_sl->state.used_as_loop_entry--;
		1687	list_del(&sl->node);
		1688	free_verifier_state(&sl->state, false);
		1689	kfree(sl);
		1690	env->free_list_size--;
		1691	sl = loop_entry_sl;
		1692	}
		1693	}
		1694
1625	/* copy verifier state from src to dst growing dst stack space	1695	/* copy verifier state from src to dst growing dst stack space
1626	* when necessary to accommodate larger src stack	1696	* when necessary to accommodate larger src stack
1627	*/	1697	*/
@@ -1661,6 +1731,7 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 33 unmodified lines
1661	dst_state->callback_unroll_depth = src->callback_unroll_depth;	1731	dst_state->callback_unroll_depth = src->callback_unroll_depth;
1662	dst_state->used_as_loop_entry = src->used_as_loop_entry;	1732	dst_state->used_as_loop_entry = src->used_as_loop_entry;
1663	dst_state->may_goto_depth = src->may_goto_depth;	1733	dst_state->may_goto_depth = src->may_goto_depth;
		1734	dst_state->loop_entry = src->loop_entry;
1664	for (i = 0; i <= src->curframe; i++) {	1735	for (i = 0; i <= src->curframe; i++) {
1665	dst = dst_state->frame[i];	1736	dst = dst_state->frame[i];
1666	if (!dst) {	1737	if (!dst) {
@@ -1681,7 +1752,7 @@ static u32 state_htab_size(struct bpf_verifier_env *env) · 14 unmodified lines
1681	return env->prog->len;	1752	return env->prog->len;
1682	}	1753	}
1683		1754
1684	static struct bpf_verifier_state_list *explored_state(struct bpf_verifier_env env, int idx)	1755	static struct list_head explored_state(struct bpf_verifier_env env, int idx)
1685	{	1756	{
1686	struct bpf_verifier_state *cur = env->cur_state;	1757	struct bpf_verifier_state *cur = env->cur_state;
1687	struct bpf_func_state *state = cur->frame[cur->curframe];	1758	struct bpf_func_state *state = cur->frame[cur->curframe];
@@ -1789,16 +1860,13 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta · 101 unmodified lines
1789	* # Find outermost loop entry known for n	1860	* # Find outermost loop entry known for n
1790	* def get_loop_entry(n):	1861	* def get_loop_entry(n):
1791	* h = entries.get(n, None)	1862	* h = entries.get(n, None)
1792	* while h in entries and entries[h] != h:	1863	* while h in entries:
1793	* h = entries[h]	1864	* h = entries[h]
1794	* return h	1865	* return h
1795	*	1866	*
1796	* # Update n's loop entry if h's outermost entry comes	1867	* # Update n's loop entry if h comes before n in current DFS path.
1797	* # before n's outermost entry in current DFS path.
1798	* def update_loop_entry(n, h):	1868	* def update_loop_entry(n, h):
1799	* n1 = get_loop_entry(n) or n	1869	* if h in path and depths[entries.get(n, n)] < depths[n]:
1800	* h1 = get_loop_entry(h) or h
1801	* if h1 in path and depths[h1] <= depths[n1]:
1802	* entries[n] = h1	1870	* entries[n] = h1
1803	*	1871	*
1804	* def dfs(n, depth):	1872	* def dfs(n, depth):
@@ -1810,7 +1878,7 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta · 5 unmodified lines
1810	* # Case A: explore succ and update cur's loop entry	1878	* # Case A: explore succ and update cur's loop entry
1811	* # only if succ's entry is in current DFS path.	1879	* # only if succ's entry is in current DFS path.
1812	* dfs(succ, depth + 1)	1880	* dfs(succ, depth + 1)
1813	* h = get_loop_entry(succ)	1881	* h = entries.get(succ, None)
1814	* update_loop_entry(n, h)	1882	* update_loop_entry(n, h)
1815	* else:	1883	* else:
1816	* # Case B or C depending on `h1 in path` check in update_loop_entry().	1884	* # Case B or C depending on `h1 in path` check in update_loop_entry().
@@ -1822,46 +1890,49 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta · 5 unmodified lines
1822	* and cur's loop entry has to be updated (case A), handle this in	1890	* and cur's loop entry has to be updated (case A), handle this in
1823	* update_branch_counts();	1891	* update_branch_counts();
1824	* - use st->branch > 0 as a signal that st is in the current DFS path;	1892	* - use st->branch > 0 as a signal that st is in the current DFS path;
1825	* - handle cases B and C in is_state_visited();	1893	* - handle cases B and C in is_state_visited().
1826	* - update topmost loop entry for intermediate states in get_loop_entry().
1827	*/	1894	*/
1828	static struct bpf_verifier_state get_loop_entry(struct bpf_verifier_state st)	1895	static struct bpf_verifier_state get_loop_entry(struct bpf_verifier_env env,
		1896	struct bpf_verifier_state *st)
1829	{	1897	{
1830	struct bpf_verifier_state topmost = st->loop_entry, old;	1898	struct bpf_verifier_state *topmost = st->loop_entry;
		1899	u32 steps = 0;
1831		1900
1832	while (topmost && topmost->loop_entry && topmost != topmost->loop_entry)	1901	while (topmost && topmost->loop_entry) {
		1902	if (steps++ > st->dfs_depth) {
		1903	WARN_ONCE(true, "verifier bug: infinite loop in get_loop_entry\n");
		1904	verbose(env, "verifier bug: infinite loop in get_loop_entry()\n");
		1905	return ERR_PTR(-EFAULT);
		1906	}
1833	topmost = topmost->loop_entry;	1907	topmost = topmost->loop_entry;
1834	/* Update loop entries for intermediate states to avoid this
1835	* traversal in future get_loop_entry() calls.
1836	*/
1837	while (st && st->loop_entry != topmost) {
1838	old = st->loop_entry;
1839	st->loop_entry = topmost;
1840	st = old;
1841	}	1908	}
1842	return topmost;	1909	return topmost;
1843	}	1910	}
1844		1911
1845	static void update_loop_entry(struct bpf_verifier_state cur, struct bpf_verifier_state hdr)	1912	static void update_loop_entry(struct bpf_verifier_env *env,
		1913	struct bpf_verifier_state cur, struct bpf_verifier_state hdr)
1846	{	1914	{
1847	struct bpf_verifier_state cur1, hdr1;	1915	/* The hdr->branches check decides between cases B and C in
1848		1916	* comment for get_loop_entry(). If hdr->branches == 0 then
1849	cur1 = get_loop_entry(cur) ?: cur;
1850	hdr1 = get_loop_entry(hdr) ?: hdr;
1851	/* The head1->branches check decides between cases B and C in
1852	* comment for get_loop_entry(). If hdr1->branches == 0 then
1853	* head's topmost loop entry is not in current DFS path,	1917	* head's topmost loop entry is not in current DFS path,
1854	* hence 'cur' and 'hdr' are not in the same loop and there is	1918	* hence 'cur' and 'hdr' are not in the same loop and there is
1855	* no need to update cur->loop_entry.	1919	* no need to update cur->loop_entry.
1856	*/	1920	*/
1857	if (hdr1->branches && hdr1->dfs_depth <= cur1->dfs_depth) {	1921	if (hdr->branches && hdr->dfs_depth < (cur->loop_entry ?: cur)->dfs_depth) {
		1922	if (cur->loop_entry) {
		1923	cur->loop_entry->used_as_loop_entry--;
		1924	maybe_free_verifier_state(env, state_loop_entry_as_list(cur));
		1925	}
1858	cur->loop_entry = hdr;	1926	cur->loop_entry = hdr;
1859	hdr->used_as_loop_entry = true;	1927	hdr->used_as_loop_entry++;
1860	}	1928	}
1861	}	1929	}
1862		1930
1863	static void update_branch_counts(struct bpf_verifier_env env, struct bpf_verifier_state st)	1931	static void update_branch_counts(struct bpf_verifier_env env, struct bpf_verifier_state st)
1864	{	1932	{
		1933	struct bpf_verifier_state_list sl = NULL, parent_sl;
		1934	struct bpf_verifier_state *parent;
		1935
1865	while (st) {	1936	while (st) {
1866	u32 br = --st->branches;	1937	u32 br = --st->branches;
1867		1938
@@ -1871,7 +1942,7 @@ static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifi · 3 unmodified lines
1871	* This is a part of 'case A' in get_loop_entry() comment.	1942	* This is a part of 'case A' in get_loop_entry() comment.
1872	*/	1943	*/
1873	if (br == 0 && st->parent && st->loop_entry)	1944	if (br == 0 && st->parent && st->loop_entry)
1874	update_loop_entry(st->parent, st->loop_entry);	1945	update_loop_entry(env, st->parent, st->loop_entry);
1875		1946
1876	/* WARN_ON(br > 1) technically makes sense here,	1947	/* WARN_ON(br > 1) technically makes sense here,
1877	* but see comment in push_stack(), hence:	1948	* but see comment in push_stack(), hence:
@@ -1881,7 +1952,12 @@ static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifi · 3 unmodified lines
1881	br);	1952	br);
1882	if (br)	1953	if (br)
1883	break;	1954	break;
1884	st = st->parent;	1955	parent = st->parent;
		1956	parent_sl = state_parent_as_list(st);
		1957	if (sl)
		1958	maybe_free_verifier_state(env, sl);
		1959	st = parent;
		1960	sl = parent_sl;
1885	}	1961	}
1886	}	1962	}
1887		1963
@@ -3206,6 +3282,21 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, · 1318 unmodified lines
3206	return res ? &res->func_model : NULL;	3282	return res ? &res->func_model : NULL;
3207	}	3283	}
3208		3284
		3285	static int add_kfunc_in_insns(struct bpf_verifier_env *env,
		3286	struct bpf_insn *insn, int cnt)
		3287	{
		3288	int i, ret;
		3289
		3290	for (i = 0; i < cnt; i++, insn++) {
		3291	if (bpf_pseudo_kfunc_call(insn)) {
		3292	ret = add_kfunc_call(env, insn->imm, insn->off);
		3293	if (ret < 0)
		3294	return ret;
		3295	}
		3296	}
		3297	return 0;
		3298	}
		3299
3209	static int add_subprog_and_kfunc(struct bpf_verifier_env *env)	3300	static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
3210	{	3301	{
3211	struct bpf_subprog_info *subprog = env->subprog_info;	3302	struct bpf_subprog_info *subprog = env->subprog_info;
@@ -3269,6 +3360,15 @@ static int add_subprog_and_kfunc(struct bpf_verifier_env *env) · 57 unmodified lines
3269	return 0;	3360	return 0;
3270	}	3361	}
3271		3362
		3363	static int jmp_offset(struct bpf_insn *insn)
		3364	{
		3365	u8 code = insn->code;
		3366
		3367	if (code == (BPF_JMP32 \| BPF_JA))
		3368	return insn->imm;
		3369	return insn->off;
		3370	}
		3371
3272	static int check_subprogs(struct bpf_verifier_env *env)	3372	static int check_subprogs(struct bpf_verifier_env *env)
3273	{	3373	{
3274	int i, subprog_start, subprog_end, off, cur_subprog = 0;	3374	int i, subprog_start, subprog_end, off, cur_subprog = 0;
@@ -3295,10 +3395,7 @@ static int check_subprogs(struct bpf_verifier_env *env) · 20 unmodified lines
3295	goto next;	3395	goto next;
3296	if (BPF_OP(code) == BPF_EXIT \|\| BPF_OP(code) == BPF_CALL)	3396	if (BPF_OP(code) == BPF_EXIT \|\| BPF_OP(code) == BPF_CALL)
3297	goto next;	3397	goto next;
3298	if (code == (BPF_JMP32 \| BPF_JA))	3398	off = i + jmp_offset(&insn[i]) + 1;
3299	off = i + insn[i].imm + 1;
3300	else
3301	off = i + insn[i].off + 1;
3302	if (off < subprog_start \|\| off >= subprog_end) {	3399	if (off < subprog_start \|\| off >= subprog_end) {
3303	verbose(env, "jump out of range from insn %d to %d\n", i, off);	3400	verbose(env, "jump out of range from insn %d to %d\n", i, off);
3304	return -EINVAL;	3401	return -EINVAL;
@@ -3483,7 +3580,7 @@ static bool is_reg64(struct bpf_verifier_env env, struct bpf_insn insn, · 178 unmodified lines
3483	}	3580	}
3484		3581
3485	if (class == BPF_STX) {	3582	if (class == BPF_STX) {
3486	/* BPF_STX (including atomic variants) has multiple source	3583	/* BPF_STX (including atomic variants) has one or more source
3487	* operands, one of which is a ptr. Check whether the caller is	3584	* operands, one of which is a ptr. Check whether the caller is
3488	* asking about it.	3585	* asking about it.
3489	*/	3586	*/
@@ -3828,6 +3925,17 @@ static const char disasm_kfunc_name(void data, const struct bpf_insn *insn) · 338 unmodified lines
3828	return btf_name_by_offset(desc_btf, func->name_off);	3925	return btf_name_by_offset(desc_btf, func->name_off);
3829	}	3926	}
3830		3927
		3928	static void verbose_insn(struct bpf_verifier_env env, struct bpf_insn insn)
		3929	{
		3930	const struct bpf_insn_cbs cbs = {
		3931	.cb_call = disasm_kfunc_name,
		3932	.cb_print = verbose,
		3933	.private_data = env,
		3934	};
		3935
		3936	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
		3937	}
		3938
3831	static inline void bt_init(struct backtrack_state *bt, u32 frame)	3939	static inline void bt_init(struct backtrack_state *bt, u32 frame)
3832	{	3940	{
3833	bt->frame = frame;	3941	bt->frame = frame;
@@ -4028,11 +4136,6 @@ static bool calls_callback(struct bpf_verifier_env *env, int insn_idx); · 194 unmodified lines
4028	static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,	4136	static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
4029	struct bpf_insn_hist_entry hist, struct backtrack_state bt)	4137	struct bpf_insn_hist_entry hist, struct backtrack_state bt)
4030	{	4138	{
4031	const struct bpf_insn_cbs cbs = {
4032	.cb_call = disasm_kfunc_name,
4033	.cb_print = verbose,
4034	.private_data = env,
4035	};
4036	struct bpf_insn *insn = env->prog->insnsi + idx;	4139	struct bpf_insn *insn = env->prog->insnsi + idx;
4037	u8 class = BPF_CLASS(insn->code);	4140	u8 class = BPF_CLASS(insn->code);
4038	u8 opcode = BPF_OP(insn->code);	4141	u8 opcode = BPF_OP(insn->code);
@@ -4050,7 +4153,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 11 unmodified lines
4050	fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));	4153	fmt_stack_mask(env->tmp_str_buf, TMP_STR_BUF_LEN, bt_stack_mask(bt));
4051	verbose(env, "stack=%s before ", env->tmp_str_buf);	4154	verbose(env, "stack=%s before ", env->tmp_str_buf);
4052	verbose(env, "%d: ", idx);	4155	verbose(env, "%d: ", idx);
4053	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);	4156	verbose_insn(env, insn);
4054	}	4157	}
4055		4158
4056	/* If there is a history record that some registers gained range at this insn,	4159	/* If there is a history record that some registers gained range at this insn,
@@ -4097,7 +4200,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 40 unmodified lines
4097	* dreg still needs precision before this insn	4200	* dreg still needs precision before this insn
4098	*/	4201	*/
4099	}	4202	}
4100	} else if (class == BPF_LDX) {	4203	} else if (class == BPF_LDX \|\| is_atomic_load_insn(insn)) {
4101	if (!bt_is_reg_set(bt, dreg))	4204	if (!bt_is_reg_set(bt, dreg))
4102	return 0;	4205	return 0;
4103	bt_clear_reg(bt, dreg);	4206	bt_clear_reg(bt, dreg);
@@ -5982,18 +6085,10 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, · 1878 unmodified lines
5982		6085
5983	/* check access to 'struct bpf_context' fields. Supports fixed offsets only */	6086	/* check access to 'struct bpf_context' fields. Supports fixed offsets only */
5984	static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,	6087	static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
5985	enum bpf_access_type t, enum bpf_reg_type *reg_type,	6088	enum bpf_access_type t, struct bpf_insn_access_aux *info)
5986	struct btf *btf, u32 btf_id, bool *is_retval, bool is_ldsx)
5987	{	6089	{
5988	struct bpf_insn_access_aux info = {
5989	.reg_type = *reg_type,
5990	.log = &env->log,
5991	.is_retval = false,
5992	.is_ldsx = is_ldsx,
5993	};
5994
5995	if (env->ops->is_valid_access &&	6090	if (env->ops->is_valid_access &&
5996	env->ops->is_valid_access(off, size, t, env->prog, &info)) {	6091	env->ops->is_valid_access(off, size, t, env->prog, info)) {
5997	/* A non zero info.ctx_field_size indicates that this field is a	6092	/* A non zero info.ctx_field_size indicates that this field is a
5998	* candidate for later verifier transformation to load the whole	6093	* candidate for later verifier transformation to load the whole
5999	* field and then apply a mask when accessed with a narrower	6094	* field and then apply a mask when accessed with a narrower
@@ -6001,14 +6096,15 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, · 1 unmodified line
6001	* will only allow for whole field access and rejects any other	6096	* will only allow for whole field access and rejects any other
6002	* type of narrower access.	6097	* type of narrower access.
6003	*/	6098	*/
6004	*reg_type = info.reg_type;	6099	if (base_type(info->reg_type) == PTR_TO_BTF_ID) {
6005	*is_retval = info.is_retval;	6100	if (info->ref_obj_id &&
6006		6101	!find_reference_state(env->cur_state, info->ref_obj_id)) {
6007	if (base_type(*reg_type) == PTR_TO_BTF_ID) {	6102	verbose(env, "invalid bpf_context access off=%d. Reference may already be released\n",
6008	*btf = info.btf;	6103	off);
6009	*btf_id = info.btf_id;	6104	return -EACCES;
		6105	}
6010	} else {	6106	} else {
6011	env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;	6107	env->insn_aux_data[insn_idx].ctx_field_size = info->ctx_field_size;
6012	}	6108	}
6013	/* remember the offset of last byte accessed in ctx */	6109	/* remember the offset of last byte accessed in ctx */
6014	if (env->prog->aux->max_ctx_offset < off + size)	6110	if (env->prog->aux->max_ctx_offset < off + size)
@@ -6118,6 +6214,26 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) · 103 unmodified lines
6118	return reg->type == PTR_TO_ARENA;	6214	return reg->type == PTR_TO_ARENA;
6119	}	6215	}
6120		6216
		6217	/* Return false if @regno contains a pointer whose type isn't supported for
		6218	* atomic instruction @insn.
		6219	*/
		6220	static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno,
		6221	struct bpf_insn *insn)
		6222	{
		6223	if (is_ctx_reg(env, regno))
		6224	return false;
		6225	if (is_pkt_reg(env, regno))
		6226	return false;
		6227	if (is_flow_key_reg(env, regno))
		6228	return false;
		6229	if (is_sk_reg(env, regno))
		6230	return false;
		6231	if (is_arena_reg(env, regno))
		6232	return bpf_jit_supports_insn(insn, true);
		6233
		6234	return true;
		6235	}
		6236
6121	static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {	6237	static u32 *reg2btf_ids[__BPF_REG_TYPE_MAX] = {
6122	#ifdef CONFIG_NET	6238	#ifdef CONFIG_NET
6123	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],	6239	[PTR_TO_SOCKET] = &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
@@ -7365,11 +7481,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1241 unmodified lines
7365	if (!err && value_regno >= 0 && (t == BPF_READ \|\| rdonly_mem))	7481	if (!err && value_regno >= 0 && (t == BPF_READ \|\| rdonly_mem))
7366	mark_reg_unknown(env, regs, value_regno);	7482	mark_reg_unknown(env, regs, value_regno);
7367	} else if (reg->type == PTR_TO_CTX) {	7483	} else if (reg->type == PTR_TO_CTX) {
7368	bool is_retval = false;
7369	struct bpf_retval_range range;	7484	struct bpf_retval_range range;
7370	enum bpf_reg_type reg_type = SCALAR_VALUE;	7485	struct bpf_insn_access_aux info = {
7371	struct btf *btf = NULL;	7486	.reg_type = SCALAR_VALUE,
7372	u32 btf_id = 0;	7487	.is_ldsx = is_ldsx,
		7488	.log = &env->log,
		7489	};
7373		7490
7374	if (t == BPF_WRITE && value_regno >= 0 &&	7491	if (t == BPF_WRITE && value_regno >= 0 &&
7375	is_pointer_value(env, value_regno)) {	7492	is_pointer_value(env, value_regno)) {
@@ -7381,8 +7498,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 5 unmodified lines
7381	if (err < 0)	7498	if (err < 0)
7382	return err;	7499	return err;
7383		7500
7384	err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf,	7501	err = check_ctx_access(env, insn_idx, off, size, t, &info);
7385	&btf_id, &is_retval, is_ldsx);
7386	if (err)	7502	if (err)
7387	verbose_linfo(env, insn_idx, "; ");	7503	verbose_linfo(env, insn_idx, "; ");
7388	if (!err && t == BPF_READ && value_regno >= 0) {	7504	if (!err && t == BPF_READ && value_regno >= 0) {
@@ -7390,8 +7506,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1 unmodified line
7390	* PTR_TO_PACKET[_META,_END]. In the latter	7506	* PTR_TO_PACKET[_META,_END]. In the latter
7391	* case, we know the offset is zero.	7507	* case, we know the offset is zero.
7392	*/	7508	*/
7393	if (reg_type == SCALAR_VALUE) {	7509	if (info.reg_type == SCALAR_VALUE) {
7394	if (is_retval && get_func_retval_range(env->prog, &range)) {	7510	if (info.is_retval && get_func_retval_range(env->prog, &range)) {
7395	err = __mark_reg_s32_range(env, regs, value_regno,	7511	err = __mark_reg_s32_range(env, regs, value_regno,
7396	range.minval, range.maxval);	7512	range.minval, range.maxval);
7397	if (err)	7513	if (err)
@@ -7402,7 +7518,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 4 unmodified lines
7402	} else {	7518	} else {
7403	mark_reg_known_zero(env, regs,	7519	mark_reg_known_zero(env, regs,
7404	value_regno);	7520	value_regno);
7405	if (type_may_be_null(reg_type))	7521	if (type_may_be_null(info.reg_type))
7406	regs[value_regno].id = ++env->id_gen;	7522	regs[value_regno].id = ++env->id_gen;
7407	/* A load of ctx field could have different	7523	/* A load of ctx field could have different
7408	* actual load size with the one encoded in the	7524	* actual load size with the one encoded in the
@@ -7410,12 +7526,13 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 1 unmodified line
7410	* a sub-register.	7526	* a sub-register.
7411	*/	7527	*/
7412	regs[value_regno].subreg_def = DEF_NOT_SUBREG;	7528	regs[value_regno].subreg_def = DEF_NOT_SUBREG;
7413	if (base_type(reg_type) == PTR_TO_BTF_ID) {	7529	if (base_type(info.reg_type) == PTR_TO_BTF_ID) {
7414	regs[value_regno].btf = btf;	7530	regs[value_regno].btf = info.btf;
7415	regs[value_regno].btf_id = btf_id;	7531	regs[value_regno].btf_id = info.btf_id;
		7532	regs[value_regno].ref_obj_id = info.ref_obj_id;
7416	}	7533	}
7417	}	7534	}
7418	regs[value_regno].type = reg_type;	7535	regs[value_regno].type = info.reg_type;
7419	}	7536	}
7420		7537
7421	} else if (reg->type == PTR_TO_STACK) {	7538	} else if (reg->type == PTR_TO_STACK) {
@@ -7518,27 +7635,72 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 96 unmodified lines
7518	static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,	7635	static int save_aux_ptr_type(struct bpf_verifier_env *env, enum bpf_reg_type type,
7519	bool allow_trust_mismatch);	7636	bool allow_trust_mismatch);
7520		7637
7521	static int check_atomic(struct bpf_verifier_env env, int insn_idx, struct bpf_insn insn)	7638	static int check_load_mem(struct bpf_verifier_env env, struct bpf_insn insn,
		7639	bool strict_alignment_once, bool is_ldsx,
		7640	bool allow_trust_mismatch, const char *ctx)
7522	{	7641	{
7523	int load_reg;	7642	struct bpf_reg_state *regs = cur_regs(env);
		7643	enum bpf_reg_type src_reg_type;
7524	int err;	7644	int err;
7525		7645
7526	switch (insn->imm) {	7646	/* check src operand */
7527	case BPF_ADD:	7647	err = check_reg_arg(env, insn->src_reg, SRC_OP);
7528	case BPF_ADD \| BPF_FETCH:	7648	if (err)
7529	case BPF_AND:	7649	return err;
7530	case BPF_AND \| BPF_FETCH:	7650
7531	case BPF_OR:	7651	/* check dst operand */
7532	case BPF_OR \| BPF_FETCH:	7652	err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7533	case BPF_XOR:	7653	if (err)
7534	case BPF_XOR \| BPF_FETCH:	7654	return err;
7535	case BPF_XCHG:	7655
7536	case BPF_CMPXCHG:	7656	src_reg_type = regs[insn->src_reg].type;
7537	break;	7657
7538	default:	7658	/* Check if (src_reg + off) is readable. The state of dst_reg will be
7539	verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);	7659	* updated by this call.
7540	return -EINVAL;	7660	*/
7541	}	7661	err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off,
		7662	BPF_SIZE(insn->code), BPF_READ, insn->dst_reg,
		7663	strict_alignment_once, is_ldsx);
		7664	err = err ?: save_aux_ptr_type(env, src_reg_type,
		7665	allow_trust_mismatch);
		7666	err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], ctx);
		7667
		7668	return err;
		7669	}
		7670
		7671	static int check_store_reg(struct bpf_verifier_env env, struct bpf_insn insn,
		7672	bool strict_alignment_once)
		7673	{
		7674	struct bpf_reg_state *regs = cur_regs(env);
		7675	enum bpf_reg_type dst_reg_type;
		7676	int err;
		7677
		7678	/* check src1 operand */
		7679	err = check_reg_arg(env, insn->src_reg, SRC_OP);
		7680	if (err)
		7681	return err;
		7682
		7683	/* check src2 operand */
		7684	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
		7685	if (err)
		7686	return err;
		7687
		7688	dst_reg_type = regs[insn->dst_reg].type;
		7689
		7690	/* Check if (dst_reg + off) is writeable. */
		7691	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
		7692	BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg,
		7693	strict_alignment_once, false);
		7694	err = err ?: save_aux_ptr_type(env, dst_reg_type, false);
		7695
		7696	return err;
		7697	}
		7698
		7699	static int check_atomic_rmw(struct bpf_verifier_env *env,
		7700	struct bpf_insn *insn)
		7701	{
		7702	int load_reg;
		7703	int err;
7542		7704
7543	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {	7705	if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
7544	verbose(env, "invalid atomic operand size\n");	7706	verbose(env, "invalid atomic operand size\n");
@@ -7574,11 +7736,7 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 29 unmodified lines
7574	return -EACCES;	7736	return -EACCES;
7575	}	7737	}
7576		7738
7577	if (is_ctx_reg(env, insn->dst_reg) \|\|	7739	if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
7578	is_pkt_reg(env, insn->dst_reg) \|\|
7579	is_flow_key_reg(env, insn->dst_reg) \|\|
7580	is_sk_reg(env, insn->dst_reg) \|\|
7581	(is_arena_reg(env, insn->dst_reg) && !bpf_jit_supports_insn(insn, true))) {
7582	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",	7740	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
7583	insn->dst_reg,	7741	insn->dst_reg,
7584	reg_type_str(env, reg_state(env, insn->dst_reg)->type));	7742	reg_type_str(env, reg_state(env, insn->dst_reg)->type));
@@ -7605,12 +7763,12 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 20 unmodified lines
7605	/* Check whether we can read the memory, with second call for fetch	7763	/* Check whether we can read the memory, with second call for fetch
7606	* case to simulate the register fill.	7764	* case to simulate the register fill.
7607	*/	7765	*/
7608	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,	7766	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
7609	BPF_SIZE(insn->code), BPF_READ, -1, true, false);	7767	BPF_SIZE(insn->code), BPF_READ, -1, true, false);
7610	if (!err && load_reg >= 0)	7768	if (!err && load_reg >= 0)
7611	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,	7769	err = check_mem_access(env, env->insn_idx, insn->dst_reg,
7612	BPF_SIZE(insn->code), BPF_READ, load_reg,	7770	insn->off, BPF_SIZE(insn->code),
7613	true, false);	7771	BPF_READ, load_reg, true, false);
7614	if (err)	7772	if (err)
7615	return err;	7773	return err;
7616		7774
@@ -7620,13 +7778,86 @@ static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_i · 3 unmodified lines
7620	return err;	7778	return err;
7621	}	7779	}
7622	/* Check whether we can write into the same memory. */	7780	/* Check whether we can write into the same memory. */
7623	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,	7781	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
7624	BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);	7782	BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
7625	if (err)	7783	if (err)
7626	return err;	7784	return err;
7627	return 0;	7785	return 0;
7628	}	7786	}
7629		7787
		7788	static int check_atomic_load(struct bpf_verifier_env *env,
		7789	struct bpf_insn *insn)
		7790	{
		7791	int err;
		7792
		7793	err = check_load_mem(env, insn, true, false, false, "atomic_load");
		7794	if (err)
		7795	return err;
		7796
		7797	if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {
		7798	verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",
		7799	insn->src_reg,
		7800	reg_type_str(env, reg_state(env, insn->src_reg)->type));
		7801	return -EACCES;
		7802	}
		7803
		7804	return 0;
		7805	}
		7806
		7807	static int check_atomic_store(struct bpf_verifier_env *env,
		7808	struct bpf_insn *insn)
		7809	{
		7810	int err;
		7811
		7812	err = check_store_reg(env, insn, true);
		7813	if (err)
		7814	return err;
		7815
		7816	if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
		7817	verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
		7818	insn->dst_reg,
		7819	reg_type_str(env, reg_state(env, insn->dst_reg)->type));
		7820	return -EACCES;
		7821	}
		7822
		7823	return 0;
		7824	}
		7825
		7826	static int check_atomic(struct bpf_verifier_env env, struct bpf_insn insn)
		7827	{
		7828	switch (insn->imm) {
		7829	case BPF_ADD:
		7830	case BPF_ADD \| BPF_FETCH:
		7831	case BPF_AND:
		7832	case BPF_AND \| BPF_FETCH:
		7833	case BPF_OR:
		7834	case BPF_OR \| BPF_FETCH:
		7835	case BPF_XOR:
		7836	case BPF_XOR \| BPF_FETCH:
		7837	case BPF_XCHG:
		7838	case BPF_CMPXCHG:
		7839	return check_atomic_rmw(env, insn);
		7840	case BPF_LOAD_ACQ:
		7841	if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
		7842	verbose(env,
		7843	"64-bit load-acquires are only supported on 64-bit arches\n");
		7844	return -EOPNOTSUPP;
		7845	}
		7846	return check_atomic_load(env, insn);
		7847	case BPF_STORE_REL:
		7848	if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
		7849	verbose(env,
		7850	"64-bit store-releases are only supported on 64-bit arches\n");
		7851	return -EOPNOTSUPP;
		7852	}
		7853	return check_atomic_store(env, insn);
		7854	default:
		7855	verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",
		7856	insn->imm);
		7857	return -EINVAL;
		7858	}
		7859	}
		7860
7630	/* When register 'regno' is used to read the stack (either directly or through	7861	/* When register 'regno' is used to read the stack (either directly or through
7631	* a helper function) make sure that it's within stack boundary and, depending	7862	* a helper function) make sure that it's within stack boundary and, depending
7632	* on the access type and privileges, that all elements of the stack are	7863	* on the access type and privileges, that all elements of the stack are
@@ -8431,10 +8662,12 @@ static struct bpf_verifier_state find_prev_entry(struct bpf_verifier_env env, · 798 unmodified lines
8431	{	8662	{
8432	struct bpf_verifier_state_list *sl;	8663	struct bpf_verifier_state_list *sl;
8433	struct bpf_verifier_state *st;	8664	struct bpf_verifier_state *st;
		8665	struct list_head pos, head;
8434		8666
8435	/* Explored states are pushed in stack order, most recent states come first */	8667	/* Explored states are pushed in stack order, most recent states come first */
8436	sl = *explored_state(env, insn_idx);	8668	head = explored_state(env, insn_idx);
8437	for (; sl; sl = sl->next) {	8669	list_for_each(pos, head) {
		8670	sl = container_of(pos, struct bpf_verifier_state_list, node);
8438	/* If st->branches != 0 state is a part of current DFS verification path,	8671	/* If st->branches != 0 state is a part of current DFS verification path,
8439	* hence cur & st for a loop.	8672	* hence cur & st for a loop.
8440	*/	8673	*/
@@ -9666,7 +9899,7 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 1225 unmodified lines
9666	if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)	9899	if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
9667	goto error;	9900	goto error;
9668	if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {	9901	if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
9669	verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");	9902	verbose(env, "mixing of tail_calls and bpf-to-bpf calls is not supported\n");
9670	return -EINVAL;	9903	return -EINVAL;
9671	}	9904	}
9672	break;	9905	break;
@@ -10237,23 +10470,18 @@ static int check_func_call(struct bpf_verifier_env env, struct bpf_insn insn, · 564 unmodified lines
10237	if (subprog_is_global(env, subprog)) {	10470	if (subprog_is_global(env, subprog)) {
10238	const char *sub_name = subprog_name(env, subprog);	10471	const char *sub_name = subprog_name(env, subprog);
10239		10472
10240	/* Only global subprogs cannot be called with a lock held. */
10241	if (env->cur_state->active_locks) {	10473	if (env->cur_state->active_locks) {
10242	verbose(env, "global function calls are not allowed while holding a lock,\n"	10474	verbose(env, "global function calls are not allowed while holding a lock,\n"
10243	"use static function instead\n");	10475	"use static function instead\n");
10244	return -EINVAL;	10476	return -EINVAL;
10245	}	10477	}
10246		10478
10247	/* Only global subprogs cannot be called with preemption disabled. */	10479	if (env->subprog_info[subprog].might_sleep &&
10248	if (env->cur_state->active_preempt_locks) {	10480	(env->cur_state->active_rcu_lock \|\| env->cur_state->active_preempt_locks \|\|
10249	verbose(env, "global function calls are not allowed with preemption disabled,\n"	10481	env->cur_state->active_irq_id \|\| !in_sleepable(env))) {
10250	"use static function instead\n");	10482	verbose(env, "global functions that may sleep are not allowed in non-sleepable context,\n"
10251	return -EINVAL;	10483	"i.e., in a RCU/IRQ/preempt-disabled section, or in\n"
10252	}	10484	"a non-sleepable BPF program context\n");
10253
10254	if (env->cur_state->active_irq_id) {
10255	verbose(env, "global function calls are not allowed with IRQs disabled,\n"
10256	"use static function instead\n");
10257	return -EINVAL;	10485	return -EINVAL;
10258	}	10486	}
10259		10487
@@ -10752,6 +10980,8 @@ record_func_key(struct bpf_verifier_env env, struct bpf_call_arg_meta meta, · 492 unmodified lines
10752	static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)	10980	static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exit)
10753	{	10981	{
10754	struct bpf_verifier_state *state = env->cur_state;	10982	struct bpf_verifier_state *state = env->cur_state;
		10983	enum bpf_prog_type type = resolve_prog_type(env->prog);
		10984	struct bpf_reg_state *reg = reg_state(env, BPF_REG_0);
10755	bool refs_lingering = false;	10985	bool refs_lingering = false;
10756	int i;	10986	int i;
10757		10987
@@ -10761,6 +10991,12 @@ static int check_reference_leak(struct bpf_verifier_env *env, bool exception_exi · 3 unmodified lines
10761	for (i = 0; i < state->acquired_refs; i++) {	10991	for (i = 0; i < state->acquired_refs; i++) {
10762	if (state->refs[i].type != REF_TYPE_PTR)	10992	if (state->refs[i].type != REF_TYPE_PTR)
10763	continue;	10993	continue;
		10994	/* Allow struct_ops programs to return a referenced kptr back to
		10995	* kernel. Type checks are performed later in check_return_code.
		10996	*/
		10997	if (type == BPF_PROG_TYPE_STRUCT_OPS && !exception_exit &&
		10998	reg->ref_obj_id == state->refs[i].id)
		10999	continue;
10764	verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",	11000	verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
10765	state->refs[i].id, state->refs[i].insn_idx);	11001	state->refs[i].id, state->refs[i].insn_idx);
10766	refs_lingering = true;	11002	refs_lingering = true;
@@ -11781,6 +12017,8 @@ enum special_kfunc_type { · 1014 unmodified lines
11781	KF_bpf_iter_num_new,	12017	KF_bpf_iter_num_new,
11782	KF_bpf_iter_num_next,	12018	KF_bpf_iter_num_next,
11783	KF_bpf_iter_num_destroy,	12019	KF_bpf_iter_num_destroy,
		12020	KF_bpf_set_dentry_xattr,
		12021	KF_bpf_remove_dentry_xattr,
11784	};	12022	};
11785		12023
11786	BTF_SET_START(special_kfunc_set)	12024	BTF_SET_START(special_kfunc_set)
@@ -11810,6 +12048,10 @@ BTF_ID(func, bpf_wq_set_callback_impl) · 23 unmodified lines
11810	#ifdef CONFIG_CGROUPS	12048	#ifdef CONFIG_CGROUPS
11811	BTF_ID(func, bpf_iter_css_task_new)	12049	BTF_ID(func, bpf_iter_css_task_new)
11812	#endif	12050	#endif
		12051	#ifdef CONFIG_BPF_LSM
		12052	BTF_ID(func, bpf_set_dentry_xattr)
		12053	BTF_ID(func, bpf_remove_dentry_xattr)
		12054	#endif
11813	BTF_SET_END(special_kfunc_set)	12055	BTF_SET_END(special_kfunc_set)
11814		12056
11815	BTF_ID_LIST(special_kfunc_list)	12057	BTF_ID_LIST(special_kfunc_list)
@@ -11859,6 +12101,13 @@ BTF_ID(func, bpf_local_irq_restore) · 43 unmodified lines
11859	BTF_ID(func, bpf_iter_num_new)	12101	BTF_ID(func, bpf_iter_num_new)
11860	BTF_ID(func, bpf_iter_num_next)	12102	BTF_ID(func, bpf_iter_num_next)
11861	BTF_ID(func, bpf_iter_num_destroy)	12103	BTF_ID(func, bpf_iter_num_destroy)
		12104	#ifdef CONFIG_BPF_LSM
		12105	BTF_ID(func, bpf_set_dentry_xattr)
		12106	BTF_ID(func, bpf_remove_dentry_xattr)
		12107	#else
		12108	BTF_ID_UNUSED
		12109	BTF_ID_UNUSED
		12110	#endif
11862		12111
11863	static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)	12112	static bool is_kfunc_ret_null(struct bpf_kfunc_call_arg_meta *meta)
11864	{	12113	{
@@ -16399,13 +16648,14 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 4534 unmodified lines
16399	const char *exit_ctx = "At program exit";	16648	const char *exit_ctx = "At program exit";
16400	struct tnum enforce_attach_type_range = tnum_unknown;	16649	struct tnum enforce_attach_type_range = tnum_unknown;
16401	const struct bpf_prog *prog = env->prog;	16650	const struct bpf_prog *prog = env->prog;
16402	struct bpf_reg_state *reg;	16651	struct bpf_reg_state *reg = reg_state(env, regno);
16403	struct bpf_retval_range range = retval_range(0, 1);	16652	struct bpf_retval_range range = retval_range(0, 1);
16404	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);	16653	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
16405	int err;	16654	int err;
16406	struct bpf_func_state *frame = env->cur_state->frame[0];	16655	struct bpf_func_state *frame = env->cur_state->frame[0];
16407	const bool is_subprog = frame->subprogno;	16656	const bool is_subprog = frame->subprogno;
16408	bool return_32bit = false;	16657	bool return_32bit = false;
		16658	const struct btf_type reg_type, ret_type = NULL;
16409		16659
16410	/* LSM and struct_ops func-ptr's return type could be "void" */	16660	/* LSM and struct_ops func-ptr's return type could be "void" */
16411	if (!is_subprog \|\| frame->in_exception_callback_fn) {	16661	if (!is_subprog \|\| frame->in_exception_callback_fn) {
@@ -16414,10 +16664,26 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 2 unmodified lines
16414	if (prog->expected_attach_type == BPF_LSM_CGROUP)	16664	if (prog->expected_attach_type == BPF_LSM_CGROUP)
16415	/* See below, can be 0 or 0-1 depending on hook. */	16665	/* See below, can be 0 or 0-1 depending on hook. */
16416	break;	16666	break;
16417	fallthrough;	16667	if (!prog->aux->attach_func_proto->type)
		16668	return 0;
		16669	break;
16418	case BPF_PROG_TYPE_STRUCT_OPS:	16670	case BPF_PROG_TYPE_STRUCT_OPS:
16419	if (!prog->aux->attach_func_proto->type)	16671	if (!prog->aux->attach_func_proto->type)
16420	return 0;	16672	return 0;
		16673
		16674	if (frame->in_exception_callback_fn)
		16675	break;
		16676
		16677	/* Allow a struct_ops program to return a referenced kptr if it
		16678	* matches the operator's return type and is in its unmodified
		16679	* form. A scalar zero (i.e., a null pointer) is also allowed.
		16680	*/
		16681	reg_type = reg->btf ? btf_type_by_id(reg->btf, reg->btf_id) : NULL;
		16682	ret_type = btf_type_resolve_ptr(prog->aux->attach_btf,
		16683	prog->aux->attach_func_proto->type,
		16684	NULL);
		16685	if (ret_type && ret_type == reg_type && reg->ref_obj_id)
		16686	return __check_ptr_off_reg(env, reg, regno, false);
16421	break;	16687	break;
16422	default:	16688	default:
16423	break;	16689	break;
@@ -16439,8 +16705,6 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 15 unmodified lines
16439	return -EACCES;	16705	return -EACCES;
16440	}	16706	}
16441		16707
16442	reg = cur_regs(env) + regno;
16443
16444	if (frame->in_async_callback_fn) {	16708	if (frame->in_async_callback_fn) {
16445	/* enforce return zero from async callbacks like timer */	16709	/* enforce return zero from async callbacks like timer */
16446	exit_ctx = "At async callback return";	16710	exit_ctx = "At async callback return";
@@ -16539,6 +16803,11 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char · 92 unmodified lines
16539	case BPF_PROG_TYPE_NETFILTER:	16803	case BPF_PROG_TYPE_NETFILTER:
16540	range = retval_range(NF_DROP, NF_ACCEPT);	16804	range = retval_range(NF_DROP, NF_ACCEPT);
16541	break;	16805	break;
		16806	case BPF_PROG_TYPE_STRUCT_OPS:
		16807	if (!ret_type)
		16808	return 0;
		16809	range = retval_range(0, 0);
		16810	break;
16542	case BPF_PROG_TYPE_EXT:	16811	case BPF_PROG_TYPE_EXT:
16543	/* freplace program can return anything as its return value	16812	/* freplace program can return anything as its return value
16544	* depends on the to-be-replaced kernel func or bpf program.	16813	* depends on the to-be-replaced kernel func or bpf program.
@@ -16582,6 +16851,14 @@ static void mark_subprog_changes_pkt_data(struct bpf_verifier_env *env, int off) · 37 unmodified lines
16582	subprog->changes_pkt_data = true;	16851	subprog->changes_pkt_data = true;
16583	}	16852	}
16584		16853
		16854	static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
		16855	{
		16856	struct bpf_subprog_info *subprog;
		16857
		16858	subprog = find_containing_subprog(env, off);
		16859	subprog->might_sleep = true;
		16860	}
		16861
16585	/* 't' is an index of a call-site.	16862	/* 't' is an index of a call-site.
16586	* 'w' is a callee entry point.	16863	* 'w' is a callee entry point.
16587	* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.	16864	* Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
@@ -16595,6 +16872,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w) · 7 unmodified lines
16595	caller = find_containing_subprog(env, t);	16872	caller = find_containing_subprog(env, t);
16596	callee = find_containing_subprog(env, w);	16873	callee = find_containing_subprog(env, w);
16597	caller->changes_pkt_data \|= callee->changes_pkt_data;	16874	caller->changes_pkt_data \|= callee->changes_pkt_data;
		16875	caller->might_sleep \|= callee->might_sleep;
16598	}	16876	}
16599		16877
16600	/* non-recursive DFS pseudo code	16878	/* non-recursive DFS pseudo code
@@ -16753,27 +17031,6 @@ static int visit_func_call_insn(int t, struct bpf_insn *insns, · 152 unmodified lines
16753	/* Bitmask with 1s for all caller saved registers */	17031	/* Bitmask with 1s for all caller saved registers */
16754	#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)	17032	#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
16755		17033
16756	/* Return a bitmask specifying which caller saved registers are
16757	* clobbered by a call to a helper as if this helper follows
16758	* bpf_fastcall contract:
16759	* - includes R0 if function is non-void;
16760	* - includes R1-R5 if corresponding parameter has is described
16761	* in the function prototype.
16762	*/
16763	static u32 helper_fastcall_clobber_mask(const struct bpf_func_proto *fn)
16764	{
16765	u32 mask;
16766	int i;
16767
16768	mask = 0;
16769	if (fn->ret_type != RET_VOID)
16770	mask \|= BIT(BPF_REG_0);
16771	for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i)
16772	if (fn->arg_type[i] != ARG_DONTCARE)
16773	mask \|= BIT(BPF_REG_1 + i);
16774	return mask;
16775	}
16776
16777	/* True if do_misc_fixups() replaces calls to helper number 'imm',	17034	/* True if do_misc_fixups() replaces calls to helper number 'imm',
16778	* replacement patch is presumed to follow bpf_fastcall contract	17035	* replacement patch is presumed to follow bpf_fastcall contract
16779	* (see mark_fastcall_pattern_for_call() below).	17036	* (see mark_fastcall_pattern_for_call() below).
@@ -16790,34 +17047,64 @@ static bool verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm) · 10 unmodified lines
16790	}	17047	}
16791	}	17048	}
16792		17049
16793	/* Same as helper_fastcall_clobber_mask() but for kfuncs, see comment above */	17050	struct call_summary {
16794	static u32 kfunc_fastcall_clobber_mask(struct bpf_kfunc_call_arg_meta *meta)	17051	u8 num_params;
		17052	bool is_void;
		17053	bool fastcall;
		17054	};
		17055
		17056	/* If @call is a kfunc or helper call, fills @cs and returns true,
		17057	* otherwise returns false.
		17058	*/
		17059	static bool get_call_summary(struct bpf_verifier_env env, struct bpf_insn call,
		17060	struct call_summary *cs)
16795	{	17061	{
16796	u32 vlen, i, mask;	17062	struct bpf_kfunc_call_arg_meta meta;
		17063	const struct bpf_func_proto *fn;
		17064	int i;
16797		17065
16798	vlen = btf_type_vlen(meta->func_proto);	17066	if (bpf_helper_call(call)) {
16799	mask = 0;
16800	if (!btf_type_is_void(btf_type_by_id(meta->btf, meta->func_proto->type)))
16801	mask \|= BIT(BPF_REG_0);
16802	for (i = 0; i < vlen; ++i)
16803	mask \|= BIT(BPF_REG_1 + i);
16804	return mask;
16805	}
16806		17067
16807	/* Same as verifier_inlines_helper_call() but for kfuncs, see comment above */	17068	if (get_helper_proto(env, call->imm, &fn) < 0)
16808	static bool is_fastcall_kfunc_call(struct bpf_kfunc_call_arg_meta *meta)	17069	/* error would be reported later */
16809	{	17070	return false;
16810	return meta->kfunc_flags & KF_FASTCALL;	17071	cs->fastcall = fn->allow_fastcall &&
16811	}	17072	(verifier_inlines_helper_call(env, call->imm) \|\|
		17073	bpf_jit_inlines_helper_call(call->imm));
		17074	cs->is_void = fn->ret_type == RET_VOID;
		17075	cs->num_params = 0;
		17076	for (i = 0; i < ARRAY_SIZE(fn->arg_type); ++i) {
		17077	if (fn->arg_type[i] == ARG_DONTCARE)
		17078	break;
		17079	cs->num_params++;
		17080	}
		17081	return true;
		17082	}
16812		17083
16813	/* LLVM define a bpf_fastcall function attribute.	17084	if (bpf_pseudo_kfunc_call(call)) {
16814	* This attribute means that function scratches only some of	17085	int err;
16815	* the caller saved registers defined by ABI.	17086
16816	* For BPF the set of such registers could be defined as follows:	17087	err = fetch_kfunc_meta(env, call, &meta, NULL);
16817	* - R0 is scratched only if function is non-void;	17088	if (err < 0)
16818	* - R1-R5 are scratched only if corresponding parameter type is defined	17089	/* error would be reported later */
16819	* in the function prototype.	17090	return false;
16820	*	17091	cs->num_params = btf_type_vlen(meta.func_proto);
		17092	cs->fastcall = meta.kfunc_flags & KF_FASTCALL;
		17093	cs->is_void = btf_type_is_void(btf_type_by_id(meta.btf, meta.func_proto->type));
		17094	return true;
		17095	}
		17096
		17097	return false;
		17098	}
		17099
		17100	/* LLVM define a bpf_fastcall function attribute.
		17101	* This attribute means that function scratches only some of
		17102	* the caller saved registers defined by ABI.
		17103	* For BPF the set of such registers could be defined as follows:
		17104	* - R0 is scratched only if function is non-void;
		17105	* - R1-R5 are scratched only if corresponding parameter type is defined
		17106	* in the function prototype.
		17107	*
16821	* The contract between kernel and clang allows to simultaneously use	17108	* The contract between kernel and clang allows to simultaneously use
16822	* such functions and maintain backwards compatibility with old	17109	* such functions and maintain backwards compatibility with old
16823	* kernels that don't understand bpf_fastcall calls:	17110	* kernels that don't understand bpf_fastcall calls:
@@ -16890,39 +17177,23 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, · 66 unmodified lines
16890	{	17177	{
16891	struct bpf_insn insns = env->prog->insnsi, stx, *ldx;	17178	struct bpf_insn insns = env->prog->insnsi, stx, *ldx;
16892	struct bpf_insn *call = &env->prog->insnsi[insn_idx];	17179	struct bpf_insn *call = &env->prog->insnsi[insn_idx];
16893	const struct bpf_func_proto *fn;	17180	u32 clobbered_regs_mask;
16894	u32 clobbered_regs_mask = ALL_CALLER_SAVED_REGS;	17181	struct call_summary cs;
16895	u32 expected_regs_mask;	17182	u32 expected_regs_mask;
16896	bool can_be_inlined = false;
16897	s16 off;	17183	s16 off;
16898	int i;	17184	int i;
16899		17185
16900	if (bpf_helper_call(call)) {	17186	if (!get_call_summary(env, call, &cs))
16901	if (get_helper_proto(env, call->imm, &fn) < 0)
16902	/* error would be reported later */
16903	return;
16904	clobbered_regs_mask = helper_fastcall_clobber_mask(fn);
16905	can_be_inlined = fn->allow_fastcall &&
16906	(verifier_inlines_helper_call(env, call->imm) \|\|
16907	bpf_jit_inlines_helper_call(call->imm));
16908	}
16909
16910	if (bpf_pseudo_kfunc_call(call)) {
16911	struct bpf_kfunc_call_arg_meta meta;
16912	int err;
16913
16914	err = fetch_kfunc_meta(env, call, &meta, NULL);
16915	if (err < 0)
16916	/* error would be reported later */
16917	return;
16918
16919	clobbered_regs_mask = kfunc_fastcall_clobber_mask(&meta);
16920	can_be_inlined = is_fastcall_kfunc_call(&meta);
16921	}
16922
16923	if (clobbered_regs_mask == ALL_CALLER_SAVED_REGS)
16924	return;	17187	return;
16925		17188
		17189	/* A bitmask specifying which caller saved registers are clobbered
		17190	* by a call to a helper/kfunc as if this helper/kfunc follows
		17191	* bpf_fastcall contract:
		17192	* - includes R0 if function is non-void;
		17193	* - includes R1-R5 if corresponding parameter has is described
		17194	* in the function prototype.
		17195	*/
		17196	clobbered_regs_mask = GENMASK(cs.num_params, cs.is_void ? 1 : 0);
16926	/* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */	17197	/* e.g. if helper call clobbers r{0,1}, expect r{2,3,4,5} in the pattern */
16927	expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;	17198	expected_regs_mask = ~clobbered_regs_mask & ALL_CALLER_SAVED_REGS;
16928		17199
@@ -16980,7 +17251,7 @@ static void mark_fastcall_pattern_for_call(struct bpf_verifier_env *env, · 51 unmodified lines
16980	* don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()	17251	* don't set 'fastcall_spills_num' for call B so that remove_fastcall_spills_fills()
16981	* does not remove spill/fill pair {4,6}.	17252	* does not remove spill/fill pair {4,6}.
16982	*/	17253	*/
16983	if (can_be_inlined)	17254	if (cs.fastcall)
16984	env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;	17255	env->insn_aux_data[insn_idx].fastcall_spills_num = i - 1;
16985	else	17256	else
16986	subprog->keep_fastcall_stack = 1;	17257	subprog->keep_fastcall_stack = 1;
@@ -17062,9 +17333,20 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 75 unmodified lines
17062	mark_prune_point(env, t);	17333	mark_prune_point(env, t);
17063	mark_jmp_point(env, t);	17334	mark_jmp_point(env, t);
17064	}	17335	}
17065	if (bpf_helper_call(insn) && bpf_helper_changes_pkt_data(insn->imm))	17336	if (bpf_helper_call(insn)) {
17066	mark_subprog_changes_pkt_data(env, t);	17337	const struct bpf_func_proto *fp;
17067	if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {	17338
		17339	ret = get_helper_proto(env, insn->imm, &fp);
		17340	/* If called in a non-sleepable context program will be
		17341	* rejected anyway, so we should end up with precise
		17342	* sleepable marks on subprogs, except for dead code
		17343	* elimination.
		17344	*/
		17345	if (ret == 0 && fp->might_sleep)
		17346	mark_subprog_might_sleep(env, t);
		17347	if (bpf_helper_changes_pkt_data(insn->imm))
		17348	mark_subprog_changes_pkt_data(env, t);
		17349	} else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
17068	struct bpf_kfunc_call_arg_meta meta;	17350	struct bpf_kfunc_call_arg_meta meta;
17069		17351
17070	ret = fetch_kfunc_meta(env, insn, &meta, NULL);	17352	ret = fetch_kfunc_meta(env, insn, &meta, NULL);
@@ -17083,6 +17365,13 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 12 unmodified lines
17083	*/	17365	*/
17084	mark_force_checkpoint(env, t);	17366	mark_force_checkpoint(env, t);
17085	}	17367	}
		17368	/* Same as helpers, if called in a non-sleepable context
		17369	* program will be rejected anyway, so we should end up
		17370	* with precise sleepable marks on subprogs, except for
		17371	* dead code elimination.
		17372	*/
		17373	if (ret == 0 && is_kfunc_sleepable(&meta))
		17374	mark_subprog_might_sleep(env, t);
17086	}	17375	}
17087	return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);	17376	return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
17088		17377
@@ -17125,9 +17414,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env) · 36 unmodified lines
17125	static int check_cfg(struct bpf_verifier_env *env)	17414	static int check_cfg(struct bpf_verifier_env *env)
17126	{	17415	{
17127	int insn_cnt = env->prog->len;	17416	int insn_cnt = env->prog->len;
17128	int insn_stack, insn_state;	17417	int insn_stack, insn_state, *insn_postorder;
17129	int ex_insn_beg, i, ret = 0;	17418	int ex_insn_beg, i, ret = 0;
17130	bool ex_done = false;
17131		17419
17132	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);	17420	insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
17133	if (!insn_state)	17421	if (!insn_state)
@@ -17139,6 +17427,17 @@ static int check_cfg(struct bpf_verifier_env *env) · 5 unmodified lines
17139	return -ENOMEM;	17427	return -ENOMEM;
17140	}	17428	}
17141		17429
		17430	insn_postorder = env->cfg.insn_postorder = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
		17431	if (!insn_postorder) {
		17432	kvfree(insn_state);
		17433	kvfree(insn_stack);
		17434	return -ENOMEM;
		17435	}
		17436
		17437	ex_insn_beg = env->exception_callback_subprog
		17438	? env->subprog_info[env->exception_callback_subprog].start
		17439	: 0;
		17440
17142	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */	17441	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
17143	insn_stack[0] = 0; /* 0 is the first instruction */	17442	insn_stack[0] = 0; /* 0 is the first instruction */
17144	env->cfg.cur_stack = 1;	17443	env->cfg.cur_stack = 1;
@@ -17152,6 +17451,7 @@ walk_cfg: · 7 unmodified lines
17152	case DONE_EXPLORING:	17451	case DONE_EXPLORING:
17153	insn_state[t] = EXPLORED;	17452	insn_state[t] = EXPLORED;
17154	env->cfg.cur_stack--;	17453	env->cfg.cur_stack--;
		17454	insn_postorder[env->cfg.cur_postorder++] = t;
17155	break;	17455	break;
17156	case KEEP_EXPLORING:	17456	case KEEP_EXPLORING:
17157	break;	17457	break;
@@ -17170,13 +17470,10 @@ walk_cfg: · 12 unmodified lines
17170	goto err_free;	17470	goto err_free;
17171	}	17471	}
17172		17472
17173	if (env->exception_callback_subprog && !ex_done) {	17473	if (ex_insn_beg && insn_state[ex_insn_beg] != EXPLORED) {
17174	ex_insn_beg = env->subprog_info[env->exception_callback_subprog].start;
17175
17176	insn_state[ex_insn_beg] = DISCOVERED;	17474	insn_state[ex_insn_beg] = DISCOVERED;
17177	insn_stack[0] = ex_insn_beg;	17475	insn_stack[0] = ex_insn_beg;
17178	env->cfg.cur_stack = 1;	17476	env->cfg.cur_stack = 1;
17179	ex_done = true;
17180	goto walk_cfg;	17477	goto walk_cfg;
17181	}	17478	}
17182		17479
@@ -17199,6 +17496,7 @@ walk_cfg: · 16 unmodified lines
17199	}	17496	}
17200	ret = 0; /* cfg looks good */	17497	ret = 0; /* cfg looks good */
17201	env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;	17498	env->prog->aux->changes_pkt_data = env->subprog_info[0].changes_pkt_data;
		17499	env->prog->aux->might_sleep = env->subprog_info[0].might_sleep;
17202		17500
17203	err_free:	17501	err_free:
17204	kvfree(insn_state);	17502	kvfree(insn_state);
@@ -17815,18 +18113,22 @@ static void clean_verifier_state(struct bpf_verifier_env *env, · 610 unmodified lines
17815	static void clean_live_states(struct bpf_verifier_env *env, int insn,	18113	static void clean_live_states(struct bpf_verifier_env *env, int insn,
17816	struct bpf_verifier_state *cur)	18114	struct bpf_verifier_state *cur)
17817	{	18115	{
		18116	struct bpf_verifier_state *loop_entry;
17818	struct bpf_verifier_state_list *sl;	18117	struct bpf_verifier_state_list *sl;
		18118	struct list_head pos, head;
17819		18119
17820	sl = *explored_state(env, insn);	18120	head = explored_state(env, insn);
17821	while (sl) {	18121	list_for_each(pos, head) {
		18122	sl = container_of(pos, struct bpf_verifier_state_list, node);
17822	if (sl->state.branches)	18123	if (sl->state.branches)
17823	goto next;	18124	continue;
		18125	loop_entry = get_loop_entry(env, &sl->state);
		18126	if (!IS_ERR_OR_NULL(loop_entry) && loop_entry->branches)
		18127	continue;
17824	if (sl->state.insn_idx != insn \|\|	18128	if (sl->state.insn_idx != insn \|\|
17825	!same_callsites(&sl->state, cur))	18129	!same_callsites(&sl->state, cur))
17826	goto next;	18130	continue;
17827	clean_verifier_state(env, &sl->state);	18131	clean_verifier_state(env, &sl->state);
17828	next:
17829	sl = sl->next;
17830	}	18132	}
17831	}	18133	}
17832		18134
@@ -18210,15 +18512,17 @@ static bool refsafe(struct bpf_verifier_state old, struct bpf_verifier_state c · 377 unmodified lines
18210	* the current state will reach 'bpf_exit' instruction safely	18512	* the current state will reach 'bpf_exit' instruction safely
18211	*/	18513	*/
18212	static bool func_states_equal(struct bpf_verifier_env env, struct bpf_func_state old,	18514	static bool func_states_equal(struct bpf_verifier_env env, struct bpf_func_state old,
18213	struct bpf_func_state *cur, enum exact_level exact)	18515	struct bpf_func_state *cur, u32 insn_idx, enum exact_level exact)
18214	{	18516	{
18215	int i;	18517	u16 live_regs = env->insn_aux_data[insn_idx].live_regs_before;
		18518	u16 i;
18216		18519
18217	if (old->callback_depth > cur->callback_depth)	18520	if (old->callback_depth > cur->callback_depth)
18218	return false;	18521	return false;
18219		18522
18220	for (i = 0; i < MAX_BPF_REG; i++)	18523	for (i = 0; i < MAX_BPF_REG; i++)
18221	if (!regsafe(env, &old->regs[i], &cur->regs[i],	18524	if (((1 << i) & live_regs) &&
		18525	!regsafe(env, &old->regs[i], &cur->regs[i],
18222	&env->idmap_scratch, exact))	18526	&env->idmap_scratch, exact))
18223	return false;	18527	return false;
18224		18528
@@ -18239,6 +18543,7 @@ static bool states_equal(struct bpf_verifier_env *env, · 14 unmodified lines
18239	struct bpf_verifier_state *cur,	18543	struct bpf_verifier_state *cur,
18240	enum exact_level exact)	18544	enum exact_level exact)
18241	{	18545	{
		18546	u32 insn_idx;
18242	int i;	18547	int i;
18243		18548
18244	if (old->curframe != cur->curframe)	18549	if (old->curframe != cur->curframe)
@@ -18262,9 +18567,12 @@ static bool states_equal(struct bpf_verifier_env *env, · 17 unmodified lines
18262	* and all frame states need to be equivalent	18567	* and all frame states need to be equivalent
18263	*/	18568	*/
18264	for (i = 0; i <= old->curframe; i++) {	18569	for (i = 0; i <= old->curframe; i++) {
		18570	insn_idx = i == old->curframe
		18571	? env->insn_idx
		18572	: old->frame[i + 1]->callsite;
18265	if (old->frame[i]->callsite != cur->frame[i]->callsite)	18573	if (old->frame[i]->callsite != cur->frame[i]->callsite)
18266	return false;	18574	return false;
18267	if (!func_states_equal(env, old->frame[i], cur->frame[i], exact))	18575	if (!func_states_equal(env, old->frame[i], cur->frame[i], insn_idx, exact))
18268	return false;	18576	return false;
18269	}	18577	}
18270	return true;	18578	return true;
@@ -18517,10 +18825,11 @@ static bool iter_active_depths_differ(struct bpf_verifier_state *old, struct bpf · 246 unmodified lines
18517	static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)	18825	static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
18518	{	18826	{
18519	struct bpf_verifier_state_list *new_sl;	18827	struct bpf_verifier_state_list *new_sl;
18520	struct bpf_verifier_state_list sl, *pprev;	18828	struct bpf_verifier_state_list *sl;
18521	struct bpf_verifier_state cur = env->cur_state, new, *loop_entry;	18829	struct bpf_verifier_state cur = env->cur_state, new, *loop_entry;
18522	int i, j, n, err, states_cnt = 0;	18830	int i, j, n, err, states_cnt = 0;
18523	bool force_new_state, add_new_state, force_exact;	18831	bool force_new_state, add_new_state, force_exact;
		18832	struct list_head pos, tmp, *head;
18524		18833
18525	force_new_state = env->test_state_freq \|\| is_force_checkpoint(env, insn_idx) \|\|	18834	force_new_state = env->test_state_freq \|\| is_force_checkpoint(env, insn_idx) \|\|
18526	/* Avoid accumulating infinitely long jmp history */	18835	/* Avoid accumulating infinitely long jmp history */
@@ -18539,15 +18848,14 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 12 unmodified lines
18539	env->insn_processed - env->prev_insn_processed >= 8)	18848	env->insn_processed - env->prev_insn_processed >= 8)
18540	add_new_state = true;	18849	add_new_state = true;
18541		18850
18542	pprev = explored_state(env, insn_idx);
18543	sl = *pprev;
18544
18545	clean_live_states(env, insn_idx, cur);	18851	clean_live_states(env, insn_idx, cur);
18546		18852
18547	while (sl) {	18853	head = explored_state(env, insn_idx);
		18854	list_for_each_safe(pos, tmp, head) {
		18855	sl = container_of(pos, struct bpf_verifier_state_list, node);
18548	states_cnt++;	18856	states_cnt++;
18549	if (sl->state.insn_idx != insn_idx)	18857	if (sl->state.insn_idx != insn_idx)
18550	goto next;	18858	continue;
18551		18859
18552	if (sl->state.branches) {	18860	if (sl->state.branches) {
18553	struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];	18861	struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
@@ -18621,7 +18929,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 67 unmodified lines
18621	spi = __get_spi(iter_reg->off + iter_reg->var_off.value);	18929	spi = __get_spi(iter_reg->off + iter_reg->var_off.value);
18622	iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;	18930	iter_state = &func(env, iter_reg)->stack[spi].spilled_ptr;
18623	if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {	18931	if (iter_state->iter.state == BPF_ITER_STATE_ACTIVE) {
18624	update_loop_entry(cur, &sl->state);	18932	update_loop_entry(env, cur, &sl->state);
18625	goto hit;	18933	goto hit;
18626	}	18934	}
18627	}	18935	}
@@ -18630,7 +18938,7 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 2 unmodified lines
18630	if (is_may_goto_insn_at(env, insn_idx)) {	18938	if (is_may_goto_insn_at(env, insn_idx)) {
18631	if (sl->state.may_goto_depth != cur->may_goto_depth &&	18939	if (sl->state.may_goto_depth != cur->may_goto_depth &&
18632	states_equal(env, &sl->state, cur, RANGE_WITHIN)) {	18940	states_equal(env, &sl->state, cur, RANGE_WITHIN)) {
18633	update_loop_entry(cur, &sl->state);	18941	update_loop_entry(env, cur, &sl->state);
18634	goto hit;	18942	goto hit;
18635	}	18943	}
18636	}	18944	}
@@ -18697,11 +19005,13 @@ skip_inf_loop_check: · 60 unmodified lines
18697	*	19005	*
18698	* Additional details are in the comment before get_loop_entry().	19006	* Additional details are in the comment before get_loop_entry().
18699	*/	19007	*/
18700	loop_entry = get_loop_entry(&sl->state);	19008	loop_entry = get_loop_entry(env, &sl->state);
		19009	if (IS_ERR(loop_entry))
		19010	return PTR_ERR(loop_entry);
18701	force_exact = loop_entry && loop_entry->branches > 0;	19011	force_exact = loop_entry && loop_entry->branches > 0;
18702	if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) {	19012	if (states_equal(env, &sl->state, cur, force_exact ? RANGE_WITHIN : NOT_EXACT)) {
18703	if (force_exact)	19013	if (force_exact)
18704	update_loop_entry(cur, loop_entry);	19014	update_loop_entry(env, cur, loop_entry);
18705	hit:	19015	hit:
18706	sl->hit_cnt++;	19016	sl->hit_cnt++;
18707	/* reached equivalent register/stack state,	19017	/* reached equivalent register/stack state,
@@ -18750,31 +19060,13 @@ miss: · 42 unmodified lines
18750	/* the state is unlikely to be useful. Remove it to	19060	/* the state is unlikely to be useful. Remove it to
18751	* speed up verification	19061	* speed up verification
18752	*/	19062	*/
18753	*pprev = sl->next;	19063	sl->in_free_list = true;
18754	if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE &&	19064	list_del(&sl->node);
18755	!sl->state.used_as_loop_entry) {	19065	list_add(&sl->node, &env->free_list);
18756	u32 br = sl->state.branches;	19066	env->free_list_size++;
18757		19067	env->explored_states_size--;
18758	WARN_ONCE(br,	19068	maybe_free_verifier_state(env, sl);
18759	"BUG live_done but branches_to_explore %d\n",
18760	br);
18761	free_verifier_state(&sl->state, false);
18762	kfree(sl);
18763	env->peak_states--;
18764	} else {
18765	/* cannot free this state, since parentage chain may
18766	* walk it later. Add it for free_list instead to
18767	* be freed at the end of verification
18768	*/
18769	sl->next = env->free_list;
18770	env->free_list = sl;
18771	}
18772	sl = *pprev;
18773	continue;
18774	}	19069	}
18775	next:
18776	pprev = &sl->next;
18777	sl = *pprev;
18778	}	19070	}
18779		19071
18780	if (env->max_states_per_insn < states_cnt)	19072	if (env->max_states_per_insn < states_cnt)
@@ -18799,7 +19091,8 @@ next: · 18 unmodified lines
18799	if (!new_sl)	19091	if (!new_sl)
18800	return -ENOMEM;	19092	return -ENOMEM;
18801	env->total_states++;	19093	env->total_states++;
18802	env->peak_states++;	19094	env->explored_states_size++;
		19095	update_peak_states(env);
18803	env->prev_jmps_processed = env->jmps_processed;	19096	env->prev_jmps_processed = env->jmps_processed;
18804	env->prev_insn_processed = env->insn_processed;	19097	env->prev_insn_processed = env->insn_processed;
18805		19098
@@ -18823,8 +19116,8 @@ next: · 17 unmodified lines
18823	cur->first_insn_idx = insn_idx;	19116	cur->first_insn_idx = insn_idx;
18824	cur->insn_hist_start = cur->insn_hist_end;	19117	cur->insn_hist_start = cur->insn_hist_end;
18825	cur->dfs_depth = new->dfs_depth + 1;	19118	cur->dfs_depth = new->dfs_depth + 1;
18826	new_sl->next = *explored_state(env, insn_idx);	19119	list_add(&new_sl->node, head);
18827	*explored_state(env, insn_idx) = new_sl;	19120
18828	/* connect new state to parentage chain. Current frame needs all	19121	/* connect new state to parentage chain. Current frame needs all
18829	* registers connected. Only r6 - r9 of the callers are alive (pushed	19122	* registers connected. Only r6 - r9 of the callers are alive (pushed
18830	* to the stack implicitly by JITs) so in callers' frames connect just	19123	* to the stack implicitly by JITs) so in callers' frames connect just
@@ -19011,19 +19304,13 @@ static int do_check(struct bpf_verifier_env *env) · 180 unmodified lines
19011	}	19304	}
19012		19305
19013	if (env->log.level & BPF_LOG_LEVEL) {	19306	if (env->log.level & BPF_LOG_LEVEL) {
19014	const struct bpf_insn_cbs cbs = {
19015	.cb_call = disasm_kfunc_name,
19016	.cb_print = verbose,
19017	.private_data = env,
19018	};
19019
19020	if (verifier_state_scratched(env))	19307	if (verifier_state_scratched(env))
19021	print_insn_state(env, state, state->curframe);	19308	print_insn_state(env, state, state->curframe);
19022		19309
19023	verbose_linfo(env, env->insn_idx, "; ");	19310	verbose_linfo(env, env->insn_idx, "; ");
19024	env->prev_log_pos = env->log.end_pos;	19311	env->prev_log_pos = env->log.end_pos;
19025	verbose(env, "%d: ", env->insn_idx);	19312	verbose(env, "%d: ", env->insn_idx);
19026	print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);	19313	verbose_insn(env, insn);
19027	env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;	19314	env->prev_insn_print_pos = env->log.end_pos - env->prev_log_pos;
19028	env->prev_log_pos = env->log.end_pos;	19315	env->prev_log_pos = env->log.end_pos;
19029	}	19316	}
@@ -19045,37 +19332,18 @@ static int do_check(struct bpf_verifier_env *env) · 15 unmodified lines
19045	return err;	19332	return err;
19046		19333
19047	} else if (class == BPF_LDX) {	19334	} else if (class == BPF_LDX) {
19048	enum bpf_reg_type src_reg_type;	19335	bool is_ldsx = BPF_MODE(insn->code) == BPF_MEMSX;
19049
19050	/* check for reserved fields is already done */
19051
19052	/* check src operand */
19053	err = check_reg_arg(env, insn->src_reg, SRC_OP);
19054	if (err)
19055	return err;
19056
19057	err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
19058	if (err)
19059	return err;
19060
19061	src_reg_type = regs[insn->src_reg].type;
19062		19336
19063	/* check that memory (src_reg + off) is readable,	19337	/* Check for reserved fields is already done in
19064	* the state of dst_reg will be updated by this func	19338	* resolve_pseudo_ldimm64().
19065	*/	19339	*/
19066	err = check_mem_access(env, env->insn_idx, insn->src_reg,	19340	err = check_load_mem(env, insn, false, is_ldsx, true,
19067	insn->off, BPF_SIZE(insn->code),	19341	"ldx");
19068	BPF_READ, insn->dst_reg, false,
19069	BPF_MODE(insn->code) == BPF_MEMSX);
19070	err = err ?: save_aux_ptr_type(env, src_reg_type, true);
19071	err = err ?: reg_bounds_sanity_check(env, &regs[insn->dst_reg], "ldx");
19072	if (err)	19342	if (err)
19073	return err;	19343	return err;
19074	} else if (class == BPF_STX) {	19344	} else if (class == BPF_STX) {
19075	enum bpf_reg_type dst_reg_type;
19076
19077	if (BPF_MODE(insn->code) == BPF_ATOMIC) {	19345	if (BPF_MODE(insn->code) == BPF_ATOMIC) {
19078	err = check_atomic(env, env->insn_idx, insn);	19346	err = check_atomic(env, insn);
19079	if (err)	19347	if (err)
19080	return err;	19348	return err;
19081	env->insn_idx++;	19349	env->insn_idx++;
@@ -19087,25 +19355,7 @@ static int do_check(struct bpf_verifier_env *env) · 5 unmodified lines
19087	return -EINVAL;	19355	return -EINVAL;
19088	}	19356	}
19089		19357
19090	/* check src1 operand */	19358	err = check_store_reg(env, insn, false);
19091	err = check_reg_arg(env, insn->src_reg, SRC_OP);
19092	if (err)
19093	return err;
19094	/* check src2 operand */
19095	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
19096	if (err)
19097	return err;
19098
19099	dst_reg_type = regs[insn->dst_reg].type;
19100
19101	/* check that memory (dst_reg + off) is writeable */
19102	err = check_mem_access(env, env->insn_idx, insn->dst_reg,
19103	insn->off, BPF_SIZE(insn->code),
19104	BPF_WRITE, insn->src_reg, false, false);
19105	if (err)
19106	return err;
19107
19108	err = save_aux_ptr_type(env, dst_reg_type, false);
19109	if (err)	19359	if (err)
19110	return err;	19360	return err;
19111	} else if (class == BPF_ST) {	19361	} else if (class == BPF_ST) {
@@ -19245,6 +19495,10 @@ process_bpf_exit: · 133 unmodified lines
19245	return err;	19495	return err;
19246	break;	19496	break;
19247	} else {	19497	} else {
		19498	if (WARN_ON_ONCE(env->cur_state->loop_entry)) {
		19499	verbose(env, "verifier bug: env->cur_state->loop_entry != NULL\n");
		19500	return -EFAULT;
		19501	}
19248	do_print_state = true;	19502	do_print_state = true;
19249	continue;	19503	continue;
19250	}	19504	}
@@ -20334,7 +20588,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 1083 unmodified lines
20334	{	20588	{
20335	struct bpf_subprog_info *subprogs = env->subprog_info;	20589	struct bpf_subprog_info *subprogs = env->subprog_info;
20336	const struct bpf_verifier_ops *ops = env->ops;	20590	const struct bpf_verifier_ops *ops = env->ops;
20337	int i, cnt, size, ctx_field_size, delta = 0, epilogue_cnt = 0;	20591	int i, cnt, size, ctx_field_size, ret, delta = 0, epilogue_cnt = 0;
20338	const int insn_cnt = env->prog->len;	20592	const int insn_cnt = env->prog->len;
20339	struct bpf_insn *epilogue_buf = env->epilogue_buf;	20593	struct bpf_insn *epilogue_buf = env->epilogue_buf;
20340	struct bpf_insn *insn_buf = env->insn_buf;	20594	struct bpf_insn *insn_buf = env->insn_buf;
@@ -20363,6 +20617,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 22 unmodified lines
20363	return -ENOMEM;	20617	return -ENOMEM;
20364	env->prog = new_prog;	20618	env->prog = new_prog;
20365	delta += cnt - 1;	20619	delta += cnt - 1;
		20620
		20621	ret = add_kfunc_in_insns(env, epilogue_buf, epilogue_cnt - 1);
		20622	if (ret < 0)
		20623	return ret;
20366	}	20624	}
20367	}	20625	}
20368		20626
@@ -20383,6 +20641,10 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 14 unmodified lines
20383		20641
20384	env->prog = new_prog;	20642	env->prog = new_prog;
20385	delta += cnt - 1;	20643	delta += cnt - 1;
		20644
		20645	ret = add_kfunc_in_insns(env, insn_buf, cnt - 1);
		20646	if (ret < 0)
		20647	return ret;
20386	}	20648	}
20387	}	20649	}
20388		20650
@@ -20415,7 +20677,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 26 unmodified lines
20415	insn->code == (BPF_ST \| BPF_MEM \| BPF_W) \|\|	20677	insn->code == (BPF_ST \| BPF_MEM \| BPF_W) \|\|
20416	insn->code == (BPF_ST \| BPF_MEM \| BPF_DW)) {	20678	insn->code == (BPF_ST \| BPF_MEM \| BPF_DW)) {
20417	type = BPF_WRITE;	20679	type = BPF_WRITE;
20418	} else if ((insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_W) \|\|	20680	} else if ((insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_B) \|\|
		20681	insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_H) \|\|
		20682	insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_W) \|\|
20419	insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_DW)) &&	20683	insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_DW)) &&
20420	env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {	20684	env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
20421	insn->code = BPF_STX \| BPF_PROBE_ATOMIC \| BPF_SIZE(insn->code);	20685	insn->code = BPF_STX \| BPF_PROBE_ATOMIC \| BPF_SIZE(insn->code);
@@ -20723,6 +20987,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 301 unmodified lines
20723	func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;	20987	func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
20724	func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;	20988	func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
20725	func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;	20989	func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
		20990	func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
20726	if (!i)	20991	if (!i)
20727	func[i]->aux->exception_boundary = env->seen_exception;	20992	func[i]->aux->exception_boundary = env->seen_exception;
20728	func[i] = bpf_int_jit_compile(func[i]);	20993	func[i] = bpf_int_jit_compile(func[i]);
@@ -20939,6 +21204,14 @@ static void specialize_kfunc(struct bpf_verifier_env *env, · 210 unmodified lines
20939	*/	21204	*/
20940	env->seen_direct_write = seen_direct_write;	21205	env->seen_direct_write = seen_direct_write;
20941	}	21206	}
		21207
		21208	if (func_id == special_kfunc_list[KF_bpf_set_dentry_xattr] &&
		21209	bpf_lsm_has_d_inode_locked(prog))
		21210	*addr = (unsigned long)bpf_set_dentry_xattr_locked;
		21211
		21212	if (func_id == special_kfunc_list[KF_bpf_remove_dentry_xattr] &&
		21213	bpf_lsm_has_d_inode_locked(prog))
		21214	*addr = (unsigned long)bpf_remove_dentry_xattr_locked;
20942	}	21215	}
20943		21216
20944	static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,	21217	static void __fixup_collection_insert_kfunc(struct bpf_insn_aux_data *insn_aux,
@@ -21373,7 +21646,50 @@ static int do_misc_fixups(struct bpf_verifier_env *env) · 428 unmodified lines
21373	goto next_insn;	21646	goto next_insn;
21374	}	21647	}
21375		21648
21376	if (is_may_goto_insn(insn)) {	21649	if (is_may_goto_insn(insn) && bpf_jit_supports_timed_may_goto()) {
		21650	int stack_off_cnt = -stack_depth - 16;
		21651
		21652	/*
		21653	* Two 8 byte slots, depth-16 stores the count, and
		21654	* depth-8 stores the start timestamp of the loop.
		21655	*
		21656	* The starting value of count is BPF_MAX_TIMED_LOOPS
		21657	* (0xffff). Every iteration loads it and subs it by 1,
		21658	* until the value becomes 0 in AX (thus, 1 in stack),
		21659	* after which we call arch_bpf_timed_may_goto, which
		21660	* either sets AX to 0xffff to keep looping, or to 0
		21661	* upon timeout. AX is then stored into the stack. In
		21662	* the next iteration, we either see 0 and break out, or
		21663	* continue iterating until the next time value is 0
		21664	* after subtraction, rinse and repeat.
		21665	*/
		21666	stack_depth_extra = 16;
		21667	insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off_cnt);
		21668	if (insn->off >= 0)
		21669	insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 5);
		21670	else
		21671	insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off - 1);
		21672	insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
		21673	insn_buf[3] = BPF_JMP_IMM(BPF_JNE, BPF_REG_AX, 0, 2);
		21674	/*
		21675	* AX is used as an argument to pass in stack_off_cnt
		21676	* (to add to r10/fp), and also as the return value of
		21677	* the call to arch_bpf_timed_may_goto.
		21678	*/
		21679	insn_buf[4] = BPF_MOV64_IMM(BPF_REG_AX, stack_off_cnt);
		21680	insn_buf[5] = BPF_EMIT_CALL(arch_bpf_timed_may_goto);
		21681	insn_buf[6] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off_cnt);
		21682	cnt = 7;
		21683
		21684	new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
		21685	if (!new_prog)
		21686	return -ENOMEM;
		21687
		21688	delta += cnt - 1;
		21689	env->prog = prog = new_prog;
		21690	insn = new_prog->insnsi + i + delta;
		21691	goto next_insn;
		21692	} else if (is_may_goto_insn(insn)) {
21377	int stack_off = -stack_depth - 8;	21693	int stack_off = -stack_depth - 8;
21378		21694
21379	stack_depth_extra = 8;	21695	stack_depth_extra = 8;
@@ -21897,6 +22213,13 @@ next_insn: · 517 unmodified lines
21897	if (subprogs[cur_subprog + 1].start == i + delta + 1) {	22213	if (subprogs[cur_subprog + 1].start == i + delta + 1) {
21898	subprogs[cur_subprog].stack_depth += stack_depth_extra;	22214	subprogs[cur_subprog].stack_depth += stack_depth_extra;
21899	subprogs[cur_subprog].stack_extra = stack_depth_extra;	22215	subprogs[cur_subprog].stack_extra = stack_depth_extra;
		22216
		22217	stack_depth = subprogs[cur_subprog].stack_depth;
		22218	if (stack_depth > MAX_BPF_STACK && !prog->jit_requested) {
		22219	verbose(env, "stack size %d(extra %d) is too large\n",
		22220	stack_depth, stack_depth_extra);
		22221	return -EINVAL;
		22222	}
21900	cur_subprog++;	22223	cur_subprog++;
21901	stack_depth = subprogs[cur_subprog].stack_depth;	22224	stack_depth = subprogs[cur_subprog].stack_depth;
21902	stack_depth_extra = 0;	22225	stack_depth_extra = 0;
@@ -21907,23 +22230,33 @@ next_insn: · 4 unmodified lines
21907		22230
21908	env->prog->aux->stack_depth = subprogs[0].stack_depth;	22231	env->prog->aux->stack_depth = subprogs[0].stack_depth;
21909	for (i = 0; i < env->subprog_cnt; i++) {	22232	for (i = 0; i < env->subprog_cnt; i++) {
		22233	int delta = bpf_jit_supports_timed_may_goto() ? 2 : 1;
21910	int subprog_start = subprogs[i].start;	22234	int subprog_start = subprogs[i].start;
21911	int stack_slots = subprogs[i].stack_extra / 8;	22235	int stack_slots = subprogs[i].stack_extra / 8;
		22236	int slots = delta, cnt = 0;
21912		22237
21913	if (!stack_slots)	22238	if (!stack_slots)
21914	continue;	22239	continue;
21915	if (stack_slots > 1) {	22240	/* We need two slots in case timed may_goto is supported. */
		22241	if (stack_slots > slots) {
21916	verbose(env, "verifier bug: stack_slots supports may_goto only\n");	22242	verbose(env, "verifier bug: stack_slots supports may_goto only\n");
21917	return -EFAULT;	22243	return -EFAULT;
21918	}	22244	}
21919		22245
21920	/* Add ST insn to subprog prologue to init extra stack */	22246	stack_depth = subprogs[i].stack_depth;
21921	insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP,	22247	if (bpf_jit_supports_timed_may_goto()) {
21922	-subprogs[i].stack_depth, BPF_MAX_LOOPS);	22248	insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
		22249	BPF_MAX_TIMED_LOOPS);
		22250	insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth + 8, 0);
		22251	} else {
		22252	/* Add ST insn to subprog prologue to init extra stack */
		22253	insn_buf[cnt++] = BPF_ST_MEM(BPF_DW, BPF_REG_FP, -stack_depth,
		22254	BPF_MAX_LOOPS);
		22255	}
21923	/* Copy first actual insn to preserve it */	22256	/* Copy first actual insn to preserve it */
21924	insn_buf[1] = env->prog->insnsi[subprog_start];	22257	insn_buf[cnt++] = env->prog->insnsi[subprog_start];
21925		22258
21926	new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2);	22259	new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, cnt);
21927	if (!new_prog)	22260	if (!new_prog)
21928	return -ENOMEM;	22261	return -ENOMEM;
21929	env->prog = prog = new_prog;	22262	env->prog = prog = new_prog;
@@ -21933,7 +22266,7 @@ next_insn: · 3 unmodified lines
21933	* to insn after BPF_ST that inits may_goto count.	22266	* to insn after BPF_ST that inits may_goto count.
21934	* Adjustment will succeed because bpf_patch_insn_data() didn't fail.	22267	* Adjustment will succeed because bpf_patch_insn_data() didn't fail.
21935	*/	22268	*/
21936	WARN_ON(adjust_jmp_off(env->prog, subprog_start, 1));	22269	WARN_ON(adjust_jmp_off(env->prog, subprog_start, delta));
21937	}	22270	}
21938		22271
21939	/* Since poke tab is now finalized, publish aux to tracker. */	22272	/* Since poke tab is now finalized, publish aux to tracker. */
@@ -22131,31 +22464,29 @@ static int remove_fastcall_spills_fills(struct bpf_verifier_env *env) · 191 unmodified lines
22131		22464
22132	static void free_states(struct bpf_verifier_env *env)	22465	static void free_states(struct bpf_verifier_env *env)
22133	{	22466	{
22134	struct bpf_verifier_state_list sl, sln;	22467	struct bpf_verifier_state_list *sl;
		22468	struct list_head head, pos, *tmp;
22135	int i;	22469	int i;
22136		22470
22137	sl = env->free_list;	22471	list_for_each_safe(pos, tmp, &env->free_list) {
22138	while (sl) {	22472	sl = container_of(pos, struct bpf_verifier_state_list, node);
22139	sln = sl->next;
22140	free_verifier_state(&sl->state, false);	22473	free_verifier_state(&sl->state, false);
22141	kfree(sl);	22474	kfree(sl);
22142	sl = sln;
22143	}	22475	}
22144	env->free_list = NULL;	22476	INIT_LIST_HEAD(&env->free_list);
22145		22477
22146	if (!env->explored_states)	22478	if (!env->explored_states)
22147	return;	22479	return;
22148		22480
22149	for (i = 0; i < state_htab_size(env); i++) {	22481	for (i = 0; i < state_htab_size(env); i++) {
22150	sl = env->explored_states[i];	22482	head = &env->explored_states[i];
22151		22483
22152	while (sl) {	22484	list_for_each_safe(pos, tmp, head) {
22153	sln = sl->next;	22485	sl = container_of(pos, struct bpf_verifier_state_list, node);
22154	free_verifier_state(&sl->state, false);	22486	free_verifier_state(&sl->state, false);
22155	kfree(sl);	22487	kfree(sl);
22156	sl = sln;
22157	}	22488	}
22158	env->explored_states[i] = NULL;	22489	INIT_LIST_HEAD(&env->explored_states[i]);
22159	}	22490	}
22160	}	22491	}
22161		22492
@@ -22163,6 +22494,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 1 unmodified line
22163	{	22494	{
22164	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);	22495	bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
22165	struct bpf_subprog_info *sub = subprog_info(env, subprog);	22496	struct bpf_subprog_info *sub = subprog_info(env, subprog);
		22497	struct bpf_prog_aux *aux = env->prog->aux;
22166	struct bpf_verifier_state *state;	22498	struct bpf_verifier_state *state;
22167	struct bpf_reg_state *regs;	22499	struct bpf_reg_state *regs;
22168	int ret, i;	22500	int ret, i;
@@ -22270,6 +22602,13 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) · 101 unmodified lines
22270	mark_reg_known_zero(env, regs, BPF_REG_1);	22602	mark_reg_known_zero(env, regs, BPF_REG_1);
22271	}	22603	}
22272		22604
		22605	/* Acquire references for struct_ops program arguments tagged with "__ref" */
		22606	if (!subprog && env->prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
		22607	for (i = 0; i < aux->ctx_arg_info_size; i++)
		22608	aux->ctx_arg_info[i].ref_obj_id = aux->ctx_arg_info[i].refcounted ?
		22609	acquire_reference(env, 0) : 0;
		22610	}
		22611
22273	ret = do_check(env);	22612	ret = do_check(env);
22274	out:	22613	out:
22275	/* check for NULL is necessary, since cur_state can be freed inside	22614	/* check for NULL is necessary, since cur_state can be freed inside
@@ -22392,6 +22731,15 @@ static void print_verification_stats(struct bpf_verifier_env *env) · 116 unmodified lines
22392	env->peak_states, env->longest_mark_read_walk);	22731	env->peak_states, env->longest_mark_read_walk);
22393	}	22732	}
22394		22733
		22734	int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog,
		22735	const struct bpf_ctx_arg_aux *info, u32 cnt)
		22736	{
		22737	prog->aux->ctx_arg_info = kmemdup_array(info, cnt, sizeof(*info), GFP_KERNEL);
		22738	prog->aux->ctx_arg_info_size = cnt;
		22739
		22740	return prog->aux->ctx_arg_info ? 0 : -ENOMEM;
		22741	}
		22742
22395	static int check_struct_ops_btf_id(struct bpf_verifier_env *env)	22743	static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
22396	{	22744	{
22397	const struct btf_type t, func_proto;	22745	const struct btf_type t, func_proto;
@@ -22399,10 +22747,11 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 1 unmodified line
22399	const struct bpf_struct_ops *st_ops;	22747	const struct bpf_struct_ops *st_ops;
22400	const struct btf_member *member;	22748	const struct btf_member *member;
22401	struct bpf_prog *prog = env->prog;	22749	struct bpf_prog *prog = env->prog;
22402	u32 btf_id, member_idx;	22750	bool has_refcounted_arg = false;
		22751	u32 btf_id, member_idx, member_off;
22403	struct btf *btf;	22752	struct btf *btf;
22404	const char *mname;	22753	const char *mname;
22405	int err;	22754	int i, err;
22406		22755
22407	if (!prog->gpl_compatible) {	22756	if (!prog->gpl_compatible) {
22408	verbose(env, "struct ops programs must have a GPL compatible license\n");	22757	verbose(env, "struct ops programs must have a GPL compatible license\n");
@@ -22450,7 +22799,8 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 41 unmodified lines
22450	return -EINVAL;	22799	return -EINVAL;
22451	}	22800	}
22452		22801
22453	err = bpf_struct_ops_supported(st_ops, __btf_member_bit_offset(t, member) / 8);	22802	member_off = __btf_member_bit_offset(t, member) / 8;
		22803	err = bpf_struct_ops_supported(st_ops, member_off);
22454	if (err) {	22804	if (err) {
22455	verbose(env, "attach to unsupported member %s of struct %s\n",	22805	verbose(env, "attach to unsupported member %s of struct %s\n",
22456	mname, st_ops->name);	22806	mname, st_ops->name);
@@ -22472,17 +22822,32 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 15 unmodified lines
22472	return -EACCES;	22822	return -EACCES;
22473	}	22823	}
22474		22824
22475	/* btf_ctx_access() used this to provide argument type info */	22825	for (i = 0; i < st_ops_desc->arg_info[member_idx].cnt; i++) {
22476	prog->aux->ctx_arg_info =	22826	if (st_ops_desc->arg_info[member_idx].info->refcounted) {
22477	st_ops_desc->arg_info[member_idx].info;	22827	has_refcounted_arg = true;
22478	prog->aux->ctx_arg_info_size =	22828	break;
22479	st_ops_desc->arg_info[member_idx].cnt;	22829	}
		22830	}
		22831
		22832	/* Tail call is not allowed for programs with refcounted arguments since we
		22833	* cannot guarantee that valid refcounted kptrs will be passed to the callee.
		22834	*/
		22835	for (i = 0; i < env->subprog_cnt; i++) {
		22836	if (has_refcounted_arg && env->subprog_info[i].has_tail_call) {
		22837	verbose(env, "program with __ref argument cannot tail call\n");
		22838	return -EINVAL;
		22839	}
		22840	}
		22841
		22842	prog->aux->st_ops = st_ops;
		22843	prog->aux->attach_st_ops_member_off = member_off;
22480		22844
22481	prog->aux->attach_func_proto = func_proto;	22845	prog->aux->attach_func_proto = func_proto;
22482	prog->aux->attach_func_name = mname;	22846	prog->aux->attach_func_name = mname;
22483	env->ops = st_ops->verifier_ops;	22847	env->ops = st_ops->verifier_ops;
22484		22848
22485	return 0;	22849	return bpf_prog_ctx_arg_info_init(prog, st_ops_desc->arg_info[member_idx].info,
		22850	st_ops_desc->arg_info[member_idx].cnt);
22486	}	22851	}
22487	#define SECURITY_PREFIX "security_"	22852	#define SECURITY_PREFIX "security_"
22488		22853
@@ -22558,6 +22923,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 69 unmodified lines
22558	if (tgt_prog) {	22923	if (tgt_prog) {
22559	struct bpf_prog_aux *aux = tgt_prog->aux;	22924	struct bpf_prog_aux *aux = tgt_prog->aux;
22560	bool tgt_changes_pkt_data;	22925	bool tgt_changes_pkt_data;
		22926	bool tgt_might_sleep;
22561		22927
22562	if (bpf_prog_is_dev_bound(prog->aux) &&	22928	if (bpf_prog_is_dev_bound(prog->aux) &&
22563	!bpf_prog_dev_bound_match(prog, tgt_prog)) {	22929	!bpf_prog_dev_bound_match(prog, tgt_prog)) {
@@ -22600,6 +22966,15 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, · 36 unmodified lines
22600	"Extension program changes packet data, while original does not\n");	22966	"Extension program changes packet data, while original does not\n");
22601	return -EINVAL;	22967	return -EINVAL;
22602	}	22968	}
		22969
		22970	tgt_might_sleep = aux->func
		22971	? aux->func[subprog]->aux->might_sleep
		22972	: aux->might_sleep;
		22973	if (prog->aux->might_sleep && !tgt_might_sleep) {
		22974	bpf_log(log,
		22975	"Extension program may sleep, while original does not\n");
		22976	return -EINVAL;
		22977	}
22603	}	22978	}
22604	if (!tgt_prog->jited) {	22979	if (!tgt_prog->jited) {
22605	bpf_log(log, "Can attach to only JITed progs\n");	22980	bpf_log(log, "Can attach to only JITed progs\n");
@@ -22856,6 +23231,33 @@ BTF_ID(func, __rcu_read_unlock) · 250 unmodified lines
22856	#endif	23231	#endif
22857	BTF_SET_END(btf_id_deny)	23232	BTF_SET_END(btf_id_deny)
22858		23233
		23234	/* fexit and fmod_ret can't be used to attach to __noreturn functions.
		23235	* Currently, we must manually list all __noreturn functions here. Once a more
		23236	* robust solution is implemented, this workaround can be removed.
		23237	*/
		23238	BTF_SET_START(noreturn_deny)
		23239	#ifdef CONFIG_IA32_EMULATION
		23240	BTF_ID(func, __ia32_sys_exit)
		23241	BTF_ID(func, __ia32_sys_exit_group)
		23242	#endif
		23243	#ifdef CONFIG_KUNIT
		23244	BTF_ID(func, __kunit_abort)
		23245	BTF_ID(func, kunit_try_catch_throw)
		23246	#endif
		23247	#ifdef CONFIG_MODULES
		23248	BTF_ID(func, __module_put_and_kthread_exit)
		23249	#endif
		23250	#ifdef CONFIG_X86_64
		23251	BTF_ID(func, __x64_sys_exit)
		23252	BTF_ID(func, __x64_sys_exit_group)
		23253	#endif
		23254	BTF_ID(func, do_exit)
		23255	BTF_ID(func, do_group_exit)
		23256	BTF_ID(func, kthread_complete_and_exit)
		23257	BTF_ID(func, kthread_exit)
		23258	BTF_ID(func, make_task_dead)
		23259	BTF_SET_END(noreturn_deny)
		23260
22859	static bool can_be_sleepable(struct bpf_prog *prog)	23261	static bool can_be_sleepable(struct bpf_prog *prog)
22860	{	23262	{
22861	if (prog->type == BPF_PROG_TYPE_TRACING) {	23263	if (prog->type == BPF_PROG_TYPE_TRACING) {
@@ -22932,9 +23334,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 70 unmodified lines
22932	prog->aux->attach_btf_trace = true;	23334	prog->aux->attach_btf_trace = true;
22933	return 0;	23335	return 0;
22934	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {	23336	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
22935	if (!bpf_iter_prog_supported(prog))	23337	return bpf_iter_prog_supported(prog);
22936	return -EINVAL;
22937	return 0;
22938	}	23338	}
22939		23339
22940	if (prog->type == BPF_PROG_TYPE_LSM) {	23340	if (prog->type == BPF_PROG_TYPE_LSM) {
@@ -22944,6 +23344,11 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 3 unmodified lines
22944	} else if (prog->type == BPF_PROG_TYPE_TRACING &&	23344	} else if (prog->type == BPF_PROG_TYPE_TRACING &&
22945	btf_id_set_contains(&btf_id_deny, btf_id)) {	23345	btf_id_set_contains(&btf_id_deny, btf_id)) {
22946	return -EINVAL;	23346	return -EINVAL;
		23347	} else if ((prog->expected_attach_type == BPF_TRACE_FEXIT \|\|
		23348	prog->expected_attach_type == BPF_MODIFY_RETURN) &&
		23349	btf_id_set_contains(&noreturn_deny, btf_id)) {
		23350	verbose(env, "Attaching fexit/fmod_ret to __noreturn functions is rejected.\n");
		23351	return -EINVAL;
22947	}	23352	}
22948		23353
22949	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);	23354	key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
@@ -23036,6 +23441,302 @@ static int process_fd_array(struct bpf_verifier_env env, union bpf_attr attr, · 86 unmodified lines
23036	return 0;	23441	return 0;
23037	}	23442	}
23038		23443
		23444	static bool can_fallthrough(struct bpf_insn *insn)
		23445	{
		23446	u8 class = BPF_CLASS(insn->code);
		23447	u8 opcode = BPF_OP(insn->code);
		23448
		23449	if (class != BPF_JMP && class != BPF_JMP32)
		23450	return true;
		23451
		23452	if (opcode == BPF_EXIT \|\| opcode == BPF_JA)
		23453	return false;
		23454
		23455	return true;
		23456	}
		23457
		23458	static bool can_jump(struct bpf_insn *insn)
		23459	{
		23460	u8 class = BPF_CLASS(insn->code);
		23461	u8 opcode = BPF_OP(insn->code);
		23462
		23463	if (class != BPF_JMP && class != BPF_JMP32)
		23464	return false;
		23465
		23466	switch (opcode) {
		23467	case BPF_JA:
		23468	case BPF_JEQ:
		23469	case BPF_JNE:
		23470	case BPF_JLT:
		23471	case BPF_JLE:
		23472	case BPF_JGT:
		23473	case BPF_JGE:
		23474	case BPF_JSGT:
		23475	case BPF_JSGE:
		23476	case BPF_JSLT:
		23477	case BPF_JSLE:
		23478	case BPF_JCOND:
		23479	return true;
		23480	}
		23481
		23482	return false;
		23483	}
		23484
		23485	static int insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2])
		23486	{
		23487	struct bpf_insn *insn = &prog->insnsi[idx];
		23488	int i = 0, insn_sz;
		23489	u32 dst;
		23490
		23491	insn_sz = bpf_is_ldimm64(insn) ? 2 : 1;
		23492	if (can_fallthrough(insn) && idx + 1 < prog->len)
		23493	succ[i++] = idx + insn_sz;
		23494
		23495	if (can_jump(insn)) {
		23496	dst = idx + jmp_offset(insn) + 1;
		23497	if (i == 0 \|\| succ[0] != dst)
		23498	succ[i++] = dst;
		23499	}
		23500
		23501	return i;
		23502	}
		23503
		23504	/* Each field is a register bitmask */
		23505	struct insn_live_regs {
		23506	u16 use; /* registers read by instruction */
		23507	u16 def; /* registers written by instruction */
		23508	u16 in; /* registers that may be alive before instruction */
		23509	u16 out; /* registers that may be alive after instruction */
		23510	};
		23511
		23512	/* Bitmask with 1s for all caller saved registers */
		23513	#define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1)
		23514
		23515	/* Compute info->{use,def} fields for the instruction */
		23516	static void compute_insn_live_regs(struct bpf_verifier_env *env,
		23517	struct bpf_insn *insn,
		23518	struct insn_live_regs *info)
		23519	{
		23520	struct call_summary cs;
		23521	u8 class = BPF_CLASS(insn->code);
		23522	u8 code = BPF_OP(insn->code);
		23523	u8 mode = BPF_MODE(insn->code);
		23524	u16 src = BIT(insn->src_reg);
		23525	u16 dst = BIT(insn->dst_reg);
		23526	u16 r0 = BIT(0);
		23527	u16 def = 0;
		23528	u16 use = 0xffff;
		23529
		23530	switch (class) {
		23531	case BPF_LD:
		23532	switch (mode) {
		23533	case BPF_IMM:
		23534	if (BPF_SIZE(insn->code) == BPF_DW) {
		23535	def = dst;
		23536	use = 0;
		23537	}
		23538	break;
		23539	case BPF_LD \| BPF_ABS:
		23540	case BPF_LD \| BPF_IND:
		23541	/* stick with defaults */
		23542	break;
		23543	}
		23544	break;
		23545	case BPF_LDX:
		23546	switch (mode) {
		23547	case BPF_MEM:
		23548	case BPF_MEMSX:
		23549	def = dst;
		23550	use = src;
		23551	break;
		23552	}
		23553	break;
		23554	case BPF_ST:
		23555	switch (mode) {
		23556	case BPF_MEM:
		23557	def = 0;
		23558	use = dst;
		23559	break;
		23560	}
		23561	break;
		23562	case BPF_STX:
		23563	switch (mode) {
		23564	case BPF_MEM:
		23565	def = 0;
		23566	use = dst \| src;
		23567	break;
		23568	case BPF_ATOMIC:
		23569	switch (insn->imm) {
		23570	case BPF_CMPXCHG:
		23571	use = r0 \| dst \| src;
		23572	def = r0;
		23573	break;
		23574	case BPF_LOAD_ACQ:
		23575	def = dst;
		23576	use = src;
		23577	break;
		23578	case BPF_STORE_REL:
		23579	def = 0;
		23580	use = dst \| src;
		23581	break;
		23582	default:
		23583	use = dst \| src;
		23584	if (insn->imm & BPF_FETCH)
		23585	def = src;
		23586	else
		23587	def = 0;
		23588	}
		23589	break;
		23590	}
		23591	break;
		23592	case BPF_ALU:
		23593	case BPF_ALU64:
		23594	switch (code) {
		23595	case BPF_END:
		23596	use = dst;
		23597	def = dst;
		23598	break;
		23599	case BPF_MOV:
		23600	def = dst;
		23601	if (BPF_SRC(insn->code) == BPF_K)
		23602	use = 0;
		23603	else
		23604	use = src;
		23605	break;
		23606	default:
		23607	def = dst;
		23608	if (BPF_SRC(insn->code) == BPF_K)
		23609	use = dst;
		23610	else
		23611	use = dst \| src;
		23612	}
		23613	break;
		23614	case BPF_JMP:
		23615	case BPF_JMP32:
		23616	switch (code) {
		23617	case BPF_JA:
		23618	case BPF_JCOND:
		23619	def = 0;
		23620	use = 0;
		23621	break;
		23622	case BPF_EXIT:
		23623	def = 0;
		23624	use = r0;
		23625	break;
		23626	case BPF_CALL:
		23627	def = ALL_CALLER_SAVED_REGS;
		23628	use = def & ~BIT(BPF_REG_0);
		23629	if (get_call_summary(env, insn, &cs))
		23630	use = GENMASK(cs.num_params, 1);
		23631	break;
		23632	default:
		23633	def = 0;
		23634	if (BPF_SRC(insn->code) == BPF_K)
		23635	use = dst;
		23636	else
		23637	use = dst \| src;
		23638	}
		23639	break;
		23640	}
		23641
		23642	info->def = def;
		23643	info->use = use;
		23644	}
		23645
		23646	/* Compute may-live registers after each instruction in the program.
		23647	* The register is live after the instruction I if it is read by some
		23648	* instruction S following I during program execution and is not
		23649	* overwritten between I and S.
		23650	*
		23651	* Store result in env->insn_aux_data[i].live_regs.
		23652	*/
		23653	static int compute_live_registers(struct bpf_verifier_env *env)
		23654	{
		23655	struct bpf_insn_aux_data *insn_aux = env->insn_aux_data;
		23656	struct bpf_insn *insns = env->prog->insnsi;
		23657	struct insn_live_regs *state;
		23658	int insn_cnt = env->prog->len;
		23659	int err = 0, i, j;
		23660	bool changed;
		23661
		23662	/* Use the following algorithm:
		23663	* - define the following:
		23664	* - I.use : a set of all registers read by instruction I;
		23665	* - I.def : a set of all registers written by instruction I;
		23666	* - I.in : a set of all registers that may be alive before I execution;
		23667	* - I.out : a set of all registers that may be alive after I execution;
		23668	* - insn_successors(I): a set of instructions S that might immediately
		23669	* follow I for some program execution;
		23670	* - associate separate empty sets 'I.in' and 'I.out' with each instruction;
		23671	* - visit each instruction in a postorder and update
		23672	* state[i].in, state[i].out as follows:
		23673	*
		23674	* state[i].out = U [state[s].in for S in insn_successors(i)]
		23675	* state[i].in = (state[i].out / state[i].def) U state[i].use
		23676	*
		23677	* (where U stands for set union, / stands for set difference)
		23678	* - repeat the computation while {in,out} fields changes for
		23679	* any instruction.
		23680	*/
		23681	state = kvcalloc(insn_cnt, sizeof(*state), GFP_KERNEL);
		23682	if (!state) {
		23683	err = -ENOMEM;
		23684	goto out;
		23685	}
		23686
		23687	for (i = 0; i < insn_cnt; ++i)
		23688	compute_insn_live_regs(env, &insns[i], &state[i]);
		23689
		23690	changed = true;
		23691	while (changed) {
		23692	changed = false;
		23693	for (i = 0; i < env->cfg.cur_postorder; ++i) {
		23694	int insn_idx = env->cfg.insn_postorder[i];
		23695	struct insn_live_regs *live = &state[insn_idx];
		23696	int succ_num;
		23697	u32 succ[2];
		23698	u16 new_out = 0;
		23699	u16 new_in = 0;
		23700
		23701	succ_num = insn_successors(env->prog, insn_idx, succ);
		23702	for (int s = 0; s < succ_num; ++s)
		23703	new_out \|= state[succ[s]].in;
		23704	new_in = (new_out & ~live->def) \| live->use;
		23705	if (new_out != live->out \|\| new_in != live->in) {
		23706	live->in = new_in;
		23707	live->out = new_out;
		23708	changed = true;
		23709	}
		23710	}
		23711	}
		23712
		23713	for (i = 0; i < insn_cnt; ++i)
		23714	insn_aux[i].live_regs_before = state[i].in;
		23715
		23716	if (env->log.level & BPF_LOG_LEVEL2) {
		23717	verbose(env, "Live regs before insn:\n");
		23718	for (i = 0; i < insn_cnt; ++i) {
		23719	verbose(env, "%3d: ", i);
		23720	for (j = BPF_REG_0; j < BPF_REG_10; ++j)
		23721	if (insn_aux[i].live_regs_before & BIT(j))
		23722	verbose(env, "%d", j);
		23723	else
		23724	verbose(env, ".");
		23725	verbose(env, " ");
		23726	verbose_insn(env, &insns[i]);
		23727	if (bpf_is_ldimm64(&insns[i]))
		23728	i++;
		23729	}
		23730	}
		23731
		23732	out:
		23733	kvfree(state);
		23734	kvfree(env->cfg.insn_postorder);
		23735	env->cfg.insn_postorder = NULL;
		23736	env->cfg.cur_postorder = 0;
		23737	return err;
		23738	}
		23739
23039	int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr, __u32 uattr_size)	23740	int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr, __u32 uattr_size)
23040	{	23741	{
23041	u64 start_time = ktime_get_ns();	23742	u64 start_time = ktime_get_ns();
@@ -23113,12 +23814,16 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr, __u3 · 71 unmodified lines
23113	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;	23814	env->test_reg_invariants = attr->prog_flags & BPF_F_TEST_REG_INVARIANTS;
23114		23815
23115	env->explored_states = kvcalloc(state_htab_size(env),	23816	env->explored_states = kvcalloc(state_htab_size(env),
23116	sizeof(struct bpf_verifier_state_list *),	23817	sizeof(struct list_head),
23117	GFP_USER);	23818	GFP_USER);
23118	ret = -ENOMEM;	23819	ret = -ENOMEM;
23119	if (!env->explored_states)	23820	if (!env->explored_states)
23120	goto skip_full_check;	23821	goto skip_full_check;
23121		23822
		23823	for (i = 0; i < state_htab_size(env); i++)
		23824	INIT_LIST_HEAD(&env->explored_states[i]);
		23825	INIT_LIST_HEAD(&env->free_list);
		23826
23122	ret = check_btf_info_early(env, attr, uattr);	23827	ret = check_btf_info_early(env, attr, uattr);
23123	if (ret < 0)	23828	if (ret < 0)
23124	goto skip_full_check;	23829	goto skip_full_check;
@@ -23153,6 +23858,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, bpfptr_t uattr, __u3 · 28 unmodified lines
23153	if (ret)	23858	if (ret)
23154	goto skip_full_check;	23859	goto skip_full_check;
23155		23860
		23861	ret = compute_live_registers(env);
		23862	if (ret < 0)
		23863	goto skip_full_check;
		23864
23156	ret = mark_fastcall_patterns(env);	23865	ret = mark_fastcall_patterns(env);
23157	if (ret < 0)	23866	if (ret < 0)
23158	goto skip_full_check;	23867	goto skip_full_check;
@@ -23291,6 +24000,7 @@ err_unlock: · 132 unmodified lines
23291	vfree(env->insn_aux_data);	24000	vfree(env->insn_aux_data);
23292	kvfree(env->insn_hist);	24001	kvfree(env->insn_hist);
23293	err_free_env:	24002	err_free_env:
		24003	kvfree(env->cfg.insn_postorder);
23294	kvfree(env);	24004	kvfree(env);
23295	return ret;	24005	return ret;
23296	}	24006	}

kernel/trace/bpf_trace.c+10−4

 	.arg2_type	= ARG_CONST_SIZE,
 };
-static void __set_printk_clr_event(void)
+static void __set_printk_clr_event(struct work_struct *work)
 {
 	/*
 	 * This program might be calling bpf_trace_printk,
 	if (trace_set_clr_event("bpf_trace", "bpf_trace_printk", 1))
 		pr_warn_ratelimited("could not enable bpf_trace_printk events");
 }
+static DECLARE_WORK(set_printk_work, __set_printk_clr_event);
 const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
 {
-	__set_printk_clr_event();
+	schedule_work(&set_printk_work);
 	return &bpf_trace_printk_proto;
 }
 const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void)
 {
-	__set_printk_clr_event();
+	schedule_work(&set_printk_work);
 	return &bpf_trace_vprintk_proto;
 }
 	.arg4_type	= ARG_CONST_SIZE,
 };
+const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void)
+{
+	return &bpf_perf_event_read_value_proto;
+}
 static __always_inline u64
 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
 			u64 flags, struct perf_raw_record *raw,
 	if (unlikely(is_global_init(task)))
 		return -EPERM;
-	if (!preemptible()) {
+	if (preempt_count() != 0 || irqs_disabled()) {
 		/* Do an early check on signal validity. Otherwise,
 		 * the error is lost in deferred irq_work.
 		 */

mm/memory.c+118−0

@@ -6807,6 +6807,124 @@ int access_process_vm(struct task_struct *tsk, unsigned long addr, · 6806 unmodified lines
6807	}	6807	}
6808	EXPORT_SYMBOL_GPL(access_process_vm);	6808	EXPORT_SYMBOL_GPL(access_process_vm);
6809		6809
		6810	#ifdef CONFIG_BPF_SYSCALL
		6811	/*
		6812	* Copy a string from another process's address space as given in mm.
		6813	* If there is any error return -EFAULT.
		6814	*/
		6815	static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
		6816	void *buf, int len, unsigned int gup_flags)
		6817	{
		6818	void *old_buf = buf;
		6819	int err = 0;
		6820
		6821	(char )buf = '\0';
		6822
		6823	if (mmap_read_lock_killable(mm))
		6824	return -EFAULT;
		6825
		6826	addr = untagged_addr_remote(mm, addr);
		6827
		6828	/* Avoid triggering the temporary warning in __get_user_pages */
		6829	if (!vma_lookup(mm, addr)) {
		6830	err = -EFAULT;
		6831	goto out;
		6832	}
		6833
		6834	while (len) {
		6835	int bytes, offset, retval;
		6836	void *maddr;
		6837	struct page *page;
		6838	struct vm_area_struct *vma = NULL;
		6839
		6840	page = get_user_page_vma_remote(mm, addr, gup_flags, &vma);
		6841	if (IS_ERR(page)) {
		6842	/*
		6843	* Treat as a total failure for now until we decide how
		6844	* to handle the CONFIG_HAVE_IOREMAP_PROT case and
		6845	* stack expansion.
		6846	*/
		6847	(char )buf = '\0';
		6848	err = -EFAULT;
		6849	goto out;
		6850	}
		6851
		6852	bytes = len;
		6853	offset = addr & (PAGE_SIZE - 1);
		6854	if (bytes > PAGE_SIZE - offset)
		6855	bytes = PAGE_SIZE - offset;
		6856
		6857	maddr = kmap_local_page(page);
		6858	retval = strscpy(buf, maddr + offset, bytes);
		6859	if (retval >= 0) {
		6860	/* Found the end of the string */
		6861	buf += retval;
		6862	unmap_and_put_page(page, maddr);
		6863	break;
		6864	}
		6865
		6866	buf += bytes - 1;
		6867	/*
		6868	* Because strscpy always NUL terminates we need to
		6869	* copy the last byte in the page if we are going to
		6870	* load more pages
		6871	*/
		6872	if (bytes != len) {
		6873	addr += bytes - 1;
		6874	copy_from_user_page(vma, page, addr, buf, maddr + (PAGE_SIZE - 1), 1);
		6875	buf += 1;
		6876	addr += 1;
		6877	}
		6878	len -= bytes;
		6879
		6880	unmap_and_put_page(page, maddr);
		6881	}
		6882
		6883	out:
		6884	mmap_read_unlock(mm);
		6885	if (err)
		6886	return err;
		6887	return buf - old_buf;
		6888	}
		6889
		6890	/**
		6891	* copy_remote_vm_str - copy a string from another process's address space.
		6892	* @tsk: the task of the target address space
		6893	* @addr: start address to read from
		6894	* @buf: destination buffer
		6895	* @len: number of bytes to copy
		6896	* @gup_flags: flags modifying lookup behaviour
		6897	*
		6898	* The caller must hold a reference on @mm.
		6899	*
		6900	* Return: number of bytes copied from @addr (source) to @buf (destination);
		6901	* not including the trailing NUL. Always guaranteed to leave NUL-terminated
		6902	* buffer. On any error, return -EFAULT.
		6903	*/
		6904	int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
		6905	void *buf, int len, unsigned int gup_flags)
		6906	{
		6907	struct mm_struct *mm;
		6908	int ret;
		6909
		6910	if (unlikely(len == 0))
		6911	return 0;
		6912
		6913	mm = get_task_mm(tsk);
		6914	if (!mm) {
		6915	(char )buf = '\0';
		6916	return -EFAULT;
		6917	}
		6918
		6919	ret = __copy_remote_vm_str(mm, addr, buf, len, gup_flags);
		6920
		6921	mmput(mm);
		6922
		6923	return ret;
		6924	}
		6925	EXPORT_SYMBOL_GPL(copy_remote_vm_str);
		6926	#endif /* CONFIG_BPF_SYSCALL */
		6927
6810	/*	6928	/*
6811	* Print the name of a VMA.	6929	* Print the name of a VMA.
6812	*/	6930	*/

mm/nommu.c+79−0

@@ -1714,6 +1714,85 @@ int access_process_vm(struct task_struct tsk, unsigned long addr, void buf, in · 1713 unmodified lines
1714	}	1714	}
1715	EXPORT_SYMBOL_GPL(access_process_vm);	1715	EXPORT_SYMBOL_GPL(access_process_vm);
1716		1716
		1717	#ifdef CONFIG_BPF_SYSCALL
		1718	/*
		1719	* Copy a string from another process's address space as given in mm.
		1720	* If there is any error return -EFAULT.
		1721	*/
		1722	static int __copy_remote_vm_str(struct mm_struct *mm, unsigned long addr,
		1723	void *buf, int len)
		1724	{
		1725	unsigned long addr_end;
		1726	struct vm_area_struct *vma;
		1727	int ret = -EFAULT;
		1728
		1729	(char )buf = '\0';
		1730
		1731	if (mmap_read_lock_killable(mm))
		1732	return ret;
		1733
		1734	/* the access must start within one of the target process's mappings */
		1735	vma = find_vma(mm, addr);
		1736	if (!vma)
		1737	goto out;
		1738
		1739	if (check_add_overflow(addr, len, &addr_end))
		1740	goto out;
		1741
		1742	/* don't overrun this mapping */
		1743	if (addr_end > vma->vm_end)
		1744	len = vma->vm_end - addr;
		1745
		1746	/* only read mappings where it is permitted */
		1747	if (vma->vm_flags & VM_MAYREAD) {
		1748	ret = strscpy(buf, (char *)addr, len);
		1749	if (ret < 0)
		1750	ret = len - 1;
		1751	}
		1752
		1753	out:
		1754	mmap_read_unlock(mm);
		1755	return ret;
		1756	}
		1757
		1758	/**
		1759	* copy_remote_vm_str - copy a string from another process's address space.
		1760	* @tsk: the task of the target address space
		1761	* @addr: start address to read from
		1762	* @buf: destination buffer
		1763	* @len: number of bytes to copy
		1764	* @gup_flags: flags modifying lookup behaviour (unused)
		1765	*
		1766	* The caller must hold a reference on @mm.
		1767	*
		1768	* Return: number of bytes copied from @addr (source) to @buf (destination);
		1769	* not including the trailing NUL. Always guaranteed to leave NUL-terminated
		1770	* buffer. On any error, return -EFAULT.
		1771	*/
		1772	int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr,
		1773	void *buf, int len, unsigned int gup_flags)
		1774	{
		1775	struct mm_struct *mm;
		1776	int ret;
		1777
		1778	if (unlikely(len == 0))
		1779	return 0;
		1780
		1781	mm = get_task_mm(tsk);
		1782	if (!mm) {
		1783	(char )buf = '\0';
		1784	return -EFAULT;
		1785	}
		1786
		1787	ret = __copy_remote_vm_str(mm, addr, buf, len);
		1788
		1789	mmput(mm);
		1790
		1791	return ret;
		1792	}
		1793	EXPORT_SYMBOL_GPL(copy_remote_vm_str);
		1794	#endif /* CONFIG_BPF_SYSCALL */
		1795
1717	/**	1796	/**
1718	* nommu_shrink_inode_mappings - Shrink the shared mappings on an inode	1797	* nommu_shrink_inode_mappings - Shrink the shared mappings on an inode
1719	* @inode: The inode to check	1798	* @inode: The inode to check

net/core/filter.c+4−2

 		return &bpf_skb_load_bytes_relative_proto;
 	case BPF_FUNC_get_socket_cookie:
 		return &bpf_get_socket_cookie_proto;
+	case BPF_FUNC_get_netns_cookie:
+		return &bpf_get_netns_cookie_proto;
 	case BPF_FUNC_get_socket_uid:
 		return &bpf_get_socket_uid_proto;
 	case BPF_FUNC_perf_event_output:
 	case offsetof(struct __sk_buff, queue_mapping):
 		if (type == BPF_WRITE) {
-			u32 off = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
+			u32 offset = bpf_target_off(struct sk_buff, queue_mapping, 2, target_size);
 			if (BPF_CLASS(si->code) == BPF_ST && si->imm >= NO_QUEUE_MAPPING) {
 				*insn++ = BPF_JMP_A(0); /* noop */
 			if (BPF_CLASS(si->code) == BPF_STX)
 				*insn++ = BPF_JMP_IMM(BPF_JGE, si->src_reg, NO_QUEUE_MAPPING, 1);
-			*insn++ = BPF_EMIT_STORE(BPF_H, si, off);
+			*insn++ = BPF_EMIT_STORE(BPF_H, si, offset);
 		} else {
 			*insn++ = BPF_LDX_MEM(BPF_H, si->dst_reg, si->src_reg,
 					      bpf_target_off(struct sk_buff,

samples/bpf/Makefile+1−1

@@ -307,7 +307,7 @@ $(obj)/$(TRACE_HELPERS): TPROGS_CFLAGS := $(TPROGS_CFLAGS) -D__must_check= · 306 unmodified lines
307		307
308	VMLINUX_BTF_PATHS ?= $(abspath $(if $(O),$(O)/vmlinux)) \	308	VMLINUX_BTF_PATHS ?= $(abspath $(if $(O),$(O)/vmlinux)) \
309	$(abspath $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux)) \	309	$(abspath $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux)) \
310	$(abspath ./vmlinux)	310	$(abspath $(objtree)/vmlinux)
311	VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))	311	VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
312		312
313	$(obj)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL)	313	$(obj)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL)

scripts/Makefile.btf+1−1

@@ -24,7 +24,7 @@ else · 23 unmodified lines
24	pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs	24	pahole-flags-$(call test-ge, $(pahole-ver), 126) = -j$(JOBS) --btf_features=encode_force,var,float,enum64,decl_tag,type_tag,optimized_func,consistent_func,decl_tag_kfuncs
25		25
26	ifneq ($(KBUILD_EXTMOD),)	26	ifneq ($(KBUILD_EXTMOD),)
27	module-pahole-flags-$(call test-ge, $(pahole-ver), 126) += --btf_features=distilled_base	27	module-pahole-flags-$(call test-ge, $(pahole-ver), 128) += --btf_features=distilled_base
28	endif	28	endif
29		29
30	endif	30	endif

security/security.c+9−6

  * @cmd: command
  * @attr: bpf attribute
  * @size: size
+ * @kernel: whether or not call originated from kernel
  *
  * Do a initial check for all bpf syscalls after the attribute is copied into
  * the kernel. The actual security module can implement their own rules to
  *
  * Return: Returns 0 if permission is granted.
  */
-int security_bpf(int cmd, union bpf_attr *attr, unsigned int size)
+int security_bpf(int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
 {
-	return call_int_hook(bpf, cmd, attr, size);
+	return call_int_hook(bpf, cmd, attr, size, kernel);
 }
 /**
  * @map: BPF map object
  * @attr: BPF syscall attributes used to create BPF map
  * @token: BPF token used to grant user access
+ * @kernel: whether or not call originated from kernel
  *
  * Do a check when the kernel creates a new BPF map. This is also the
  * point where LSM blob is allocated for LSMs that need them.
  * Return: Returns 0 on success, error on failure.
  */
 int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
-			    struct bpf_token *token)
+			    struct bpf_token *token, bool kernel)
 {
-	return call_int_hook(bpf_map_create, map, attr, token);
+	return call_int_hook(bpf_map_create, map, attr, token, kernel);
 }
 /**
  * @prog: BPF program object
  * @attr: BPF syscall attributes used to create BPF program
  * @token: BPF token used to grant user access to BPF subsystem
+ * @kernel: whether or not call originated from kernel
  *
  * Perform an access control check when the kernel loads a BPF program and
  * allocates associated BPF program object. This hook is also responsible for
  * Return: Returns 0 on success, error on failure.
  */
 int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
-			   struct bpf_token *token)
+			   struct bpf_token *token, bool kernel)
 {
-	return call_int_hook(bpf_prog_load, prog, attr, token);
+	return call_int_hook(bpf_prog_load, prog, attr, token, kernel);
 }
 /**

security/selinux/hooks.c+3−3

@@ -6907,7 +6907,7 @@ static int selinux_ib_alloc_security(void *ib_sec) · 6906 unmodified lines
6907		6907
6908	#ifdef CONFIG_BPF_SYSCALL	6908	#ifdef CONFIG_BPF_SYSCALL
6909	static int selinux_bpf(int cmd, union bpf_attr *attr,	6909	static int selinux_bpf(int cmd, union bpf_attr *attr,
6910	unsigned int size)	6910	unsigned int size, bool kernel)
6911	{	6911	{
6912	u32 sid = current_sid();	6912	u32 sid = current_sid();
6913	int ret;	6913	int ret;
@@ -6994,7 +6994,7 @@ static int selinux_bpf_prog(struct bpf_prog *prog) · 80 unmodified lines
6994	}	6994	}
6995		6995
6996	static int selinux_bpf_map_create(struct bpf_map map, union bpf_attr attr,	6996	static int selinux_bpf_map_create(struct bpf_map map, union bpf_attr attr,
6997	struct bpf_token *token)	6997	struct bpf_token *token, bool kernel)
6998	{	6998	{
6999	struct bpf_security_struct *bpfsec;	6999	struct bpf_security_struct *bpfsec;
7000		7000
@@ -7017,7 +7017,7 @@ static void selinux_bpf_map_free(struct bpf_map *map) · 16 unmodified lines
7017	}	7017	}
7018		7018
7019	static int selinux_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,	7019	static int selinux_bpf_prog_load(struct bpf_prog prog, union bpf_attr attr,
7020	struct bpf_token *token)	7020	struct bpf_token *token, bool kernel)
7021	{	7021	{
7022	struct bpf_security_struct *bpfsec;	7022	struct bpf_security_struct *bpfsec;
7023		7023

tools/bpf/bpftool/Makefile+6−1

 bash_compdir ?= /usr/share/bash-completion/completions
 CFLAGS += -O2
-CFLAGS += -W -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers
+CFLAGS += -W
+CFLAGS += -Wall
+CFLAGS += -Wextra
+CFLAGS += -Wformat-signedness
+CFLAGS += -Wno-unused-parameter
+CFLAGS += -Wno-missing-field-initializers
 CFLAGS += $(filter-out -Wswitch-enum -Wnested-externs,$(EXTRA_WARNINGS))
 CFLAGS += -DPACKAGE='"bpftool"' -D__EXPORTED_HEADERS__ \
 	-I$(or $(OUTPUT),.) \

tools/bpf/bpftool/btf.c+7−7

@@ -253,7 +253,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, · 252 unmodified lines
253	if (btf_kflag(t))	253	if (btf_kflag(t))
254	printf("\n\t'%s' val=%d", name, v->val);	254	printf("\n\t'%s' val=%d", name, v->val);
255	else	255	else
256	printf("\n\t'%s' val=%u", name, v->val);	256	printf("\n\t'%s' val=%u", name, (__u32)v->val);
257	}	257	}
258	}	258	}
259	if (json_output)	259	if (json_output)
@@ -1022,7 +1022,7 @@ static int do_dump(int argc, char **argv) · 762 unmodified lines
1022	for (i = 0; i < root_type_cnt; i++) {	1022	for (i = 0; i < root_type_cnt; i++) {
1023	if (root_type_ids[i] == root_id) {	1023	if (root_type_ids[i] == root_id) {
1024	err = -EINVAL;	1024	err = -EINVAL;
1025	p_err("duplicate root_id %d supplied", root_id);	1025	p_err("duplicate root_id %u supplied", root_id);
1026	goto done;	1026	goto done;
1027	}	1027	}
1028	}	1028	}
@@ -1132,7 +1132,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type, · 103 unmodified lines
1132	break;	1132	break;
1133	default:	1133	default:
1134	err = -1;	1134	err = -1;
1135	p_err("unexpected object type: %d", type);	1135	p_err("unexpected object type: %u", type);
1136	goto err_free;	1136	goto err_free;
1137	}	1137	}
1138	if (err) {	1138	if (err) {
@@ -1155,7 +1155,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type, · 16 unmodified lines
1155	break;	1155	break;
1156	default:	1156	default:
1157	err = -1;	1157	err = -1;
1158	p_err("unexpected object type: %d", type);	1158	p_err("unexpected object type: %u", type);
1159	goto err_free;	1159	goto err_free;
1160	}	1160	}
1161	if (fd < 0) {	1161	if (fd < 0) {
@@ -1188,7 +1188,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type, · 26 unmodified lines
1188	break;	1188	break;
1189	default:	1189	default:
1190	err = -1;	1190	err = -1;
1191	p_err("unexpected object type: %d", type);	1191	p_err("unexpected object type: %u", type);
1192	goto err_free;	1192	goto err_free;
1193	}	1193	}
1194	if (!btf_id)	1194	if (!btf_id)
@@ -1254,12 +1254,12 @@ show_btf_plain(struct bpf_btf_info *info, int fd, · 59 unmodified lines
1254		1254
1255	n = 0;	1255	n = 0;
1256	hashmap__for_each_key_entry(btf_prog_table, entry, info->id) {	1256	hashmap__for_each_key_entry(btf_prog_table, entry, info->id) {
1257	printf("%s%lu", n++ == 0 ? " prog_ids " : ",", entry->value);	1257	printf("%s%lu", n++ == 0 ? " prog_ids " : ",", (unsigned long)entry->value);
1258	}	1258	}
1259		1259
1260	n = 0;	1260	n = 0;
1261	hashmap__for_each_key_entry(btf_map_table, entry, info->id) {	1261	hashmap__for_each_key_entry(btf_map_table, entry, info->id) {
1262	printf("%s%lu", n++ == 0 ? " map_ids " : ",", entry->value);	1262	printf("%s%lu", n++ == 0 ? " map_ids " : ",", (unsigned long)entry->value);
1263	}	1263	}
1264		1264
1265	emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");	1265	emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");

tools/bpf/bpftool/btf_dumper.c+1−1

@@ -653,7 +653,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, · 652 unmodified lines
653	case BTF_KIND_ARRAY:	653	case BTF_KIND_ARRAY:
654	array = (struct btf_array *)(t + 1);	654	array = (struct btf_array *)(t + 1);
655	BTF_PRINT_TYPE(array->type);	655	BTF_PRINT_TYPE(array->type);
656	BTF_PRINT_ARG("[%d]", array->nelems);	656	BTF_PRINT_ARG("[%u]", array->nelems);
657	break;	657	break;
658	case BTF_KIND_PTR:	658	case BTF_KIND_PTR:
659	BTF_PRINT_TYPE(t->type);	659	BTF_PRINT_TYPE(t->type);

tools/bpf/bpftool/cgroup.c+1−1

@@ -191,7 +191,7 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, · 190 unmodified lines
191	if (attach_btf_name)	191	if (attach_btf_name)
192	printf(" %-15s", attach_btf_name);	192	printf(" %-15s", attach_btf_name);
193	else if (info.attach_btf_id)	193	else if (info.attach_btf_id)
194	printf(" attach_btf_obj_id=%d attach_btf_id=%d",	194	printf(" attach_btf_obj_id=%u attach_btf_id=%u",
195	info.attach_btf_obj_id, info.attach_btf_id);	195	info.attach_btf_obj_id, info.attach_btf_id);
196	printf("\n");	196	printf("\n");
197	}	197	}

tools/bpf/bpftool/common.c+4−3

 		p_err("can't read link type: %s", strerror(errno));
 		return -1;
 	}
-	if (n == sizeof(path)) {
+	if (n == sizeof(buf)) {
 		p_err("can't read link type: path too long!");
 		return -1;
 	}
+	buf[n] = '\0';
 	if (strstr(buf, "bpf-map"))
 		return BPF_OBJ_MAP;
 	int vendor_id;
 	if (!ifindex_to_name_ns(ifindex, ns_dev, ns_ino, devname)) {
-		p_err("Can't get net device name for ifindex %d: %s", ifindex,
+		p_err("Can't get net device name for ifindex %u: %s", ifindex,
 		      strerror(errno));
 		return NULL;
 	}
 	/* No NFP support in LLVM, we have no valid triple to return. */
 	default:
 		p_err("Can't get arch name for device vendor id 0x%04x",
-		      vendor_id);
+		      (unsigned int)vendor_id);
 		return NULL;
 	}
 }

tools/bpf/bpftool/gen.c+6−6

@@ -670,7 +670,7 @@ static void codegen_destroy(struct bpf_object obj, const char obj_name) · 669 unmodified lines
670	continue;	670	continue;
671	if (bpf_map__is_internal(map) &&	671	if (bpf_map__is_internal(map) &&
672	(bpf_map__map_flags(map) & BPF_F_MMAPABLE))	672	(bpf_map__map_flags(map) & BPF_F_MMAPABLE))
673	printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zd);\n",	673	printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n",
674	ident, bpf_map_mmap_sz(map));	674	ident, bpf_map_mmap_sz(map));
675	codegen("\	675	codegen("\
676	\n\	676	\n\
@@ -984,7 +984,7 @@ static int walk_st_ops_shadow_vars(struct btf btf, const char ident, · 307 unmodified lines
984		984
985	offset = m->offset / 8;	985	offset = m->offset / 8;
986	if (next_offset < offset)	986	if (next_offset < offset)
987	printf("\t\t\tchar __padding_%d[%d];\n", i, offset - next_offset);	987	printf("\t\t\tchar __padding_%d[%u];\n", i, offset - next_offset);
988		988
989	switch (btf_kind(member_type)) {	989	switch (btf_kind(member_type)) {
990	case BTF_KIND_INT:	990	case BTF_KIND_INT:
@@ -1052,7 +1052,7 @@ static int walk_st_ops_shadow_vars(struct btf btf, const char ident, · 61 unmodified lines
1052	/* Cannot fail since it must be a struct type */	1052	/* Cannot fail since it must be a struct type */
1053	size = btf__resolve_size(btf, map_type_id);	1053	size = btf__resolve_size(btf, map_type_id);
1054	if (next_offset < (__u32)size)	1054	if (next_offset < (__u32)size)
1055	printf("\t\t\tchar __padding_end[%d];\n", size - next_offset);	1055	printf("\t\t\tchar __padding_end[%u];\n", size - next_offset);
1056		1056
1057	out:	1057	out:
1058	btf_dump__free(d);	1058	btf_dump__free(d);
@@ -2095,7 +2095,7 @@ btfgen_mark_type(struct btfgen_info *info, unsigned int type_id, bool follow_poi · 1036 unmodified lines
2095	break;	2095	break;
2096	/* tells if some other type needs to be handled */	2096	/* tells if some other type needs to be handled */
2097	default:	2097	default:
2098	p_err("unsupported kind: %s (%d)", btf_kind_str(btf_type), type_id);	2098	p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id);
2099	return -EINVAL;	2099	return -EINVAL;
2100	}	2100	}
2101		2101
@@ -2147,7 +2147,7 @@ static int btfgen_record_field_relo(struct btfgen_info *info, struct bpf_core_sp · 45 unmodified lines
2147	btf_type = btf__type_by_id(btf, type_id);	2147	btf_type = btf__type_by_id(btf, type_id);
2148	break;	2148	break;
2149	default:	2149	default:
2150	p_err("unsupported kind: %s (%d)",	2150	p_err("unsupported kind: %s (%u)",
2151	btf_kind_str(btf_type), btf_type->type);	2151	btf_kind_str(btf_type), btf_type->type);
2152	return -EINVAL;	2152	return -EINVAL;
2153	}	2153	}
@@ -2246,7 +2246,7 @@ static int btfgen_mark_type_match(struct btfgen_info *info, __u32 type_id, bool · 92 unmodified lines
2246	}	2246	}
2247	/* tells if some other type needs to be handled */	2247	/* tells if some other type needs to be handled */
2248	default:	2248	default:
2249	p_err("unsupported kind: %s (%d)", btf_kind_str(btf_type), type_id);	2249	p_err("unsupported kind: %s (%u)", btf_kind_str(btf_type), type_id);
2250	return -EINVAL;	2250	return -EINVAL;
2251	}	2251	}
2252		2252

tools/bpf/bpftool/jit_disasm.c+2−1

@@ -343,7 +343,8 @@ int disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, · 342 unmodified lines
343	{	343	{
344	const struct bpf_line_info *linfo = NULL;	344	const struct bpf_line_info *linfo = NULL;
345	unsigned int nr_skip = 0;	345	unsigned int nr_skip = 0;
346	int count, i, pc = 0;	346	int count, i;
		347	unsigned int pc = 0;
347	disasm_ctx_t ctx;	348	disasm_ctx_t ctx;
348		349
349	if (!len)	350	if (!len)

tools/bpf/bpftool/link.c+7−7

@@ -107,7 +107,7 @@ static int link_parse_fd(int argc, char **argv) · 106 unmodified lines
107		107
108	fd = bpf_link_get_fd_by_id(id);	108	fd = bpf_link_get_fd_by_id(id);
109	if (fd < 0)	109	if (fd < 0)
110	p_err("failed to get link with ID %d: %s", id, strerror(errno));	110	p_err("failed to get link with ID %u: %s", id, strerror(errno));
111	return fd;	111	return fd;
112	} else if (is_prefix(**argv, "pinned")) {	112	} else if (is_prefix(**argv, "pinned")) {
113	char *path;	113	char *path;
@@ -404,7 +404,7 @@ static char *perf_config_hw_cache_str(__u64 config) · 290 unmodified lines
404	if (hw_cache)	404	if (hw_cache)
405	snprintf(str, PERF_HW_CACHE_LEN, "%s-", hw_cache);	405	snprintf(str, PERF_HW_CACHE_LEN, "%s-", hw_cache);
406	else	406	else
407	snprintf(str, PERF_HW_CACHE_LEN, "%lld-", config & 0xff);	407	snprintf(str, PERF_HW_CACHE_LEN, "%llu-", config & 0xff);
408		408
409	op = perf_event_name(evsel__hw_cache_op, (config >> 8) & 0xff);	409	op = perf_event_name(evsel__hw_cache_op, (config >> 8) & 0xff);
410	if (op)	410	if (op)
@@ -412,7 +412,7 @@ static char *perf_config_hw_cache_str(__u64 config) · 1 unmodified line
412	"%s-", op);	412	"%s-", op);
413	else	413	else
414	snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),	414	snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),
415	"%lld-", (config >> 8) & 0xff);	415	"%llu-", (config >> 8) & 0xff);
416		416
417	result = perf_event_name(evsel__hw_cache_result, config >> 16);	417	result = perf_event_name(evsel__hw_cache_result, config >> 16);
418	if (result)	418	if (result)
@@ -420,7 +420,7 @@ static char *perf_config_hw_cache_str(__u64 config) · 1 unmodified line
420	"%s", result);	420	"%s", result);
421	else	421	else
422	snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),	422	snprintf(str + strlen(str), PERF_HW_CACHE_LEN - strlen(str),
423	"%lld", config >> 16);	423	"%llu", config >> 16);
424	return str;	424	return str;
425	}	425	}
426		426
@@ -623,7 +623,7 @@ static void show_link_ifindex_plain(__u32 ifindex) · 196 unmodified lines
623	else	623	else
624	snprintf(devname, sizeof(devname), "(detached)");	624	snprintf(devname, sizeof(devname), "(detached)");
625	if (ret)	625	if (ret)
626	snprintf(devname, sizeof(devname), "%s(%d)",	626	snprintf(devname, sizeof(devname), "%s(%u)",
627	tmpname, ifindex);	627	tmpname, ifindex);
628	printf("ifindex %s ", devname);	628	printf("ifindex %s ", devname);
629	}	629	}
@@ -699,7 +699,7 @@ void netfilter_dump_plain(const struct bpf_link_info *info) · 69 unmodified lines
699	if (pfname)	699	if (pfname)
700	printf("\n\t%s", pfname);	700	printf("\n\t%s", pfname);
701	else	701	else
702	printf("\n\tpf: %d", pf);	702	printf("\n\tpf: %u", pf);
703		703
704	if (hookname)	704	if (hookname)
705	printf(" %s", hookname);	705	printf(" %s", hookname);
@@ -773,7 +773,7 @@ static void show_uprobe_multi_plain(struct bpf_link_info *info) · 67 unmodified lines
773	printf("func_cnt %u ", info->uprobe_multi.count);	773	printf("func_cnt %u ", info->uprobe_multi.count);
774		774
775	if (info->uprobe_multi.pid)	775	if (info->uprobe_multi.pid)
776	printf("pid %d ", info->uprobe_multi.pid);	776	printf("pid %u ", info->uprobe_multi.pid);
777		777
778	printf("\n\t%-16s %-16s %-16s", "offset", "ref_ctr_offset", "cookies");	778	printf("\n\t%-16s %-16s %-16s", "offset", "ref_ctr_offset", "cookies");
779	for (i = 0; i < info->uprobe_multi.count; i++) {	779	for (i = 0; i < info->uprobe_multi.count; i++) {

tools/bpf/bpftool/main.c+4−4

@@ -152,7 +152,7 @@ static int do_version(int argc, char **argv) · 151 unmodified lines
152	BPFTOOL_MINOR_VERSION, BPFTOOL_PATCH_VERSION);	152	BPFTOOL_MINOR_VERSION, BPFTOOL_PATCH_VERSION);
153	#endif	153	#endif
154	jsonw_name(json_wtr, "libbpf_version");	154	jsonw_name(json_wtr, "libbpf_version");
155	jsonw_printf(json_wtr, "\"%d.%d\"",	155	jsonw_printf(json_wtr, "\"%u.%u\"",
156	libbpf_major_version(), libbpf_minor_version());	156	libbpf_major_version(), libbpf_minor_version());
157		157
158	jsonw_name(json_wtr, "features");	158	jsonw_name(json_wtr, "features");
@@ -370,7 +370,7 @@ static int do_batch(int argc, char **argv) · 211 unmodified lines
370	while ((cp = strstr(buf, "\\\n")) != NULL) {	370	while ((cp = strstr(buf, "\\\n")) != NULL) {
371	if (!fgets(contline, sizeof(contline), fp) \|\|	371	if (!fgets(contline, sizeof(contline), fp) \|\|
372	strlen(contline) == 0) {	372	strlen(contline) == 0) {
373	p_err("missing continuation line on command %d",	373	p_err("missing continuation line on command %u",
374	lines);	374	lines);
375	err = -1;	375	err = -1;
376	goto err_close;	376	goto err_close;
@@ -381,7 +381,7 @@ static int do_batch(int argc, char **argv) · 4 unmodified lines
381	*cp = '\0';	381	*cp = '\0';
382		382
383	if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) {	383	if (strlen(buf) + strlen(contline) + 1 > sizeof(buf)) {
384	p_err("command %d is too long", lines);	384	p_err("command %u is too long", lines);
385	err = -1;	385	err = -1;
386	goto err_close;	386	goto err_close;
387	}	387	}
@@ -423,7 +423,7 @@ static int do_batch(int argc, char **argv) · 35 unmodified lines
423	err = -1;	423	err = -1;
424	} else {	424	} else {
425	if (!json_output)	425	if (!json_output)
426	printf("processed %d commands\n", lines);	426	printf("processed %u commands\n", lines);
427	}	427	}
428	err_close:	428	err_close:
429	if (fp != stdin)	429	if (fp != stdin)

tools/bpf/bpftool/map.c+9−5

 		}
 		if (info->value_size) {
 			for (i = 0; i < n; i++) {
-				printf("value (CPU %02d):%c",
+				printf("value (CPU %02u):%c",
 				       i, info->value_size > 16 ? '\n' : ' ');
 				fprint_hex(stdout, value + i * step,
 					   info->value_size, " ");
 	}
 	if (i != n) {
-		p_err("%s expected %d bytes got %d", name, n, i);
+		p_err("%s expected %u bytes got %u", name, n, i);
 		return NULL;
 	}
 		jsonw_string_field(wtr, "name", info->name);
 	jsonw_name(wtr, "flags");
-	jsonw_printf(wtr, "%d", info->map_flags);
+	jsonw_printf(wtr, "%u", info->map_flags);
 }
 static int show_map_close_json(int fd, struct bpf_map_info *info)
 			if (prog_type_str)
 				printf("owner_prog_type %s  ", prog_type_str);
 			else
-				printf("owner_prog_type %d  ", prog_type);
+				printf("owner_prog_type %u  ", prog_type);
 		}
 		if (owner_jited)
 			printf("owner%s jited",
 		printf("\n\t");
 	if (info->btf_id)
-		printf("btf_id %d", info->btf_id);
+		printf("btf_id %u", info->btf_id);
 	if (frozen)
 		printf("%sfrozen", info->btf_id ? "  " : "");
 		} else if (is_prefix(*argv, "name")) {
 			NEXT_ARG();
 			map_name = GET_ARG();
+			if (strlen(map_name) > BPF_OBJ_NAME_LEN - 1) {
+				p_info("Warning: map name is longer than %u characters, it will be truncated.",
+				      BPF_OBJ_NAME_LEN - 1);
+			}
 		} else if (is_prefix(*argv, "key")) {
 			if (parse_u32_arg(&argc, &argv, &key_size,
 					  "key size"))

tools/bpf/bpftool/map_perf_ring.c+3−3

@@ -91,15 +91,15 @@ print_bpf_output(void private_data, int cpu, struct perf_event_header event) · 90 unmodified lines
91	jsonw_end_object(json_wtr);	91	jsonw_end_object(json_wtr);
92	} else {	92	} else {
93	if (e->header.type == PERF_RECORD_SAMPLE) {	93	if (e->header.type == PERF_RECORD_SAMPLE) {
94	printf("== @%lld.%09lld CPU: %d index: %d =====\n",	94	printf("== @%llu.%09llu CPU: %d index: %d =====\n",
95	e->time / 1000000000ULL, e->time % 1000000000ULL,	95	e->time / 1000000000ULL, e->time % 1000000000ULL,
96	cpu, idx);	96	cpu, idx);
97	fprint_hex(stdout, e->data, e->size, " ");	97	fprint_hex(stdout, e->data, e->size, " ");
98	printf("\n");	98	printf("\n");
99	} else if (e->header.type == PERF_RECORD_LOST) {	99	} else if (e->header.type == PERF_RECORD_LOST) {
100	printf("lost %lld events\n", lost->lost);	100	printf("lost %llu events\n", lost->lost);
101	} else {	101	} else {
102	printf("unknown event type=%d size=%d\n",	102	printf("unknown event type=%u size=%u\n",
103	e->header.type, e->header.size);	103	e->header.type, e->header.size);
104	}	104	}
105	}	105	}

tools/bpf/bpftool/net.c+2−2

@@ -476,7 +476,7 @@ static void __show_dev_tc_bpf(const struct ip_devname_ifindex *dev, · 475 unmodified lines
476	for (i = 0; i < optq.count; i++) {	476	for (i = 0; i < optq.count; i++) {
477	NET_START_OBJECT;	477	NET_START_OBJECT;
478	NET_DUMP_STR("devname", "%s", dev->devname);	478	NET_DUMP_STR("devname", "%s", dev->devname);
479	NET_DUMP_UINT("ifindex", "(%u)", dev->ifindex);	479	NET_DUMP_UINT("ifindex", "(%u)", (unsigned int)dev->ifindex);
480	NET_DUMP_STR("kind", " %s", attach_loc_strings[loc]);	480	NET_DUMP_STR("kind", " %s", attach_loc_strings[loc]);
481	ret = __show_dev_tc_bpf_name(prog_ids[i], prog_name,	481	ret = __show_dev_tc_bpf_name(prog_ids[i], prog_name,
482	sizeof(prog_name));	482	sizeof(prog_name));
@@ -831,7 +831,7 @@ static void show_link_netfilter(void) · 348 unmodified lines
831	if (err) {	831	if (err) {
832	if (errno == ENOENT)	832	if (errno == ENOENT)
833	break;	833	break;
834	p_err("can't get next link: %s (id %d)", strerror(errno), id);	834	p_err("can't get next link: %s (id %u)", strerror(errno), id);
835	break;	835	break;
836	}	836	}
837		837

tools/bpf/bpftool/netlink_dumper.c+3−3

@@ -45,7 +45,7 @@ static int do_xdp_dump_one(struct nlattr *attr, unsigned int ifindex, · 44 unmodified lines
45	NET_START_OBJECT;	45	NET_START_OBJECT;
46	if (name)	46	if (name)
47	NET_DUMP_STR("devname", "%s", name);	47	NET_DUMP_STR("devname", "%s", name);
48	NET_DUMP_UINT("ifindex", "(%d)", ifindex);	48	NET_DUMP_UINT("ifindex", "(%u)", ifindex);
49		49
50	if (mode == XDP_ATTACHED_MULTI) {	50	if (mode == XDP_ATTACHED_MULTI) {
51	if (json_output) {	51	if (json_output) {
@@ -74,7 +74,7 @@ int do_xdp_dump(struct ifinfomsg ifinfo, struct nlattr *tb) · 22 unmodified lines
74	if (!tb[IFLA_XDP])	74	if (!tb[IFLA_XDP])
75	return 0;	75	return 0;
76		76
77	return do_xdp_dump_one(tb[IFLA_XDP], ifinfo->ifi_index,	77	return do_xdp_dump_one(tb[IFLA_XDP], (unsigned int)ifinfo->ifi_index,
78	libbpf_nla_getattr_str(tb[IFLA_IFNAME]));	78	libbpf_nla_getattr_str(tb[IFLA_IFNAME]));
79	}	79	}
80		80
@@ -168,7 +168,7 @@ int do_filter_dump(struct tcmsg info, struct nlattr tb, const char kind, · 87 unmodified lines
168	NET_START_OBJECT;	168	NET_START_OBJECT;
169	if (devname[0] != '\0')	169	if (devname[0] != '\0')
170	NET_DUMP_STR("devname", "%s", devname);	170	NET_DUMP_STR("devname", "%s", devname);
171	NET_DUMP_UINT("ifindex", "(%u)", ifindex);	171	NET_DUMP_UINT("ifindex", "(%u)", (unsigned int)ifindex);
172	NET_DUMP_STR("kind", " %s", kind);	172	NET_DUMP_STR("kind", " %s", kind);
173	ret = do_bpf_filter_dump(tb[TCA_OPTIONS]);	173	ret = do_bpf_filter_dump(tb[TCA_OPTIONS]);
174	NET_END_OBJECT_FINAL;	174	NET_END_OBJECT_FINAL;

tools/bpf/bpftool/prog.c+7−6

 	print_dev_plain(info->ifindex, info->netns_dev, info->netns_ino);
 	printf("%s", info->gpl_compatible ? "  gpl" : "");
 	if (info->run_time_ns)
-		printf(" run_time_ns %lld run_cnt %lld",
+		printf(" run_time_ns %llu run_cnt %llu",
 		       info->run_time_ns, info->run_cnt);
 	if (info->recursion_misses)
-		printf(" recursion_misses %lld", info->recursion_misses);
+		printf(" recursion_misses %llu", info->recursion_misses);
 	printf("\n");
 }
 	}
 	if (info->btf_id)
-		printf("\n\tbtf_id %d", info->btf_id);
+		printf("\n\tbtf_id %u", info->btf_id);
 	emit_obj_refs_plain(refs_table, info->id, "\n\tpids ");
 		}
 		if (nb_read > buf_size - block_size) {
 			if (buf_size == UINT32_MAX) {
-				p_err("data_in/ctx_in is too long (max: %d)",
+				p_err("data_in/ctx_in is too long (max: %u)",
 				      UINT32_MAX);
 				goto err_free;
 			}
 	obj = bpf_object__open_file(file, &open_opts);
 	if (!obj) {
+		err = -1;
 		p_err("failed to open object file");
 		goto err_close_obj;
 	}
 	t = btf__type_by_id(btf, func_info.type_id);
 	if (!t) {
-		p_err("btf %d doesn't have type %d",
+		p_err("btf %u doesn't have type %u",
 		      info.btf_id, func_info.type_id);
 		goto out;
 	}
 			continue;
 		for (cpu = 0; cpu < obj->rodata->num_cpu; cpu++) {
 			if (profile_open_perf_event(m, cpu, map_fd)) {
-				p_err("failed to create event %s on cpu %d",
+				p_err("failed to create event %s on cpu %u",
 				      metrics[m].name, cpu);
 				return -1;
 			}

tools/bpf/bpftool/tracelog.c+1−1

@@ -78,7 +78,7 @@ static bool get_tracefs_pipe(char *mnt) · 77 unmodified lines
78	return false;	78	return false;
79		79
80	/* Allow room for NULL terminating byte and pipe file name */	80	/* Allow room for NULL terminating byte and pipe file name */
81	snprintf(format, sizeof(format), "%%s %%%zds %%99s %%s %%d %%d\\n",	81	snprintf(format, sizeof(format), "%%s %%%zus %%99s %%s %%d %%d\\n",
82	PATH_MAX - strlen(pipe_name) - 1);	82	PATH_MAX - strlen(pipe_name) - 1);
83	while (fscanf(fp, format, mnt, type) == 2)	83	while (fscanf(fp, format, mnt, type) == 2)
84	if (strcmp(type, fstype) == 0) {	84	if (strcmp(type, fstype) == 0) {

tools/bpf/bpftool/xlated_dumper.c+3−3

@@ -199,13 +199,13 @@ static const char print_imm(void private_data, · 198 unmodified lines
199		199
200	if (insn->src_reg == BPF_PSEUDO_MAP_FD)	200	if (insn->src_reg == BPF_PSEUDO_MAP_FD)
201	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),	201	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
202	"map[id:%u]", insn->imm);	202	"map[id:%d]", insn->imm);
203	else if (insn->src_reg == BPF_PSEUDO_MAP_VALUE)	203	else if (insn->src_reg == BPF_PSEUDO_MAP_VALUE)
204	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),	204	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
205	"map[id:%u][0]+%u", insn->imm, (insn + 1)->imm);	205	"map[id:%d][0]+%d", insn->imm, (insn + 1)->imm);
206	else if (insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)	206	else if (insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)
207	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),	207	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
208	"map[idx:%u]+%u", insn->imm, (insn + 1)->imm);	208	"map[idx:%d]+%d", insn->imm, (insn + 1)->imm);
209	else if (insn->src_reg == BPF_PSEUDO_FUNC)	209	else if (insn->src_reg == BPF_PSEUDO_FUNC)
210	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),	210	snprintf(dd->scratch_buff, sizeof(dd->scratch_buff),
211	"subprog[%+d]", insn->imm);	211	"subprog[%+d]", insn->imm);

tools/bpf/runqslower/Makefile+2−1

 BPFTOOL_OUTPUT := $(OUTPUT)bpftool/
 DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bootstrap/bpftool
 BPFTOOL ?= $(DEFAULT_BPFTOOL)
+BPF_TARGET_ENDIAN ?= --target=bpf
 LIBBPF_SRC := $(abspath ../../lib/bpf)
 BPFOBJ_OUTPUT := $(OUTPUT)libbpf/
 BPFOBJ := $(BPFOBJ_OUTPUT)libbpf.a
 	$(QUIET_GEN)$(BPFTOOL) gen skeleton $< > $@
 $(OUTPUT)/%.bpf.o: %.bpf.c $(BPFOBJ) | $(OUTPUT)
-	$(QUIET_GEN)$(CLANG) -g -O2 --target=bpf $(INCLUDES)		      \
+	$(QUIET_GEN)$(CLANG) -g -O2 $(BPF_TARGET_ENDIAN) $(INCLUDES)	      \
 		 -c $(filter %.c,$^) -o $@ &&				      \
 	$(LLVM_STRIP) -g $@

tools/include/uapi/linux/bpf.h+9−1

 #define BPF_XCHG	(0xe0 | BPF_FETCH)	/* atomic exchange */
 #define BPF_CMPXCHG	(0xf0 | BPF_FETCH)	/* atomic compare-and-write */
+#define BPF_LOAD_ACQ	0x100	/* load-acquire */
+#define BPF_STORE_REL	0x110	/* store-release */
 enum bpf_cond_pseudo_jmp {
 	BPF_MAY_GOTO = 0,
 };
 #define BPF_F_BEFORE		(1U << 3)
 #define BPF_F_AFTER		(1U << 4)
 #define BPF_F_ID		(1U << 5)
+#define BPF_F_PREORDER		(1U << 6)
 #define BPF_F_LINK		BPF_F_LINK /* 1 << 13 */
 /* If BPF_F_STRICT_ALIGNMENT is used in BPF_PROG_LOAD command, the
 		};
 		__u32		next_id;
 		__u32		open_flags;
+		__s32		fd_by_id_token_fd;
 	};
 	struct { /* anonymous struct used by BPF_OBJ_GET_INFO_BY_FD */
 	FN(user_ringbuf_drain, 209, ##ctx)		\
 	FN(cgrp_storage_get, 210, ##ctx)		\
 	FN(cgrp_storage_delete, 211, ##ctx)		\
-	/* */
+	/* This helper list is effectively frozen. If you are trying to	\
+	 * add a new helper, you should add a kfunc instead which has	\
+	 * less stability guarantees. See Documentation/bpf/kfuncs.rst	\
+	 */
 /* backwards-compatibility macros for users of __BPF_FUNC_MAPPER that don't
  * know or care about integer value that is now passed as second argument

tools/include/uapi/linux/btf.h+2−1

 	 * bits 24-28: kind (e.g. int, ptr, array...etc)
 	 * bits 29-30: unused
 	 * bit     31: kind_flag, currently used by
-	 *             struct, union, enum, fwd and enum64
+	 *             struct, union, enum, fwd, enum64,
+	 *             decl_tag and type_tag
 	 */
 	__u32 info;
 	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.

tools/lib/bpf/bpf.c+2−1

 int bpf_btf_get_fd_by_id_opts(__u32 id,
 			      const struct bpf_get_fd_by_id_opts *opts)
 {
-	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
+	const size_t attr_sz = offsetofend(union bpf_attr, fd_by_id_token_fd);
 	union bpf_attr attr;
 	int fd;
 	memset(&attr, 0, attr_sz);
 	attr.btf_id = id;
 	attr.open_flags = OPTS_GET(opts, open_flags, 0);
+	attr.fd_by_id_token_fd = OPTS_GET(opts, token_fd, 0);
 	fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz);
 	return libbpf_err_errno(fd);

tools/lib/bpf/bpf.h+2−1

 struct bpf_get_fd_by_id_opts {
 	size_t sz; /* size of this struct for forward/backward compatibility */
 	__u32 open_flags; /* permissions requested for the operation on fd */
+	__u32 token_fd;
 	size_t :0;
 };
-#define bpf_get_fd_by_id_opts__last_field open_flags
+#define bpf_get_fd_by_id_opts__last_field token_fd
 LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id);
 LIBBPF_API int bpf_prog_get_fd_by_id_opts(__u32 id,

tools/lib/bpf/btf.c+78−27

 	return btf;
 }
-struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf)
+struct btf *btf_load_from_kernel(__u32 id, struct btf *base_btf, int token_fd)
 {
 	struct btf *btf;
 	int btf_fd;
+	LIBBPF_OPTS(bpf_get_fd_by_id_opts, opts);
+	if (token_fd) {
+		opts.open_flags |= BPF_F_TOKEN_FD;
+		opts.token_fd = token_fd;
+	}
-	btf_fd = bpf_btf_get_fd_by_id(id);
+	btf_fd = bpf_btf_get_fd_by_id_opts(id, &opts);
 	if (btf_fd < 0)
 		return libbpf_err_ptr(-errno);
 	return libbpf_ptr(btf);
 }
+struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf)
+{
+	return btf_load_from_kernel(id, base_btf, 0);
+}
 struct btf *btf__load_from_kernel_by_id(__u32 id)
 {
 	return btf__load_from_kernel_by_id_split(id, NULL);
 }
 /* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
-static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id)
+static int btf_add_ref_kind(struct btf *btf, int kind, const char *name, int ref_type_id, int kflag)
 {
 	struct btf_type *t;
 	int sz, name_off = 0;
 	}
 	t->name_off = name_off;
-	t->info = btf_type_info(kind, 0, 0);
+	t->info = btf_type_info(kind, 0, kflag);
 	t->type = ref_type_id;
 	return btf_commit_type(btf, sz);
  */
 int btf__add_ptr(struct btf *btf, int ref_type_id)
 {
-	return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id);
+	return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id, 0);
 }
 /*
 		struct btf_type *t;
 		int id;
-		id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0);
+		id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0, 0);
 		if (id <= 0)
 			return id;
 		t = btf_type_by_id(btf, id);
 	if (!name || !name[0])
 		return libbpf_err(-EINVAL);
-	return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id);
+	return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id, 0);
 }
 /*
  */
 int btf__add_volatile(struct btf *btf, int ref_type_id)
 {
-	return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id);
+	return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id, 0);
 }
 /*
  */
 int btf__add_const(struct btf *btf, int ref_type_id)
 {
-	return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id);
+	return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id, 0);
 }
 /*
  */
 int btf__add_restrict(struct btf *btf, int ref_type_id)
 {
-	return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id);
+	return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id, 0);
 }
 /*
 	if (!value || !value[0])
 		return libbpf_err(-EINVAL);
-	return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id);
+	return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 0);
+}
+/*
+ * Append new BTF_KIND_TYPE_TAG type with:
+ *   - *value*, non-empty/non-NULL tag value;
+ *   - *ref_type_id* - referenced type ID, it might not exist yet;
+ * Set info->kflag to 1, indicating this tag is an __attribute__
+ * Returns:
+ *   - >0, type ID of newly added BTF type;
+ *   - <0, on error.
+ */
+int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id)
+{
+	if (!value || !value[0])
+		return libbpf_err(-EINVAL);
+	return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id, 1);
 }
 /*
 	    linkage != BTF_FUNC_EXTERN)
 		return libbpf_err(-EINVAL);
-	id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id);
+	id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id, 0);
 	if (id > 0) {
 		struct btf_type *t = btf_type_by_id(btf, id);
 	return 0;
 }
-/*
+static int btf_add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
- * Append new BTF_KIND_DECL_TAG type with:
+			    int component_idx, int kflag)
- *   - *value* - non-empty/non-NULL string;
- *   - *ref_type_id* - referenced type ID, it might not exist yet;
- *   - *component_idx* - -1 for tagging reference type, otherwise struct/union
- *     member or function argument index;
- * Returns:
- *   - >0, type ID of newly added BTF type;
- *   - <0, on error.
- */
-int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
-		 int component_idx)
 {
 	struct btf_type *t;
 	int sz, value_off;
 		return value_off;
 	t->name_off = value_off;
-	t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, false);
+	t->info = btf_type_info(BTF_KIND_DECL_TAG, 0, kflag);
 	t->type = ref_type_id;
 	btf_decl_tag(t)->component_idx = component_idx;
 	return btf_commit_type(btf, sz);
 }
+/*
+ * Append new BTF_KIND_DECL_TAG type with:
+ *   - *value* - non-empty/non-NULL string;
+ *   - *ref_type_id* - referenced type ID, it might not exist yet;
+ *   - *component_idx* - -1 for tagging reference type, otherwise struct/union
+ *     member or function argument index;
+ * Returns:
+ *   - >0, type ID of newly added BTF type;
+ *   - <0, on error.
+ */
+int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
+		      int component_idx)
+{
+	return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 0);
+}
+/*
+ * Append new BTF_KIND_DECL_TAG type with:
+ *   - *value* - non-empty/non-NULL string;
+ *   - *ref_type_id* - referenced type ID, it might not exist yet;
+ *   - *component_idx* - -1 for tagging reference type, otherwise struct/union
+ *     member or function argument index;
+ * Set info->kflag to 1, indicating this tag is an __attribute__
+ * Returns:
+ *   - >0, type ID of newly added BTF type;
+ *   - <0, on error.
+ */
+int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id,
+		       int component_idx)
+{
+	return btf_add_decl_tag(btf, value, ref_type_id, component_idx, 1);
+}
 struct btf_ext_sec_info_param {
 	__u32 off;
 	__u32 len;
 		.desc = "line_info",
 	};
 	struct btf_ext_sec_info_param core_relo = {
-		.off = btf_ext->hdr->core_relo_off,
-		.len = btf_ext->hdr->core_relo_len,
 		.min_rec_size = sizeof(struct bpf_core_relo),
 		.ext_info = &btf_ext->core_relo_info,
 		.desc = "core_relo",
 	if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len))
 		return 0; /* skip core relos parsing */
+	core_relo.off = btf_ext->hdr->core_relo_off;
+	core_relo.len = btf_ext->hdr->core_relo_len;
 	err = btf_ext_parse_sec_info(btf_ext, &core_relo, is_native);
 	if (err)
 		return err;

tools/lib/bpf/btf.h+3−0

 LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id);
 LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id);
 LIBBPF_API int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id);
+LIBBPF_API int btf__add_type_attr(struct btf *btf, const char *value, int ref_type_id);
 /* func and func_proto construction APIs */
 LIBBPF_API int btf__add_func(struct btf *btf, const char *name,
 /* tag construction API */
 LIBBPF_API int btf__add_decl_tag(struct btf *btf, const char *value, int ref_type_id,
 			    int component_idx);
+LIBBPF_API int btf__add_decl_attr(struct btf *btf, const char *value, int ref_type_id,
+				  int component_idx);
 struct btf_dedup_opts {
 	size_t sz;

tools/lib/bpf/btf_dump.c+4−1

 		case BTF_KIND_TYPE_TAG:
 			btf_dump_emit_mods(d, decls);
 			name = btf_name_of(d, t->name_off);
-			btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name);
+			if (btf_kflag(t))
+				btf_dump_printf(d, " __attribute__((%s))", name);
+			else
+				btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name);
 			break;
 		case BTF_KIND_ARRAY: {
 			const struct btf_array *a = btf_array(t);

tools/lib/bpf/libbpf.c+151−86

 struct usdt_manager;
+enum bpf_object_state {
+	OBJ_OPEN,
+	OBJ_PREPARED,
+	OBJ_LOADED,
+};
 struct bpf_object {
 	char name[BPF_OBJ_NAME_LEN];
 	char license[64];
 	__u32 kern_version;
+	enum bpf_object_state state;
 	struct bpf_program *programs;
 	size_t nr_programs;
 	struct bpf_map *maps;
 	int nr_extern;
 	int kconfig_map_idx;
-	bool loaded;
 	bool has_subcalls;
 	bool has_rodata;
 	obj->kconfig_map_idx = -1;
 	obj->kern_version = get_kernel_version();
-	obj->loaded = false;
+	obj->state  = OBJ_OPEN;
 	return obj;
 }
 	}
 	len = strlen(value);
-	if (value[len - 1] != '"') {
+	if (len < 2 || value[len - 1] != '"') {
 		pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
 			ext->name, value);
 		return -EINVAL;
 	return 0;
 }
+static bool map_is_created(const struct bpf_map *map)
+{
+	return map->obj->state >= OBJ_PREPARED || map->reused;
+}
 bool bpf_map__autocreate(const struct bpf_map *map)
 {
 	return map->autocreate;
 int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
 {
-	if (map->obj->loaded)
+	if (map_is_created(map))
 		return libbpf_err(-EBUSY);
 	map->autocreate = autocreate;
 int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
 {
-	if (map->obj->loaded)
+	if (map_is_created(map))
 		return libbpf_err(-EBUSY);
 	map->def.max_entries = max_entries;
 static void bpf_map__destroy(struct bpf_map *map);
-static bool map_is_created(const struct bpf_map *map)
-{
-	return map->obj->loaded || map->reused;
-}
 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
 {
 	LIBBPF_OPTS(bpf_map_create_opts, create_attr);
 	size_t i;
 	int err;
-	for (i = 0; i < obj->nr_programs; i++) {
-		prog = &obj->programs[i];
-		err = bpf_object__sanitize_prog(obj, prog);
-		if (err)
-			return err;
-	}
 	for (i = 0; i < obj->nr_programs; i++) {
 		prog = &obj->programs[i];
 		if (prog_is_subprog(obj, prog))
 	return 0;
 }
+static int bpf_object_prepare_progs(struct bpf_object *obj)
+{
+	struct bpf_program *prog;
+	size_t i;
+	int err;
+	for (i = 0; i < obj->nr_programs; i++) {
+		prog = &obj->programs[i];
+		err = bpf_object__sanitize_prog(obj, prog);
+		if (err)
+			return err;
+	}
+	return 0;
+}
 static const struct bpf_sec_def *find_sec_def(const char *sec_name);
 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
 	return 0;
 }
+static void bpf_object_unpin(struct bpf_object *obj)
+{
+	int i;
+	/* unpin any maps that were auto-pinned during load */
+	for (i = 0; i < obj->nr_maps; i++)
+		if (obj->maps[i].pinned && !obj->maps[i].reused)
+			bpf_map__unpin(&obj->maps[i], NULL);
+}
+static void bpf_object_post_load_cleanup(struct bpf_object *obj)
+{
+	int i;
+	/* clean up fd_array */
+	zfree(&obj->fd_array);
+	/* clean up module BTFs */
+	for (i = 0; i < obj->btf_module_cnt; i++) {
+		close(obj->btf_modules[i].fd);
+		btf__free(obj->btf_modules[i].btf);
+		free(obj->btf_modules[i].name);
+	}
+	obj->btf_module_cnt = 0;
+	zfree(&obj->btf_modules);
+	/* clean up vmlinux BTF */
+	btf__free(obj->btf_vmlinux);
+	obj->btf_vmlinux = NULL;
+}
+static int bpf_object_prepare(struct bpf_object *obj, const char *target_btf_path)
+{
+	int err;
+	if (obj->state >= OBJ_PREPARED) {
+		pr_warn("object '%s': prepare loading can't be attempted twice\n", obj->name);
+		return -EINVAL;
+	}
+	err = bpf_object_prepare_token(obj);
+	err = err ? : bpf_object__probe_loading(obj);
+	err = err ? : bpf_object__load_vmlinux_btf(obj, false);
+	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
+	err = err ? : bpf_object__sanitize_maps(obj);
+	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
+	err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
+	err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
+	err = err ? : bpf_object__sanitize_and_load_btf(obj);
+	err = err ? : bpf_object__create_maps(obj);
+	err = err ? : bpf_object_prepare_progs(obj);
+	if (err) {
+		bpf_object_unpin(obj);
+		bpf_object_unload(obj);
+		obj->state = OBJ_LOADED;
+		return err;
+	}
+	obj->state = OBJ_PREPARED;
+	return 0;
+}
 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
 {
-	int err, i;
+	int err;
 	if (!obj)
 		return libbpf_err(-EINVAL);
-	if (obj->loaded) {
+	if (obj->state >= OBJ_LOADED) {
 		pr_warn("object '%s': load can't be attempted twice\n", obj->name);
 		return libbpf_err(-EINVAL);
 	}
 		return libbpf_err(-LIBBPF_ERRNO__ENDIAN);
 	}
-	err = bpf_object_prepare_token(obj);
+	if (obj->state < OBJ_PREPARED) {
-	err = err ? : bpf_object__probe_loading(obj);
+		err = bpf_object_prepare(obj, target_btf_path);
-	err = err ? : bpf_object__load_vmlinux_btf(obj, false);
+		if (err)
-	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
+			return libbpf_err(err);
-	err = err ? : bpf_object__sanitize_maps(obj);
+	}
-	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
+	err = bpf_object__load_progs(obj, extra_log_level);
-	err = err ? : bpf_object_adjust_struct_ops_autoload(obj);
-	err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
-	err = err ? : bpf_object__sanitize_and_load_btf(obj);
-	err = err ? : bpf_object__create_maps(obj);
-	err = err ? : bpf_object__load_progs(obj, extra_log_level);
 	err = err ? : bpf_object_init_prog_arrays(obj);
 	err = err ? : bpf_object_prepare_struct_ops(obj);
 			err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
 	}
-	/* clean up fd_array */
+	bpf_object_post_load_cleanup(obj);
-	zfree(&obj->fd_array);
+	obj->state = OBJ_LOADED; /* doesn't matter if successfully or not */
-	/* clean up module BTFs */
+	if (err) {
-	for (i = 0; i < obj->btf_module_cnt; i++) {
+		bpf_object_unpin(obj);
-		close(obj->btf_modules[i].fd);
+		bpf_object_unload(obj);
-		btf__free(obj->btf_modules[i].btf);
+		pr_warn("failed to load object '%s'\n", obj->path);
-		free(obj->btf_modules[i].name);
+		return libbpf_err(err);
 	}
-	free(obj->btf_modules);
-	/* clean up vmlinux BTF */
-	btf__free(obj->btf_vmlinux);
-	obj->btf_vmlinux = NULL;
-	obj->loaded = true; /* doesn't matter if successfully or not */
-	if (err)
-		goto out;
 	return 0;
-out:
+}
-	/* unpin any maps that were auto-pinned during load */
-	for (i = 0; i < obj->nr_maps; i++)
-		if (obj->maps[i].pinned && !obj->maps[i].reused)
-			bpf_map__unpin(&obj->maps[i], NULL);
-	bpf_object_unload(obj);
+int bpf_object__prepare(struct bpf_object *obj)
-	pr_warn("failed to load object '%s'\n", obj->path);
+{
-	return libbpf_err(err);
+	return libbpf_err(bpf_object_prepare(obj, NULL));
 }
 int bpf_object__load(struct bpf_object *obj)
 	if (!obj)
 		return libbpf_err(-ENOENT);
-	if (!obj->loaded) {
+	if (obj->state < OBJ_PREPARED) {
 		pr_warn("object not yet loaded; load it first\n");
 		return libbpf_err(-ENOENT);
 	}
 	if (!obj)
 		return libbpf_err(-ENOENT);
-	if (!obj->loaded) {
+	if (obj->state < OBJ_LOADED) {
 		pr_warn("object not yet loaded; load it first\n");
 		return libbpf_err(-ENOENT);
 	}
 	if (IS_ERR_OR_NULL(obj))
 		return;
+	/*
+	 * if user called bpf_object__prepare() without ever getting to
+	 * bpf_object__load(), we need to clean up stuff that is normally
+	 * cleaned up at the end of loading step
+	 */
+	bpf_object_post_load_cleanup(obj);
 	usdt_manager_free(obj->usdt_man);
 	obj->usdt_man = NULL;
 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
 {
-	if (obj->loaded)
+	if (obj->state >= OBJ_LOADED)
 		return libbpf_err(-EINVAL);
 	obj->kern_version = kern_version;
 	struct bpf_gen *gen;
 	if (!opts)
-		return -EFAULT;
+		return libbpf_err(-EFAULT);
 	if (!OPTS_VALID(opts, gen_loader_opts))
-		return -EINVAL;
+		return libbpf_err(-EINVAL);
 	gen = calloc(sizeof(*gen), 1);
 	if (!gen)
-		return -ENOMEM;
+		return libbpf_err(-ENOMEM);
 	gen->opts = opts;
 	gen->swapped_endian = !is_native_endianness(obj);
 	obj->gen_loader = gen;
 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
 {
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
 		return libbpf_err(-EINVAL);
 	prog->autoload = autoload;
 {
 	struct bpf_insn *insns;
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
-		return -EBUSY;
+		return libbpf_err(-EBUSY);
 	insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
 	/* NULL is a valid return from reallocarray if the new count is zero */
 	if (!insns && new_insn_cnt) {
 		pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
-		return -ENOMEM;
+		return libbpf_err(-ENOMEM);
 	}
 	memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns));
 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
 {
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
 		return libbpf_err(-EBUSY);
 	/* if type is not changed, do nothing */
 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
 					   enum bpf_attach_type type)
 {
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
 		return libbpf_err(-EBUSY);
 	prog->expected_attach_type = type;
 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
 {
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
 		return libbpf_err(-EBUSY);
 	prog->prog_flags = flags;
 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
 {
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
 		return libbpf_err(-EBUSY);
 	prog->log_level = log_level;
 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
 {
 	if (log_size && !log_buf)
-		return -EINVAL;
+		return libbpf_err(-EINVAL);
 	if (prog->log_size > UINT_MAX)
-		return -EINVAL;
+		return libbpf_err(-EINVAL);
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
-		return -EBUSY;
+		return libbpf_err(-EBUSY);
 	prog->log_buf = log_buf;
 	prog->log_size = log_size;
 	return libbpf_err(err);
 }
-static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
+static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd, int token_fd)
 {
 	struct bpf_prog_info info;
 	__u32 info_len = sizeof(info);
 		pr_warn("The target program doesn't have BTF\n");
 		goto out;
 	}
-	btf = btf__load_from_kernel_by_id(info.btf_id);
+	btf = btf_load_from_kernel(info.btf_id, NULL, token_fd);
 	err = libbpf_get_error(btf);
 	if (err) {
 		pr_warn("Failed to get BTF %d of the program: %s\n", info.btf_id, errstr(err));
 			pr_warn("prog '%s': attach program FD is not set\n", prog->name);
 			return -EINVAL;
 		}
-		err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
+		err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd, prog->obj->token_fd);
 		if (err < 0) {
 			pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %s\n",
 				prog->name, attach_prog_fd, attach_name, errstr(err));
 int bpf_map__set_value_size(struct bpf_map *map, __u32 size)
 {
-	if (map->obj->loaded || map->reused)
+	if (map_is_created(map))
 		return libbpf_err(-EBUSY);
 	if (map->mmaped) {
 		int err;
 		if (map->def.type != BPF_MAP_TYPE_ARRAY)
-			return -EOPNOTSUPP;
+			return libbpf_err(-EOPNOTSUPP);
 		mmap_old_sz = bpf_map_mmap_sz(map);
 		mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);
 		if (err) {
 			pr_warn("map '%s': failed to resize memory-mapped region: %s\n",
 				bpf_map__name(map), errstr(err));
-			return err;
+			return libbpf_err(err);
 		}
 		err = map_btf_datasec_resize(map, size);
 		if (err && err != -ENOENT) {
 {
 	size_t actual_sz;
-	if (map->obj->loaded || map->reused)
+	if (map_is_created(map))
 		return libbpf_err(-EBUSY);
 	if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG)
 	if (target_fd) {
 		LIBBPF_OPTS(bpf_link_create_opts, target_opts);
-		btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
+		btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd, prog->obj->token_fd);
 		if (btf_id < 0)
 			return libbpf_err_ptr(btf_id);
 	int err;
 	if (!bpf_map__is_struct_ops(map))
-		return -EINVAL;
+		return libbpf_err(-EINVAL);
 	if (map->fd < 0) {
 		pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);
-		return -EINVAL;
+		return libbpf_err(-EINVAL);
 	}
 	st_ops_link = container_of(link, struct bpf_link_struct_ops, link);
 	/* Ensure the type of a link is correct */
 	if (st_ops_link->map_fd < 0)
-		return -EINVAL;
+		return libbpf_err(-EINVAL);
 	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);
 	/* It can be EBUSY if the map has been used to create or
 	if (!prog || attach_prog_fd < 0)
 		return libbpf_err(-EINVAL);
-	if (prog->obj->loaded)
+	if (prog->obj->state >= OBJ_LOADED)
 		return libbpf_err(-EINVAL);
 	if (attach_prog_fd && !attach_func_name) {
 	if (attach_prog_fd) {
 		btf_id = libbpf_find_prog_btf_id(attach_func_name,
-						 attach_prog_fd);
+						 attach_prog_fd, prog->obj->token_fd);
 		if (btf_id < 0)
 			return libbpf_err(btf_id);
 	} else {

tools/lib/bpf/libbpf.h+13−0

@@ -241,6 +241,19 @@ LIBBPF_API struct bpf_object * · 240 unmodified lines
241	bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,	241	bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
242	const struct bpf_object_open_opts *opts);	242	const struct bpf_object_open_opts *opts);
243		243
		244	/**
		245	* @brief bpf_object__prepare() prepares BPF object for loading:
		246	* performs ELF processing, relocations, prepares final state of BPF program
		247	* instructions (accessible with bpf_program__insns()), creates and
		248	* (potentially) pins maps. Leaves BPF object in the state ready for program
		249	* loading.
		250	* @param obj Pointer to a valid BPF object instance returned by
		251	* *bpf_object__open()** API
		252	* @return 0, on success; negative error code, otherwise, error code is
		253	* stored in errno
		254	*/
		255	int bpf_object__prepare(struct bpf_object *obj);
		256
244	/**	257	/**
245	* @brief bpf_object__load() loads BPF object into kernel.	258	* @brief bpf_object__load() loads BPF object into kernel.
246	* @param obj Pointer to a valid BPF object instance returned by	259	* @param obj Pointer to a valid BPF object instance returned by

tools/lib/bpf/libbpf.map+3−0

@@ -436,4 +436,7 @@ LIBBPF_1.6.0 { · 435 unmodified lines
436	bpf_linker__add_buf;	436	bpf_linker__add_buf;
437	bpf_linker__add_fd;	437	bpf_linker__add_fd;
438	bpf_linker__new_fd;	438	bpf_linker__new_fd;
		439	bpf_object__prepare;
		440	btf__add_decl_attr;
		441	btf__add_type_attr;
439	} LIBBPF_1.5.0;	442	} LIBBPF_1.5.0;

tools/lib/bpf/libbpf_internal.h+1−0

@@ -409,6 +409,7 @@ int libbpf__load_raw_btf(const char *raw_types, size_t types_len, · 408 unmodified lines
409	int btf_load_into_kernel(struct btf *btf,	409	int btf_load_into_kernel(struct btf *btf,
410	char *log_buf, size_t log_sz, __u32 log_level,	410	char *log_buf, size_t log_sz, __u32 log_level,
411	int token_fd);	411	int token_fd);
		412	struct btf btf_load_from_kernel(__u32 id, struct btf base_btf, int token_fd);
412		413
413	struct btf btf_get_from_fd(int btf_fd, struct btf base_btf);	414	struct btf btf_get_from_fd(int btf_fd, struct btf base_btf);
414	void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,	415	void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type,

tools/lib/bpf/linker.c+1−1

@@ -2163,7 +2163,7 @@ add_sym: · 2162 unmodified lines
2163		2163
2164	obj->sym_map[src_sym_idx] = dst_sym_idx;	2164	obj->sym_map[src_sym_idx] = dst_sym_idx;
2165		2165
2166	if (sym_type == STT_SECTION && dst_sym) {	2166	if (sym_type == STT_SECTION && dst_sec) {
2167	dst_sec->sec_sym_idx = dst_sym_idx;	2167	dst_sec->sec_sym_idx = dst_sym_idx;
2168	dst_sym->st_value = 0;	2168	dst_sym->st_value = 0;
2169	}	2169	}

tools/lib/bpf/relo_core.c+20−4

 {
 	const struct bpf_core_accessor *acc;
 	const struct btf_type *t;
-	__u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
+	__u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id, elem_id;
 	const struct btf_member *m;
 	const struct btf_type *mt;
 	bool bitfield;
 	if (!acc->name) {
 		if (relo->kind == BPF_CORE_FIELD_BYTE_OFFSET) {
 			*val = spec->bit_offset / 8;
-			/* remember field size for load/store mem size */
+			/* remember field size for load/store mem size;
-			sz = btf__resolve_size(spec->btf, acc->type_id);
+			 * note, for arrays we care about individual element
+			 * sizes, not the overall array size
+			 */
+			t = skip_mods_and_typedefs(spec->btf, acc->type_id, &elem_id);
+			while (btf_is_array(t))
+				t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
+			sz = btf__resolve_size(spec->btf, elem_id);
 			if (sz < 0)
 				return -EINVAL;
 			*field_sz = sz;
 	case BPF_CORE_FIELD_BYTE_OFFSET:
 		*val = byte_off;
 		if (!bitfield) {
-			*field_sz = byte_sz;
+			/* remember field size for load/store mem size;
+			 * note, for arrays we care about individual element
+			 * sizes, not the overall array size
+			 */
+			t = skip_mods_and_typedefs(spec->btf, field_type_id, &elem_id);
+			while (btf_is_array(t))
+				t = skip_mods_and_typedefs(spec->btf, btf_array(t)->type, &elem_id);
+			sz = btf__resolve_size(spec->btf, elem_id);
+			if (sz < 0)
+				return -EINVAL;
+			*field_sz = sz;
 			*type_id = field_type_id;
 		}
 		break;

tools/lib/bpf/str_error.c+1−1

@@ -36,7 +36,7 @@ char libbpf_strerror_r(int err, char dst, int len) · 35 unmodified lines
36	return dst;	36	return dst;
37	}	37	}
38		38
39	const char *errstr(int err)	39	const char *libbpf_errstr(int err)
40	{	40	{
41	static __thread char buf[12];	41	static __thread char buf[12];
42		42

tools/lib/bpf/str_error.h+5−2

 char *libbpf_strerror_r(int err, char *dst, int len);
 /**
- * @brief **errstr()** returns string corresponding to numeric errno
+ * @brief **libbpf_errstr()** returns string corresponding to numeric errno
  * @param err negative numeric errno
  * @return pointer to string representation of the errno, that is invalidated
  * upon the next call.
  */
-const char *errstr(int err);
+const char *libbpf_errstr(int err);
+#define errstr(err) libbpf_errstr(err)
 #endif /* __LIBBPF_STR_ERROR_H */

tools/lib/bpf/usdt.bpf.h+32−0

@@ -108,6 +108,38 @@ int bpf_usdt_arg_cnt(struct pt_regs *ctx) · 107 unmodified lines
108	return spec->arg_cnt;	108	return spec->arg_cnt;
109	}	109	}
110		110
		111	/* Returns the size in bytes of the #arg_num (zero-indexed) USDT argument.
		112	* Returns negative error if argument is not found or arg_num is invalid.
		113	*/
		114	static __always_inline
		115	int bpf_usdt_arg_size(struct pt_regs *ctx, __u64 arg_num)
		116	{
		117	struct __bpf_usdt_arg_spec *arg_spec;
		118	struct __bpf_usdt_spec *spec;
		119	int spec_id;
		120
		121	spec_id = __bpf_usdt_spec_id(ctx);
		122	if (spec_id < 0)
		123	return -ESRCH;
		124
		125	spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id);
		126	if (!spec)
		127	return -ESRCH;
		128
		129	if (arg_num >= BPF_USDT_MAX_ARG_CNT)
		130	return -ENOENT;
		131	barrier_var(arg_num);
		132	if (arg_num >= spec->arg_cnt)
		133	return -ENOENT;
		134
		135	arg_spec = &spec->args[arg_num];
		136
		137	/* arg_spec->arg_bitshift = 64 - arg_sz * 8
		138	* so: arg_sz = (64 - arg_spec->arg_bitshift) / 8
		139	*/
		140	return (unsigned int)(64 - arg_spec->arg_bitshift) / 8;
		141	}
		142
111	/* Fetch USDT argument #arg_num (zero-indexed) and put its value into *res.	143	/* Fetch USDT argument #arg_num (zero-indexed) and put its value into *res.
112	* Returns 0 on success; negative error, otherwise.	144	* Returns 0 on success; negative error, otherwise.
113	* On error *res is guaranteed to be set to zero.	145	* On error *res is guaranteed to be set to zero.

tools/testing/selftests/bpf/DENYLIST.aarch64+0−9

@@ -1,12 +1,3 @@
1	bpf_cookie/multi_kprobe_attach_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
2	bpf_cookie/multi_kprobe_link_api # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
3	kprobe_multi_bench_attach # needs CONFIG_FPROBE
4	kprobe_multi_test # needs CONFIG_FPROBE
5	module_attach # prog 'kprobe_multi': failed to auto-attach: -95
6	fentry_test/fentry_many_args # fentry_many_args:FAIL:fentry_many_args_attach unexpected error: -524	1	fentry_test/fentry_many_args # fentry_many_args:FAIL:fentry_many_args_attach unexpected error: -524
7	fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_many_args_attach unexpected error: -524	2	fexit_test/fexit_many_args # fexit_many_args:FAIL:fexit_many_args_attach unexpected error: -524
8	tracing_struct/struct_many_args # struct_many_args:FAIL:tracing_struct_many_args__attach unexpected error: -524	3	tracing_struct/struct_many_args # struct_many_args:FAIL:tracing_struct_many_args__attach unexpected error: -524
9	fill_link_info/kprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
10	fill_link_info/kretprobe_multi_link_info # bpf_program__attach_kprobe_multi_opts unexpected error: -95
11	fill_link_info/kprobe_multi_invalid_ubuff # bpf_program__attach_kprobe_multi_opts unexpected error: -95
12	missed/kprobe_recursion # missed_kprobe_recursion__attach unexpected error: -95 (errno 95)

tools/testing/selftests/bpf/Makefile+14−14

 TEST_INST_SUBDIRS += cpuv4
 endif
-TEST_GEN_FILES = test_lwt_ip_encap.bpf.o test_tc_edt.bpf.o
+TEST_GEN_FILES = test_tc_edt.bpf.o
 TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c)
 # Order correspond to 'make run_tests' order
 TEST_PROGS := test_kmod.sh \
-	test_xdp_redirect_multi.sh \
-	test_tunnel.sh \
-	test_lwt_seg6local.sh \
 	test_lirc_mode2.sh \
-	test_xdp_vlan_mode_generic.sh \
-	test_xdp_vlan_mode_native.sh \
-	test_lwt_ip_encap.sh \
 	test_tc_tunnel.sh \
 	test_tc_edt.sh \
 	test_xdping.sh \
 	test_xsk.sh \
 	test_xdp_features.sh
-TEST_PROGS_EXTENDED := with_addr.sh \
+TEST_PROGS_EXTENDED := \
-	with_tunnels.sh ima_setup.sh verify_sig_setup.sh \
+	ima_setup.sh verify_sig_setup.sh \
-	test_xdp_vlan.sh test_bpftool.py
+	test_bpftool.py
 TEST_KMODS := bpf_testmod.ko bpf_test_no_cfi.ko bpf_test_modorder_x.ko \
 	bpf_test_modorder_y.ko
 	veristat \
 	xdp_features \
 	xdp_hw_metadata \
-	xdp_redirect_multi \
 	xdp_synproxy \
 	xdping \
 	xskxceiver
   LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
   # both llvm-config and lib.mk add -D_GNU_SOURCE, which ends up as conflict
   LLVM_CFLAGS  += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
-  LLVM_LDLIBS  += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
+  # Prefer linking statically if it's available, otherwise fallback to shared
-  LLVM_LDLIBS  += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
+  ifeq ($(shell $(LLVM_CONFIG) --link-static --libs >/dev/null 2>&1 && echo static),static)
-  LLVM_LDLIBS  += -lstdc++
+    LLVM_LDLIBS  += $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
+    LLVM_LDLIBS  += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
+    LLVM_LDLIBS  += -lstdc++
+  else
+    LLVM_LDLIBS  += $(shell $(LLVM_CONFIG) --link-shared --libs $(LLVM_CONFIG_LIB_COMPONENTS))
+  endif
   LLVM_LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
 endif
 		    BPFTOOL_OUTPUT=$(HOST_BUILD_DIR)/bpftool/		       \
 		    BPFOBJ_OUTPUT=$(BUILD_DIR)/libbpf/			       \
 		    BPFOBJ=$(BPFOBJ) BPF_INCLUDE=$(INCLUDE_DIR)		       \
+		    BPF_TARGET_ENDIAN=$(BPF_TARGET_ENDIAN)		       \
 		    EXTRA_CFLAGS='-g $(OPT_FLAGS) $(SAN_CFLAGS) $(EXTRA_CFLAGS)' \
 		    EXTRA_LDFLAGS='$(SAN_LDFLAGS) $(EXTRA_LDFLAGS)' &&	       \
 		    cp $(RUNQSLOWER_OUTPUT)runqslower $@
 			     $(TRUNNER_EXTRA_OBJS) $$(BPFOBJ)		\
 			     $(RESOLVE_BTFIDS)				\
 			     $(TRUNNER_BPFTOOL)				\
+			     $(OUTPUT)/veristat				\
 			     | $(TRUNNER_BINARY)-extras
 	$$(call msg,BINARY,,$$@)
 	$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) $$(LDFLAGS) -o $$@

tools/testing/selftests/bpf/bpf_arena_spin_lock.h+533−0

@@ -0,0 +1,533 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#ifndef BPF_ARENA_SPIN_LOCK_H
	4	#define BPF_ARENA_SPIN_LOCK_H
	5
	6	#include <vmlinux.h>
	7	#include <bpf/bpf_helpers.h>
	8	#include "bpf_atomic.h"
	9
	10	#define arch_mcs_spin_lock_contended_label(l, label) smp_cond_load_acquire_label(l, VAL, label)
	11	#define arch_mcs_spin_unlock_contended(l) smp_store_release((l), 1)
	12
	13	#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
	14
	15	#define EBUSY 16
	16	#define EOPNOTSUPP 95
	17	#define ETIMEDOUT 110
	18
	19	#ifndef __arena
	20	#define __arena __attribute__((address_space(1)))
	21	#endif
	22
	23	extern unsigned long CONFIG_NR_CPUS __kconfig;
	24
	25	/*
	26	* Typically, we'd just rely on the definition in vmlinux.h for qspinlock, but
	27	* PowerPC overrides the definition to define lock->val as u32 instead of
	28	* atomic_t, leading to compilation errors. Import a local definition below so
	29	* that we don't depend on the vmlinux.h version.
	30	*/
	31
	32	struct __qspinlock {
	33	union {
	34	atomic_t val;
	35	struct {
	36	u8 locked;
	37	u8 pending;
	38	};
	39	struct {
	40	u16 locked_pending;
	41	u16 tail;
	42	};
	43	};
	44	};
	45
	46	#define arena_spinlock_t struct __qspinlock
	47	/* FIXME: Using typedef causes CO-RE relocation error */
	48	/* typedef struct qspinlock arena_spinlock_t; */
	49
	50	struct arena_mcs_spinlock {
	51	struct arena_mcs_spinlock __arena *next;
	52	int locked;
	53	int count;
	54	};
	55
	56	struct arena_qnode {
	57	struct arena_mcs_spinlock mcs;
	58	};
	59
	60	#define _Q_MAX_NODES 4
	61	#define _Q_PENDING_LOOPS 1
	62
	63	/*
	64	* Bitfields in the atomic value:
	65	*
	66	* 0- 7: locked byte
	67	* 8: pending
	68	* 9-15: not used
	69	* 16-17: tail index
	70	* 18-31: tail cpu (+1)
	71	*/
	72	#define _Q_MAX_CPUS 1024
	73
	74	#define _Q_SET_MASK(type) (((1U << _Q_ ## type ## _BITS) - 1)\
	75	<< _Q_ ## type ## _OFFSET)
	76	#define _Q_LOCKED_OFFSET 0
	77	#define _Q_LOCKED_BITS 8
	78	#define _Q_LOCKED_MASK _Q_SET_MASK(LOCKED)
	79
	80	#define _Q_PENDING_OFFSET (_Q_LOCKED_OFFSET + _Q_LOCKED_BITS)
	81	#define _Q_PENDING_BITS 8
	82	#define _Q_PENDING_MASK _Q_SET_MASK(PENDING)
	83
	84	#define _Q_TAIL_IDX_OFFSET (_Q_PENDING_OFFSET + _Q_PENDING_BITS)
	85	#define _Q_TAIL_IDX_BITS 2
	86	#define _Q_TAIL_IDX_MASK _Q_SET_MASK(TAIL_IDX)
	87
	88	#define _Q_TAIL_CPU_OFFSET (_Q_TAIL_IDX_OFFSET + _Q_TAIL_IDX_BITS)
	89	#define _Q_TAIL_CPU_BITS (32 - _Q_TAIL_CPU_OFFSET)
	90	#define _Q_TAIL_CPU_MASK _Q_SET_MASK(TAIL_CPU)
	91
	92	#define _Q_TAIL_OFFSET _Q_TAIL_IDX_OFFSET
	93	#define _Q_TAIL_MASK (_Q_TAIL_IDX_MASK \| _Q_TAIL_CPU_MASK)
	94
	95	#define _Q_LOCKED_VAL (1U << _Q_LOCKED_OFFSET)
	96	#define _Q_PENDING_VAL (1U << _Q_PENDING_OFFSET)
	97
	98	#define likely(x) __builtin_expect(!!(x), 1)
	99	#define unlikely(x) __builtin_expect(!!(x), 0)
	100
	101	struct arena_qnode __arena qnodes[_Q_MAX_CPUS][_Q_MAX_NODES];
	102
	103	static inline u32 encode_tail(int cpu, int idx)
	104	{
	105	u32 tail;
	106
	107	tail = (cpu + 1) << _Q_TAIL_CPU_OFFSET;
	108	tail \|= idx << _Q_TAIL_IDX_OFFSET; /* assume < 4 */
	109
	110	return tail;
	111	}
	112
	113	static inline struct arena_mcs_spinlock __arena *decode_tail(u32 tail)
	114	{
	115	u32 cpu = (tail >> _Q_TAIL_CPU_OFFSET) - 1;
	116	u32 idx = (tail & _Q_TAIL_IDX_MASK) >> _Q_TAIL_IDX_OFFSET;
	117
	118	return &qnodes[cpu][idx].mcs;
	119	}
	120
	121	static inline
	122	struct arena_mcs_spinlock __arena grab_mcs_node(struct arena_mcs_spinlock __arena base, int idx)
	123	{
	124	return &((struct arena_qnode __arena *)base + idx)->mcs;
	125	}
	126
	127	#define _Q_LOCKED_PENDING_MASK (_Q_LOCKED_MASK \| _Q_PENDING_MASK)
	128
	129	/**
	130	* xchg_tail - Put in the new queue tail code word & retrieve previous one
	131	* @lock : Pointer to queued spinlock structure
	132	* @tail : The new queue tail code word
	133	* Return: The previous queue tail code word
	134	*
	135	* xchg(lock, tail)
	136	*
	137	* p,, -> n,, ; prev = xchg(lock, node)
	138	*/
	139	static __always_inline u32 xchg_tail(arena_spinlock_t __arena *lock, u32 tail)
	140	{
	141	u32 old, new;
	142
	143	old = atomic_read(&lock->val);
	144	do {
	145	new = (old & _Q_LOCKED_PENDING_MASK) \| tail;
	146	/*
	147	* We can use relaxed semantics since the caller ensures that
	148	* the MCS node is properly initialized before updating the
	149	* tail.
	150	*/
	151	/* These loops are not expected to stall, but we still need to
	152	* prove to the verifier they will terminate eventually.
	153	*/
	154	cond_break_label(out);
	155	} while (!atomic_try_cmpxchg_relaxed(&lock->val, &old, new));
	156
	157	return old;
	158	out:
	159	bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
	160	return old;
	161	}
	162
	163	/**
	164	* clear_pending - clear the pending bit.
	165	* @lock: Pointer to queued spinlock structure
	166	*
	167	* ,1, -> ,0,
	168	*/
	169	static __always_inline void clear_pending(arena_spinlock_t __arena *lock)
	170	{
	171	WRITE_ONCE(lock->pending, 0);
	172	}
	173
	174	/**
	175	* clear_pending_set_locked - take ownership and clear the pending bit.
	176	* @lock: Pointer to queued spinlock structure
	177	*
	178	* ,1,0 -> ,0,1
	179	*
	180	* Lock stealing is not allowed if this function is used.
	181	*/
	182	static __always_inline void clear_pending_set_locked(arena_spinlock_t __arena *lock)
	183	{
	184	WRITE_ONCE(lock->locked_pending, _Q_LOCKED_VAL);
	185	}
	186
	187	/**
	188	* set_locked - Set the lock bit and own the lock
	189	* @lock: Pointer to queued spinlock structure
	190	*
	191	* ,,0 -> *,0,1
	192	*/
	193	static __always_inline void set_locked(arena_spinlock_t __arena *lock)
	194	{
	195	WRITE_ONCE(lock->locked, _Q_LOCKED_VAL);
	196	}
	197
	198	static __always_inline
	199	u32 arena_fetch_set_pending_acquire(arena_spinlock_t __arena *lock)
	200	{
	201	u32 old, new;
	202
	203	old = atomic_read(&lock->val);
	204	do {
	205	new = old \| _Q_PENDING_VAL;
	206	/*
	207	* These loops are not expected to stall, but we still need to
	208	* prove to the verifier they will terminate eventually.
	209	*/
	210	cond_break_label(out);
	211	} while (!atomic_try_cmpxchg_acquire(&lock->val, &old, new));
	212
	213	return old;
	214	out:
	215	bpf_printk("RUNTIME ERROR: %s unexpected cond_break exit!!!", __func__);
	216	return old;
	217	}
	218
	219	/**
	220	* arena_spin_trylock - try to acquire the queued spinlock
	221	* @lock : Pointer to queued spinlock structure
	222	* Return: 1 if lock acquired, 0 if failed
	223	*/
	224	static __always_inline int arena_spin_trylock(arena_spinlock_t __arena *lock)
	225	{
	226	int val = atomic_read(&lock->val);
	227
	228	if (unlikely(val))
	229	return 0;
	230
	231	return likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL));
	232	}
	233
	234	__noinline
	235	int arena_spin_lock_slowpath(arena_spinlock_t __arena __arg_arena *lock, u32 val)
	236	{
	237	struct arena_mcs_spinlock __arena prev, next, node0, node;
	238	int ret = -ETIMEDOUT;
	239	u32 old, tail;
	240	int idx;
	241
	242	/*
	243	* Wait for in-progress pending->locked hand-overs with a bounded
	244	* number of spins so that we guarantee forward progress.
	245	*
	246	* 0,1,0 -> 0,0,1
	247	*/
	248	if (val == _Q_PENDING_VAL) {
	249	int cnt = _Q_PENDING_LOOPS;
	250	val = atomic_cond_read_relaxed_label(&lock->val,
	251	(VAL != _Q_PENDING_VAL) \|\| !cnt--,
	252	release_err);
	253	}
	254
	255	/*
	256	* If we observe any contention; queue.
	257	*/
	258	if (val & ~_Q_LOCKED_MASK)
	259	goto queue;
	260
	261	/*
	262	* trylock \|\| pending
	263	*
	264	* 0,0,* -> 0,1,* -> 0,0,1 pending, trylock
	265	*/
	266	val = arena_fetch_set_pending_acquire(lock);
	267
	268	/*
	269	* If we observe contention, there is a concurrent locker.
	270	*
	271	* Undo and queue; our setting of PENDING might have made the
	272	* n,0,0 -> 0,0,0 transition fail and it will now be waiting
	273	* on @next to become !NULL.
	274	*/
	275	if (unlikely(val & ~_Q_LOCKED_MASK)) {
	276
	277	/* Undo PENDING if we set it. */
	278	if (!(val & _Q_PENDING_MASK))
	279	clear_pending(lock);
	280
	281	goto queue;
	282	}
	283
	284	/*
	285	* We're pending, wait for the owner to go away.
	286	*
	287	* 0,1,1 -> *,1,0
	288	*
	289	* this wait loop must be a load-acquire such that we match the
	290	* store-release that clears the locked bit and create lock
	291	* sequentiality; this is because not all
	292	* clear_pending_set_locked() implementations imply full
	293	* barriers.
	294	*/
	295	if (val & _Q_LOCKED_MASK)
	296	smp_cond_load_acquire_label(&lock->locked, !VAL, release_err);
	297
	298	/*
	299	* take ownership and clear the pending bit.
	300	*
	301	* 0,1,0 -> 0,0,1
	302	*/
	303	clear_pending_set_locked(lock);
	304	return 0;
	305
	306	/*
	307	* End of pending bit optimistic spinning and beginning of MCS
	308	* queuing.
	309	*/
	310	queue:
	311	node0 = &(qnodes[bpf_get_smp_processor_id()])[0].mcs;
	312	idx = node0->count++;
	313	tail = encode_tail(bpf_get_smp_processor_id(), idx);
	314
	315	/*
	316	* 4 nodes are allocated based on the assumption that there will not be
	317	* nested NMIs taking spinlocks. That may not be true in some
	318	* architectures even though the chance of needing more than 4 nodes
	319	* will still be extremely unlikely. When that happens, we simply return
	320	* an error. Original qspinlock has a trylock fallback in this case.
	321	*/
	322	if (unlikely(idx >= _Q_MAX_NODES)) {
	323	ret = -EBUSY;
	324	goto release_node_err;
	325	}
	326
	327	node = grab_mcs_node(node0, idx);
	328
	329	/*
	330	* Ensure that we increment the head node->count before initialising
	331	* the actual node. If the compiler is kind enough to reorder these
	332	* stores, then an IRQ could overwrite our assignments.
	333	*/
	334	barrier();
	335
	336	node->locked = 0;
	337	node->next = NULL;
	338
	339	/*
	340	* We touched a (possibly) cold cacheline in the per-cpu queue node;
	341	* attempt the trylock once more in the hope someone let go while we
	342	* weren't watching.
	343	*/
	344	if (arena_spin_trylock(lock))
	345	goto release;
	346
	347	/*
	348	* Ensure that the initialisation of @node is complete before we
	349	* publish the updated tail via xchg_tail() and potentially link
	350	* @node into the waitqueue via WRITE_ONCE(prev->next, node) below.
	351	*/
	352	smp_wmb();
	353
	354	/*
	355	* Publish the updated tail.
	356	* We have already touched the queueing cacheline; don't bother with
	357	* pending stuff.
	358	*
	359	* p,, -> n,,
	360	*/
	361	old = xchg_tail(lock, tail);
	362	next = NULL;
	363
	364	/*
	365	* if there was a previous node; link it and wait until reaching the
	366	* head of the waitqueue.
	367	*/
	368	if (old & _Q_TAIL_MASK) {
	369	prev = decode_tail(old);
	370
	371	/* Link @node into the waitqueue. */
	372	WRITE_ONCE(prev->next, node);
	373
	374	arch_mcs_spin_lock_contended_label(&node->locked, release_node_err);
	375
	376	/*
	377	* While waiting for the MCS lock, the next pointer may have
	378	* been set by another lock waiter. We cannot prefetch here
	379	* due to lack of equivalent instruction in BPF ISA.
	380	*/
	381	next = READ_ONCE(node->next);
	382	}
	383
	384	/*
	385	* we're at the head of the waitqueue, wait for the owner & pending to
	386	* go away.
	387	*
	388	* ,x,y -> ,0,0
	389	*
	390	* this wait loop must use a load-acquire such that we match the
	391	* store-release that clears the locked bit and create lock
	392	* sequentiality; this is because the set_locked() function below
	393	* does not imply a full barrier.
	394	*/
	395	val = atomic_cond_read_acquire_label(&lock->val, !(VAL & _Q_LOCKED_PENDING_MASK),
	396	release_node_err);
	397
	398	/*
	399	* claim the lock:
	400	*
	401	* n,0,0 -> 0,0,1 : lock, uncontended
	402	* ,,0 -> ,,1 : lock, contended
	403	*
	404	* If the queue head is the only one in the queue (lock value == tail)
	405	* and nobody is pending, clear the tail code and grab the lock.
	406	* Otherwise, we only need to grab the lock.
	407	*/
	408
	409	/*
	410	* In the PV case we might already have _Q_LOCKED_VAL set, because
	411	* of lock stealing; therefore we must also allow:
	412	*
	413	* n,0,1 -> 0,0,1
	414	*
	415	* Note: at this point: (val & _Q_PENDING_MASK) == 0, because of the
	416	* above wait condition, therefore any concurrent setting of
	417	* PENDING will make the uncontended transition fail.
	418	*/
	419	if ((val & _Q_TAIL_MASK) == tail) {
	420	if (atomic_try_cmpxchg_relaxed(&lock->val, &val, _Q_LOCKED_VAL))
	421	goto release; /* No contention */
	422	}
	423
	424	/*
	425	* Either somebody is queued behind us or _Q_PENDING_VAL got set
	426	* which will then detect the remaining tail and queue behind us
	427	* ensuring we'll see a @next.
	428	*/
	429	set_locked(lock);
	430
	431	/*
	432	* contended path; wait for next if not observed yet, release.
	433	*/
	434	if (!next)
	435	next = smp_cond_load_relaxed_label(&node->next, (VAL), release_node_err);
	436
	437	arch_mcs_spin_unlock_contended(&next->locked);
	438
	439	release:;
	440	/*
	441	* release the node
	442	*
	443	* Doing a normal dec vs this_cpu_dec is fine. An upper context always
	444	* decrements count it incremented before returning, thus we're fine.
	445	* For contexts interrupting us, they either observe our dec or not.
	446	* Just ensure the compiler doesn't reorder this statement, as a
	447	* this_cpu_dec implicitly implied that.
	448	*/
	449	barrier();
	450	node0->count--;
	451	return 0;
	452	release_node_err:
	453	barrier();
	454	node0->count--;
	455	goto release_err;
	456	release_err:
	457	return ret;
	458	}
	459
	460	/**
	461	* arena_spin_lock - acquire a queued spinlock
	462	* @lock: Pointer to queued spinlock structure
	463	*
	464	* On error, returned value will be negative.
	465	* On success, zero is returned.
	466	*
	467	* The return value _must_ be tested against zero for success,
	468	* instead of checking it against negative, for passing the
	469	* BPF verifier.
	470	*
	471	* The user should do:
	472	* if (arena_spin_lock(...) != 0) // failure
	473	* or
	474	* if (arena_spin_lock(...) == 0) // success
	475	* or
	476	* if (arena_spin_lock(...)) // failure
	477	* or
	478	* if (!arena_spin_lock(...)) // success
	479	* instead of:
	480	* if (arena_spin_lock(...) < 0) // failure
	481	*
	482	* The return value can still be inspected later.
	483	*/
	484	static __always_inline int arena_spin_lock(arena_spinlock_t __arena *lock)
	485	{
	486	int val = 0;
	487
	488	if (CONFIG_NR_CPUS > 1024)
	489	return -EOPNOTSUPP;
	490
	491	bpf_preempt_disable();
	492	if (likely(atomic_try_cmpxchg_acquire(&lock->val, &val, _Q_LOCKED_VAL)))
	493	return 0;
	494
	495	val = arena_spin_lock_slowpath(lock, val);
	496	/* FIXME: bpf_assert_range(-MAX_ERRNO, 0) once we have it working for all cases. */
	497	if (val)
	498	bpf_preempt_enable();
	499	return val;
	500	}
	501
	502	/**
	503	* arena_spin_unlock - release a queued spinlock
	504	* @lock : Pointer to queued spinlock structure
	505	*/
	506	static __always_inline void arena_spin_unlock(arena_spinlock_t __arena *lock)
	507	{
	508	/*
	509	* unlock() needs release semantics:
	510	*/
	511	smp_store_release(&lock->locked, 0);
	512	bpf_preempt_enable();
	513	}
	514
	515	#define arena_spin_lock_irqsave(lock, flags) \
	516	({ \
	517	int __ret; \
	518	bpf_local_irq_save(&(flags)); \
	519	__ret = arena_spin_lock((lock)); \
	520	if (__ret) \
	521	bpf_local_irq_restore(&(flags)); \
	522	(__ret); \
	523	})
	524
	525	#define arena_spin_unlock_irqrestore(lock, flags) \
	526	({ \
	527	arena_spin_unlock((lock)); \
	528	bpf_local_irq_restore(&(flags)); \
	529	})
	530
	531	#endif
	532
	533	#endif /* BPF_ARENA_SPIN_LOCK_H */

tools/testing/selftests/bpf/bpf_atomic.h+140−0

@@ -0,0 +1,140 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#ifndef BPF_ATOMIC_H
	4	#define BPF_ATOMIC_H
	5
	6	#include <vmlinux.h>
	7	#include <bpf/bpf_helpers.h>
	8	#include "bpf_experimental.h"
	9
	10	extern bool CONFIG_X86_64 __kconfig __weak;
	11
	12	/*
	13	* __unqual_typeof(x) - Declare an unqualified scalar type, leaving
	14	* non-scalar types unchanged,
	15	*
	16	* Prefer C11 _Generic for better compile-times and simpler code. Note: 'char'
	17	* is not type-compatible with 'signed char', and we define a separate case.
	18	*
	19	* This is copied verbatim from kernel's include/linux/compiler_types.h, but
	20	* with default expression (for pointers) changed from (x) to (typeof(x)0).
	21	*
	22	* This is because LLVM has a bug where for lvalue (x), it does not get rid of
	23	* an extra address_space qualifier, but does in case of rvalue (typeof(x)0).
	24	* Hence, for pointers, we need to create an rvalue expression to get the
	25	* desired type. See https://github.com/llvm/llvm-project/issues/53400.
	26	*/
	27	#define __scalar_type_to_expr_cases(type) \
	28	unsigned type : (unsigned type)0, signed type : (signed type)0
	29
	30	#define __unqual_typeof(x) \
	31	typeof(_Generic((x), \
	32	char: (char)0, \
	33	__scalar_type_to_expr_cases(char), \
	34	__scalar_type_to_expr_cases(short), \
	35	__scalar_type_to_expr_cases(int), \
	36	__scalar_type_to_expr_cases(long), \
	37	__scalar_type_to_expr_cases(long long), \
	38	default: (typeof(x))0))
	39
	40	/* No-op for BPF */
	41	#define cpu_relax() ({})
	42
	43	#define READ_ONCE(x) ((volatile typeof(x) )&(x))
	44
	45	#define WRITE_ONCE(x, val) (((volatile typeof(x) )&(x)) = (val))
	46
	47	#define cmpxchg(p, old, new) __sync_val_compare_and_swap((p), old, new)
	48
	49	#define try_cmpxchg(p, pold, new) \
	50	({ \
	51	__unqual_typeof((pold)) __o = (pold); \
	52	__unqual_typeof(*(p)) __r = cmpxchg(p, __o, new); \
	53	if (__r != __o) \
	54	*(pold) = __r; \
	55	__r == __o; \
	56	})
	57
	58	#define try_cmpxchg_relaxed(p, pold, new) try_cmpxchg(p, pold, new)
	59
	60	#define try_cmpxchg_acquire(p, pold, new) try_cmpxchg(p, pold, new)
	61
	62	#define smp_mb() \
	63	({ \
	64	unsigned long __val; \
	65	__sync_fetch_and_add(&__val, 0); \
	66	})
	67
	68	#define smp_rmb() \
	69	({ \
	70	if (!CONFIG_X86_64) \
	71	smp_mb(); \
	72	else \
	73	barrier(); \
	74	})
	75
	76	#define smp_wmb() \
	77	({ \
	78	if (!CONFIG_X86_64) \
	79	smp_mb(); \
	80	else \
	81	barrier(); \
	82	})
	83
	84	/* Control dependency provides LOAD->STORE, provide LOAD->LOAD */
	85	#define smp_acquire__after_ctrl_dep() ({ smp_rmb(); })
	86
	87	#define smp_load_acquire(p) \
	88	({ \
	89	__unqual_typeof((p)) __v = READ_ONCE((p)); \
	90	if (!CONFIG_X86_64) \
	91	smp_mb(); \
	92	barrier(); \
	93	__v; \
	94	})
	95
	96	#define smp_store_release(p, val) \
	97	({ \
	98	if (!CONFIG_X86_64) \
	99	smp_mb(); \
	100	barrier(); \
	101	WRITE_ONCE(*(p), val); \
	102	})
	103
	104	#define smp_cond_load_relaxed_label(p, cond_expr, label) \
	105	({ \
	106	typeof(p) __ptr = (p); \
	107	__unqual_typeof(*(p)) VAL; \
	108	for (;;) { \
	109	VAL = (__unqual_typeof((p)))READ_ONCE(__ptr); \
	110	if (cond_expr) \
	111	break; \
	112	cond_break_label(label); \
	113	cpu_relax(); \
	114	} \
	115	(typeof(*(p)))VAL; \
	116	})
	117
	118	#define smp_cond_load_acquire_label(p, cond_expr, label) \
	119	({ \
	120	__unqual_typeof(*p) __val = \
	121	smp_cond_load_relaxed_label(p, cond_expr, label); \
	122	smp_acquire__after_ctrl_dep(); \
	123	(typeof(*(p)))__val; \
	124	})
	125
	126	#define atomic_read(p) READ_ONCE((p)->counter)
	127
	128	#define atomic_cond_read_relaxed_label(p, cond_expr, label) \
	129	smp_cond_load_relaxed_label(&(p)->counter, cond_expr, label)
	130
	131	#define atomic_cond_read_acquire_label(p, cond_expr, label) \
	132	smp_cond_load_acquire_label(&(p)->counter, cond_expr, label)
	133
	134	#define atomic_try_cmpxchg_relaxed(p, pold, new) \
	135	try_cmpxchg_relaxed(&(p)->counter, pold, new)
	136
	137	#define atomic_try_cmpxchg_acquire(p, pold, new) \
	138	try_cmpxchg_acquire(&(p)->counter, pold, new)
	139
	140	#endif /* BPF_ATOMIC_H */

tools/testing/selftests/bpf/bpf_experimental.h+9−6

 	ret;						\
 	})
-#define cond_break					\
+#define __cond_break(expr)				\
 	({ __label__ l_break, l_continue;		\
 	asm volatile goto("may_goto %l[l_break]"	\
 		      :::: l_break);			\
 	goto l_continue;				\
-	l_break: break;					\
+	l_break: expr;					\
 	l_continue:;					\
 	})
 #else
 	ret;						\
 	})
-#define cond_break					\
+#define __cond_break(expr)				\
 	({ __label__ l_break, l_continue;		\
 	asm volatile goto("1:.byte 0xe5;		\
 		      .byte 0;				\
 		      .short 0"				\
 		      :::: l_break);			\
 	goto l_continue;				\
-	l_break: break;					\
+	l_break: expr;					\
 	l_continue:;					\
 	})
 #else
 	ret;						\
 	})
-#define cond_break					\
+#define __cond_break(expr)				\
 	({ __label__ l_break, l_continue;		\
 	asm volatile goto("1:.byte 0xe5;		\
 		      .byte 0;				\
 		      .short 0"				\
 		      :::: l_break);			\
 	goto l_continue;				\
-	l_break: break;					\
+	l_break: expr;					\
 	l_continue:;					\
 	})
 #endif
 #endif
+#define cond_break __cond_break(break)
+#define cond_break_label(label) __cond_break(goto label)
 #ifndef bpf_nop_mov
 #define bpf_nop_mov(var) \
 	asm volatile("%[reg]=%[reg]"::[reg]"r"((short)var))

tools/testing/selftests/bpf/bpf_kfuncs.h+5−0

@@ -87,4 +87,9 @@ struct dentry; · 86 unmodified lines
87	*/	87	*/
88	extern int bpf_get_dentry_xattr(struct dentry dentry, const char name,	88	extern int bpf_get_dentry_xattr(struct dentry dentry, const char name,
89	struct bpf_dynptr *value_ptr) __ksym __weak;	89	struct bpf_dynptr *value_ptr) __ksym __weak;
		90
		91	extern int bpf_set_dentry_xattr(struct dentry dentry, const char name__str,
		92	const struct bpf_dynptr *value_p, int flags) __ksym __weak;
		93	extern int bpf_remove_dentry_xattr(struct dentry dentry, const char name__str) __ksym __weak;
		94
90	#endif	95	#endif

tools/testing/selftests/bpf/cap_helpers.c+4−4

@@ -19,7 +19,7 @@ int cap_enable_effective(__u64 caps, __u64 *old_caps) · 18 unmodified lines
19		19
20	err = capget(&hdr, data);	20	err = capget(&hdr, data);
21	if (err)	21	if (err)
22	return err;	22	return -errno;
23		23
24	if (old_caps)	24	if (old_caps)
25	*old_caps = (__u64)(data[1].effective) << 32 \| data[0].effective;	25	*old_caps = (__u64)(data[1].effective) << 32 \| data[0].effective;
@@ -32,7 +32,7 @@ int cap_enable_effective(__u64 caps, __u64 *old_caps) · 6 unmodified lines
32	data[1].effective \|= cap1;	32	data[1].effective \|= cap1;
33	err = capset(&hdr, data);	33	err = capset(&hdr, data);
34	if (err)	34	if (err)
35	return err;	35	return -errno;
36		36
37	return 0;	37	return 0;
38	}	38	}
@@ -49,7 +49,7 @@ int cap_disable_effective(__u64 caps, __u64 *old_caps) · 10 unmodified lines
49		49
50	err = capget(&hdr, data);	50	err = capget(&hdr, data);
51	if (err)	51	if (err)
52	return err;	52	return -errno;
53		53
54	if (old_caps)	54	if (old_caps)
55	*old_caps = (__u64)(data[1].effective) << 32 \| data[0].effective;	55	*old_caps = (__u64)(data[1].effective) << 32 \| data[0].effective;
@@ -61,7 +61,7 @@ int cap_disable_effective(__u64 caps, __u64 *old_caps) · 5 unmodified lines
61	data[1].effective &= ~cap1;	61	data[1].effective &= ~cap1;
62	err = capset(&hdr, data);	62	err = capset(&hdr, data);
63	if (err)	63	if (err)
64	return err;	64	return -errno;
65		65
66	return 0;	66	return 0;
67	}	67	}

tools/testing/selftests/bpf/cap_helpers.h+1−0

@@ -4,6 +4,7 @@ · 3 unmodified lines
4		4
5	#include <linux/types.h>	5	#include <linux/types.h>
6	#include <linux/capability.h>	6	#include <linux/capability.h>
		7	#include <errno.h>
7		8
8	#ifndef CAP_PERFMON	9	#ifndef CAP_PERFMON
9	#define CAP_PERFMON 38	10	#define CAP_PERFMON 38

tools/testing/selftests/bpf/network_helpers.c+76−35

 	return "ping";
 }
+int append_tid(char *str, size_t sz)
+{
+	size_t end;
+	if (!str)
+		return -1;
+	end = strlen(str);
+	if (end + 8 > sz)
+		return -1;
+	sprintf(&str[end], "%07d", gettid());
+	str[end + 7] = '\0';
+	return 0;
+}
 int remove_netns(const char *name)
 {
 	char *cmd;
 	int pcap_fd;
 };
+static int __base_pr(const char *format, va_list args)
+{
+	return vfprintf(stdout, format, args);
+}
+static tm_print_fn_t __tm_pr = __base_pr;
+tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn)
+{
+	tm_print_fn_t old_print_fn;
+	old_print_fn = __atomic_exchange_n(&__tm_pr, fn, __ATOMIC_RELAXED);
+	return old_print_fn;
+}
+void tm_print(const char *format, ...)
+{
+	tm_print_fn_t print_fn;
+	va_list args;
+	print_fn = __atomic_load_n(&__tm_pr, __ATOMIC_RELAXED);
+	if (!print_fn)
+		return;
+	va_start(args, format);
+	print_fn(format, args);
+	va_end(args);
+}
 /* Is this packet captured with a Ethernet protocol type? */
 static bool is_ethernet(const u_char *packet)
 {
 	case 770: /* ARPHRD_FRAD */
 	case 778: /* ARPHDR_IPGRE */
 	case 803: /* ARPHRD_IEEE80211_RADIOTAP */
-		printf("Packet captured: arphdr_type=%d\n", arphdr_type);
+		tm_print("Packet captured: arphdr_type=%d\n", arphdr_type);
 		return false;
 	}
 	return true;
 	return "Unknown";
 }
+#define MAX_FLAGS_STRLEN 21
 /* Show the information of the transport layer in the packet */
 static void show_transport(const u_char *packet, u16 len, u32 ifindex,
 			   const char *src_addr, const char *dst_addr,
 			   u16 proto, bool ipv6, u8 pkt_type)
 {
-	char *ifname, _ifname[IF_NAMESIZE];
+	char *ifname, _ifname[IF_NAMESIZE], flags[MAX_FLAGS_STRLEN] = "";
 	const char *transport_str;
 	u16 src_port, dst_port;
 	struct udphdr *udp;
 		dst_port = ntohs(tcp->dest);
 		transport_str = "TCP";
 	} else if (proto == IPPROTO_ICMP) {
-		printf("%-7s %-3s IPv4 %s > %s: ICMP, length %d, type %d, code %d\n",
+		tm_print("%-7s %-3s IPv4 %s > %s: ICMP, length %d, type %d, code %d\n",
-		       ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
+			 ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
-		       packet[0], packet[1]);
+			 packet[0], packet[1]);
 		return;
 	} else if (proto == IPPROTO_ICMPV6) {
-		printf("%-7s %-3s IPv6 %s > %s: ICMPv6, length %d, type %d, code %d\n",
+		tm_print("%-7s %-3s IPv6 %s > %s: ICMPv6, length %d, type %d, code %d\n",
-		       ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
+			 ifname, pkt_type_str(pkt_type), src_addr, dst_addr, len,
-		       packet[0], packet[1]);
+			 packet[0], packet[1]);
 		return;
 	} else {
-		printf("%-7s %-3s %s %s > %s: protocol %d\n",
+		tm_print("%-7s %-3s %s %s > %s: protocol %d\n",
-		       ifname, pkt_type_str(pkt_type), ipv6 ? "IPv6" : "IPv4",
+			 ifname, pkt_type_str(pkt_type), ipv6 ? "IPv6" : "IPv4",
-		       src_addr, dst_addr, proto);
+			 src_addr, dst_addr, proto);
 		return;
 	}
 	/* TCP or UDP*/
-	flockfile(stdout);
+	if (proto == IPPROTO_TCP)
+		snprintf(flags, MAX_FLAGS_STRLEN, "%s%s%s%s",
+			 tcp->fin ? ", FIN" : "",
+			 tcp->syn ? ", SYN" : "",
+			 tcp->rst ? ", RST" : "",
+			 tcp->ack ? ", ACK" : "");
 	if (ipv6)
-		printf("%-7s %-3s IPv6 %s.%d > %s.%d: %s, length %d",
+		tm_print("%-7s %-3s IPv6 %s.%d > %s.%d: %s, length %d%s\n",
-		       ifname, pkt_type_str(pkt_type), src_addr, src_port,
+			 ifname, pkt_type_str(pkt_type), src_addr, src_port,
-		       dst_addr, dst_port, transport_str, len);
+			 dst_addr, dst_port, transport_str, len, flags);
 	else
-		printf("%-7s %-3s IPv4 %s:%d > %s:%d: %s, length %d",
+		tm_print("%-7s %-3s IPv4 %s:%d > %s:%d: %s, length %d%s\n",
-		       ifname, pkt_type_str(pkt_type), src_addr, src_port,
+			 ifname, pkt_type_str(pkt_type), src_addr, src_port,
-		       dst_addr, dst_port, transport_str, len);
+			 dst_addr, dst_port, transport_str, len, flags);
-	if (proto == IPPROTO_TCP) {
-		if (tcp->fin)
-			printf(", FIN");
-		if (tcp->syn)
-			printf(", SYN");
-		if (tcp->rst)
-			printf(", RST");
-		if (tcp->ack)
-			printf(", ACK");
-	}
-	printf("\n");
-	funlockfile(stdout);
 }
 static void show_ipv6_packet(const u_char *packet, u32 ifindex, u8 pkt_type)
 				ifname = _ifname;
 			}
-			printf("%-7s %-3s Unknown network protocol type 0x%x\n",
+			tm_print("%-7s %-3s Unknown network protocol type 0x%x\n",
-			       ifname, pkt_type_str(ptype), proto);
+				 ifname, pkt_type_str(ptype), proto);
 		}
 	}
 	write(ctx->wake_fd, &w, sizeof(w));
 	pthread_join(ctx->thread, NULL);
-	printf("Packet file: %s\n", strrchr(ctx->pkt_fname, '/') + 1);
+	tm_print("Packet file: %s\n", strrchr(ctx->pkt_fname, '/') + 1);
 	traffic_monitor_release(ctx);
 }
 #endif /* TRAFFIC_MONITOR */

tools/testing/selftests/bpf/network_helpers.h+21−0

 #include <netinet/udp.h>
 #include <bpf/bpf_endian.h>
 #include <net/if.h>
+#include <stdio.h>
 #define MAGIC_VAL 0x1234
 #define NUM_ITER 100000
 int make_netns(const char *name);
 int remove_netns(const char *name);
+/**
+ * append_tid() - Append thread ID to the given string.
+ *
+ * @str: string to extend
+ * @sz: string's size
+ *
+ * 8 characters are used to append the thread ID (7 digits + '\0')
+ *
+ * Returns -1 on errors, 0 otherwise
+ */
+int append_tid(char *str, size_t sz);
 static __u16 csum_fold(__u32 csum)
 {
 	csum = (csum & 0xffff) + (csum >> 16);
 struct tmonitor_ctx;
+typedef int (*tm_print_fn_t)(const char *format, va_list args);
 #ifdef TRAFFIC_MONITOR
 struct tmonitor_ctx *traffic_monitor_start(const char *netns, const char *test_name,
 					   const char *subtest_name);
 void traffic_monitor_stop(struct tmonitor_ctx *ctx);
+tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn);
 #else
 static inline struct tmonitor_ctx *traffic_monitor_start(const char *netns, const char *test_name,
 							 const char *subtest_name)
 static inline void traffic_monitor_stop(struct tmonitor_ctx *ctx)
 {
 }
+static inline tm_print_fn_t traffic_monitor_set_print(tm_print_fn_t fn)
+{
+	return NULL;
+}
 #endif
 #endif

tools/testing/selftests/bpf/prog_tests/align.c+10−1

 		.log_size = sizeof(bpf_vlog),
 		.log_level = 2,
 	);
+	const char *main_pass_start = "0: R1=ctx() R10=fp0";
 	const char *line_ptr;
 	int cur_line = -1;
 	int prog_len, i;
+	char *start;
 	int fd_prog;
 	int ret;
 		ret = 0;
 		/* We make a local copy so that we can strtok() it */
 		strncpy(bpf_vlog_copy, bpf_vlog, sizeof(bpf_vlog_copy));
-		line_ptr = strtok(bpf_vlog_copy, "\n");
+		start = strstr(bpf_vlog_copy, main_pass_start);
+		if (!start) {
+			ret = 1;
+			printf("Can't find initial line '%s'\n", main_pass_start);
+			goto out;
+		}
+		line_ptr = strtok(start, "\n");
 		for (i = 0; i < MAX_MATCHES; i++) {
 			struct bpf_reg_match m = test->matches[i];
 			const char *p;
 				break;
 			}
 		}
+out:
 		if (fd_prog >= 0)
 			close(fd_prog);
 	}

tools/testing/selftests/bpf/prog_tests/arena_atomics.c+65−1

@@ -162,6 +162,66 @@ static void test_uaf(struct arena_atomics *skel) · 161 unmodified lines
162	ASSERT_EQ(skel->arena->uaf_recovery_fails, 0, "uaf_recovery_fails");	162	ASSERT_EQ(skel->arena->uaf_recovery_fails, 0, "uaf_recovery_fails");
163	}	163	}
164		164
		165	static void test_load_acquire(struct arena_atomics *skel)
		166	{
		167	LIBBPF_OPTS(bpf_test_run_opts, topts);
		168	int err, prog_fd;
		169
		170	if (skel->data->skip_lacq_srel_tests) {
		171	printf("%s:SKIP: ENABLE_ATOMICS_TESTS not defined, Clang doesn't support addr_space_cast, and/or JIT doesn't support load-acquire\n",
		172	__func__);
		173	test__skip();
		174	return;
		175	}
		176
		177	/* No need to attach it, just run it directly */
		178	prog_fd = bpf_program__fd(skel->progs.load_acquire);
		179	err = bpf_prog_test_run_opts(prog_fd, &topts);
		180	if (!ASSERT_OK(err, "test_run_opts err"))
		181	return;
		182	if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
		183	return;
		184
		185	ASSERT_EQ(skel->arena->load_acquire8_result, 0x12,
		186	"load_acquire8_result");
		187	ASSERT_EQ(skel->arena->load_acquire16_result, 0x1234,
		188	"load_acquire16_result");
		189	ASSERT_EQ(skel->arena->load_acquire32_result, 0x12345678,
		190	"load_acquire32_result");
		191	ASSERT_EQ(skel->arena->load_acquire64_result, 0x1234567890abcdef,
		192	"load_acquire64_result");
		193	}
		194
		195	static void test_store_release(struct arena_atomics *skel)
		196	{
		197	LIBBPF_OPTS(bpf_test_run_opts, topts);
		198	int err, prog_fd;
		199
		200	if (skel->data->skip_lacq_srel_tests) {
		201	printf("%s:SKIP: ENABLE_ATOMICS_TESTS not defined, Clang doesn't support addr_space_cast, and/or JIT doesn't support store-release\n",
		202	__func__);
		203	test__skip();
		204	return;
		205	}
		206
		207	/* No need to attach it, just run it directly */
		208	prog_fd = bpf_program__fd(skel->progs.store_release);
		209	err = bpf_prog_test_run_opts(prog_fd, &topts);
		210	if (!ASSERT_OK(err, "test_run_opts err"))
		211	return;
		212	if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
		213	return;
		214
		215	ASSERT_EQ(skel->arena->store_release8_result, 0x12,
		216	"store_release8_result");
		217	ASSERT_EQ(skel->arena->store_release16_result, 0x1234,
		218	"store_release16_result");
		219	ASSERT_EQ(skel->arena->store_release32_result, 0x12345678,
		220	"store_release32_result");
		221	ASSERT_EQ(skel->arena->store_release64_result, 0x1234567890abcdef,
		222	"store_release64_result");
		223	}
		224
165	void test_arena_atomics(void)	225	void test_arena_atomics(void)
166	{	226	{
167	struct arena_atomics *skel;	227	struct arena_atomics *skel;
@@ -171,7 +231,7 @@ void test_arena_atomics(void) · 3 unmodified lines
171	if (!ASSERT_OK_PTR(skel, "arena atomics skeleton open"))	231	if (!ASSERT_OK_PTR(skel, "arena atomics skeleton open"))
172	return;	232	return;
173		233
174	if (skel->data->skip_tests) {	234	if (skel->data->skip_all_tests) {
175	printf("%s:SKIP:no ENABLE_ATOMICS_TESTS or no addr_space_cast support in clang",	235	printf("%s:SKIP:no ENABLE_ATOMICS_TESTS or no addr_space_cast support in clang",
176	__func__);	236	__func__);
177	test__skip();	237	test__skip();
@@ -198,6 +258,10 @@ void test_arena_atomics(void) · 20 unmodified lines
198	test_xchg(skel);	258	test_xchg(skel);
199	if (test__start_subtest("uaf"))	259	if (test__start_subtest("uaf"))
200	test_uaf(skel);	260	test_uaf(skel);
		261	if (test__start_subtest("load_acquire"))
		262	test_load_acquire(skel);
		263	if (test__start_subtest("store_release"))
		264	test_store_release(skel);
201		265
202	cleanup:	266	cleanup:
203	arena_atomics__destroy(skel);	267	arena_atomics__destroy(skel);

tools/testing/selftests/bpf/prog_tests/arena_spin_lock.c+108−0

@@ -0,0 +1,108 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#include <test_progs.h>
	4	#include <network_helpers.h>
	5	#include <sys/sysinfo.h>
	6
	7	struct __qspinlock { int val; };
	8	typedef struct __qspinlock arena_spinlock_t;
	9
	10	struct arena_qnode {
	11	unsigned long next;
	12	int count;
	13	int locked;
	14	};
	15
	16	#include "arena_spin_lock.skel.h"
	17
	18	static long cpu;
	19	static int repeat;
	20
	21	pthread_barrier_t barrier;
	22
	23	static void spin_lock_thread(void arg)
	24	{
	25	int err, prog_fd = (u32 )arg;
	26	LIBBPF_OPTS(bpf_test_run_opts, topts,
	27	.data_in = &pkt_v4,
	28	.data_size_in = sizeof(pkt_v4),
	29	.repeat = repeat,
	30	);
	31	cpu_set_t cpuset;
	32
	33	CPU_ZERO(&cpuset);
	34	CPU_SET(__sync_fetch_and_add(&cpu, 1), &cpuset);
	35	ASSERT_OK(pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset), "cpu affinity");
	36
	37	err = pthread_barrier_wait(&barrier);
	38	if (err != PTHREAD_BARRIER_SERIAL_THREAD && err != 0)
	39	ASSERT_FALSE(true, "pthread_barrier");
	40
	41	err = bpf_prog_test_run_opts(prog_fd, &topts);
	42	ASSERT_OK(err, "test_run err");
	43	ASSERT_EQ((int)topts.retval, 0, "test_run retval");
	44
	45	pthread_exit(arg);
	46	}
	47
	48	static void test_arena_spin_lock_size(int size)
	49	{
	50	LIBBPF_OPTS(bpf_test_run_opts, topts);
	51	struct arena_spin_lock *skel;
	52	pthread_t thread_id[16];
	53	int prog_fd, i, err;
	54	void *ret;
	55
	56	if (get_nprocs() < 2) {
	57	test__skip();
	58	return;
	59	}
	60
	61	skel = arena_spin_lock__open_and_load();
	62	if (!ASSERT_OK_PTR(skel, "arena_spin_lock__open_and_load"))
	63	return;
	64	if (skel->data->test_skip == 2) {
	65	test__skip();
	66	goto end;
	67	}
	68	skel->bss->cs_count = size;
	69	skel->bss->limit = repeat * 16;
	70
	71	ASSERT_OK(pthread_barrier_init(&barrier, NULL, 16), "barrier init");
	72
	73	prog_fd = bpf_program__fd(skel->progs.prog);
	74	for (i = 0; i < 16; i++) {
	75	err = pthread_create(&thread_id[i], NULL, &spin_lock_thread, &prog_fd);
	76	if (!ASSERT_OK(err, "pthread_create"))
	77	goto end_barrier;
	78	}
	79
	80	for (i = 0; i < 16; i++) {
	81	if (!ASSERT_OK(pthread_join(thread_id[i], &ret), "pthread_join"))
	82	goto end_barrier;
	83	if (!ASSERT_EQ(ret, &prog_fd, "ret == prog_fd"))
	84	goto end_barrier;
	85	}
	86
	87	ASSERT_EQ(skel->bss->counter, repeat * 16, "check counter value");
	88
	89	end_barrier:
	90	pthread_barrier_destroy(&barrier);
	91	end:
	92	arena_spin_lock__destroy(skel);
	93	return;
	94	}
	95
	96	void test_arena_spin_lock(void)
	97	{
	98	repeat = 1000;
	99	if (test__start_subtest("arena_spin_lock_1"))
	100	test_arena_spin_lock_size(1);
	101	cpu = 0;
	102	if (test__start_subtest("arena_spin_lock_1000"))
	103	test_arena_spin_lock_size(1000);
	104	cpu = 0;
	105	repeat = 100;
	106	if (test__start_subtest("arena_spin_lock_50000"))
	107	test_arena_spin_lock_size(50000);
	108	}

tools/testing/selftests/bpf/prog_tests/bloom_filter_map.c+5−0

 #include <test_progs.h>
 #include "bloom_filter_map.skel.h"
+#ifndef NUMA_NO_NODE
+#define NUMA_NO_NODE	(-1)
+#endif
 static void test_fail_cases(void)
 {
 	LIBBPF_OPTS(bpf_map_create_opts, opts);
 	/* Create a map */
 	opts.map_flags = BPF_F_ZERO_SEED | BPF_F_NUMA_NODE;
+	opts.numa_node = NUMA_NO_NODE;
 	fd = bpf_map_create(BPF_MAP_TYPE_BLOOM_FILTER, NULL, 0, sizeof(value), 100, &opts);
 	if (!ASSERT_GE(fd, 0, "bpf_map_create bloom filter success case"))
 		return;

tools/testing/selftests/bpf/prog_tests/bpf_iter.c+68−0

@@ -323,19 +323,87 @@ static void test_task_pidfd(void) · 322 unmodified lines
323	static void test_task_sleepable(void)	323	static void test_task_sleepable(void)
324	{	324	{
325	struct bpf_iter_tasks *skel;	325	struct bpf_iter_tasks *skel;
		326	int pid, status, err, data_pipe[2], finish_pipe[2], c;
		327	char *test_data = NULL;
		328	char *test_data_long = NULL;
		329	char *data[2];
		330
		331	if (!ASSERT_OK(pipe(data_pipe), "data_pipe") \|\|
		332	!ASSERT_OK(pipe(finish_pipe), "finish_pipe"))
		333	return;
326		334
327	skel = bpf_iter_tasks__open_and_load();	335	skel = bpf_iter_tasks__open_and_load();
328	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))	336	if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))
329	return;	337	return;
330		338
		339	pid = fork();
		340	if (!ASSERT_GE(pid, 0, "fork"))
		341	return;
		342
		343	if (pid == 0) {
		344	/* child */
		345	close(data_pipe[0]);
		346	close(finish_pipe[1]);
		347
		348	test_data = malloc(sizeof(char) * 10);
		349	strncpy(test_data, "test_data", 10);
		350	test_data[9] = '\0';
		351
		352	test_data_long = malloc(sizeof(char) * 5000);
		353	for (int i = 0; i < 5000; ++i) {
		354	if (i % 2 == 0)
		355	test_data_long[i] = 'b';
		356	else
		357	test_data_long[i] = 'a';
		358	}
		359	test_data_long[4999] = '\0';
		360
		361	data[0] = test_data;
		362	data[1] = test_data_long;
		363
		364	write(data_pipe[1], &data, sizeof(data));
		365
		366	/* keep child alive until after the test */
		367	err = read(finish_pipe[0], &c, 1);
		368	if (err != 1)
		369	exit(-1);
		370
		371	close(data_pipe[1]);
		372	close(finish_pipe[0]);
		373	_exit(0);
		374	}
		375
		376	/* parent */
		377	close(data_pipe[1]);
		378	close(finish_pipe[0]);
		379
		380	err = read(data_pipe[0], &data, sizeof(data));
		381	ASSERT_EQ(err, sizeof(data), "read_check");
		382
		383	skel->bss->user_ptr = data[0];
		384	skel->bss->user_ptr_long = data[1];
		385	skel->bss->pid = pid;
		386
331	do_dummy_read(skel->progs.dump_task_sleepable);	387	do_dummy_read(skel->progs.dump_task_sleepable);
332		388
333	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,	389	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,
334	"num_expected_failure_copy_from_user_task");	390	"num_expected_failure_copy_from_user_task");
335	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,	391	ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,
336	"num_success_copy_from_user_task");	392	"num_success_copy_from_user_task");
		393	ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task_str, 0,
		394	"num_expected_failure_copy_from_user_task_str");
		395	ASSERT_GT(skel->bss->num_success_copy_from_user_task_str, 0,
		396	"num_success_copy_from_user_task_str");
337		397
338	bpf_iter_tasks__destroy(skel);	398	bpf_iter_tasks__destroy(skel);
		399
		400	write(finish_pipe[1], &c, 1);
		401	err = waitpid(pid, &status, 0);
		402	ASSERT_EQ(err, pid, "waitpid");
		403	ASSERT_EQ(status, 0, "zero_child_exit");
		404
		405	close(data_pipe[0]);
		406	close(finish_pipe[1]);
339	}	407	}
340		408
341	static void test_task_stack(void)	409	static void test_task_stack(void)

tools/testing/selftests/bpf/prog_tests/bpf_nf.c+6−3

 	if (!ASSERT_OK(system(cmd), cmd))
 		goto end;
-	srv_port = (mode == TEST_XDP) ? 5005 : 5006;
+	srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", 0, TIMEOUT_MS);
-	srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", srv_port, TIMEOUT_MS);
 	if (!ASSERT_GE(srv_fd, 0, "start_server"))
 		goto end;
+	srv_port = get_socket_local_port(srv_fd);
+	if (!ASSERT_GE(srv_port, 0, "get_sock_local_port"))
+		goto end;
 	client_fd = connect_to_server(srv_fd);
 	if (!ASSERT_GE(client_fd, 0, "connect_to_server"))
 		goto end;
 	skel->bss->saddr = peer_addr.sin_addr.s_addr;
 	skel->bss->sport = peer_addr.sin_port;
 	skel->bss->daddr = peer_addr.sin_addr.s_addr;
-	skel->bss->dport = htons(srv_port);
+	skel->bss->dport = srv_port;
 	if (mode == TEST_XDP)
 		prog_fd = bpf_program__fd(skel->progs.nf_xdp_ct_test);

tools/testing/selftests/bpf/prog_tests/btf.c+19−4

 	.err_str = "vlen != 0",
 },
 {
-	.descr = "decl_tag test #8, invalid kflag",
+	.descr = "decl_tag test #8, tag with kflag",
 	.raw_types = {
 		BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),	/* [1] */
 		BTF_VAR_ENC(NAME_TBD, 1, 0),			/* [2] */
-		BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 1, 0), 2), (-1),
+		BTF_DECL_ATTR_ENC(NAME_TBD, 2, -1),
 		BTF_END_RAW,
 	},
 	BTF_STR_SEC("\0local\0tag1"),
 	.key_type_id = 1,
 	.value_type_id = 1,
 	.max_entries = 1,
-	.btf_load_err = true,
-	.err_str = "Invalid btf_info kind_flag",
 },
 {
 	.descr = "decl_tag test #9, var, invalid component_idx",
 	.btf_load_err = true,
 	.err_str = "Type tags don't precede modifiers",
 },
+{
+	.descr = "type_tag test #7, tag with kflag",
+	.raw_types = {
+		BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),	/* [1] */
+		BTF_TYPE_ATTR_ENC(NAME_TBD, 1),			/* [2] */
+		BTF_PTR_ENC(2),					/* [3] */
+		BTF_END_RAW,
+	},
+	BTF_STR_SEC("\0tag"),
+	.map_type = BPF_MAP_TYPE_ARRAY,
+	.map_name = "tag_type_check_btf",
+	.key_size = sizeof(int),
+	.value_size = 4,
+	.key_type_id = 1,
+	.value_type_id = 1,
+	.max_entries = 1,
+},
 {
 	.descr = "enum64 test #1, unsigned, size 8",
 	.raw_types = {

tools/testing/selftests/bpf/prog_tests/btf_dump.c+110−37

@@ -126,26 +126,69 @@ done: · 125 unmodified lines
126	return err;	126	return err;
127	}	127	}
128		128
129	static char *dump_buf;	129	struct test_ctx {
130	static size_t dump_buf_sz;	130	struct btf *btf;
131	static FILE *dump_buf_file;	131	struct btf_dump *d;
		132	char *dump_buf;
		133	size_t dump_buf_sz;
		134	FILE *dump_buf_file;
		135	};
132		136
133	static void test_btf_dump_incremental(void)	137	static void test_ctx__free(struct test_ctx *t)
134	{	138	{
135	struct btf *btf = NULL;	139	fclose(t->dump_buf_file);
136	struct btf_dump *d = NULL;	140	free(t->dump_buf);
137	int id, err, i;	141	btf_dump__free(t->d);
		142	btf__free(t->btf);
		143	}
138		144
139	dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz);	145	static int test_ctx__init(struct test_ctx *t)
140	if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream"))	146	{
141	return;	147	t->dump_buf_file = open_memstream(&t->dump_buf, &t->dump_buf_sz);
142	btf = btf__new_empty();	148	if (!ASSERT_OK_PTR(t->dump_buf_file, "dump_memstream"))
143	if (!ASSERT_OK_PTR(btf, "new_empty"))	149	return -1;
		150	t->btf = btf__new_empty();
		151	if (!ASSERT_OK_PTR(t->btf, "new_empty"))
144	goto err_out;	152	goto err_out;
145	d = btf_dump__new(btf, btf_dump_printf, dump_buf_file, NULL);	153	t->d = btf_dump__new(t->btf, btf_dump_printf, t->dump_buf_file, NULL);
146	if (!ASSERT_OK(libbpf_get_error(d), "btf_dump__new"))	154	if (!ASSERT_OK(libbpf_get_error(t->d), "btf_dump__new"))
147	goto err_out;	155	goto err_out;
148		156
		157	return 0;
		158
		159	err_out:
		160	test_ctx__free(t);
		161	return -1;
		162	}
		163
		164	static void test_ctx__dump_and_compare(struct test_ctx *t,
		165	const char *expected_output,
		166	const char *message)
		167	{
		168	int i, err;
		169
		170	for (i = 1; i < btf__type_cnt(t->btf); i++) {
		171	err = btf_dump__dump_type(t->d, i);
		172	ASSERT_OK(err, "dump_type_ok");
		173	}
		174
		175	fflush(t->dump_buf_file);
		176	t->dump_buf[t->dump_buf_sz] = 0; /* some libc implementations don't do this */
		177
		178	ASSERT_STREQ(t->dump_buf, expected_output, message);
		179	}
		180
		181	static void test_btf_dump_incremental(void)
		182	{
		183	struct test_ctx t = {};
		184	struct btf *btf;
		185	int id, err;
		186
		187	if (test_ctx__init(&t))
		188	return;
		189
		190	btf = t.btf;
		191
149	/* First, generate BTF corresponding to the following C code:	192	/* First, generate BTF corresponding to the following C code:
150	*	193	*
151	* enum x;	194	* enum x;
@@ -182,15 +225,7 @@ static void test_btf_dump_incremental(void) · 30 unmodified lines
182	err = btf__add_field(btf, "x", 4, 0, 0);	225	err = btf__add_field(btf, "x", 4, 0, 0);
183	ASSERT_OK(err, "field_ok");	226	ASSERT_OK(err, "field_ok");
184		227
185	for (i = 1; i < btf__type_cnt(btf); i++) {	228	test_ctx__dump_and_compare(&t,
186	err = btf_dump__dump_type(d, i);
187	ASSERT_OK(err, "dump_type_ok");
188	}
189
190	fflush(dump_buf_file);
191	dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
192
193	ASSERT_STREQ(dump_buf,
194	"enum x;\n"	229	"enum x;\n"
195	"\n"	230	"\n"
196	"enum x {\n"	231	"enum x {\n"
@@ -221,7 +256,7 @@ static void test_btf_dump_incremental(void) · 24 unmodified lines
221	* enum values don't conflict;	256	* enum values don't conflict;
222	*	257	*
223	*/	258	*/
224	fseek(dump_buf_file, 0, SEEK_SET);	259	fseek(t.dump_buf_file, 0, SEEK_SET);
225		260
226	id = btf__add_struct(btf, "s", 4);	261	id = btf__add_struct(btf, "s", 4);
227	ASSERT_EQ(id, 7, "struct_id");	262	ASSERT_EQ(id, 7, "struct_id");
@@ -232,14 +267,7 @@ static void test_btf_dump_incremental(void) · 4 unmodified lines
232	err = btf__add_field(btf, "s", 6, 64, 0);	267	err = btf__add_field(btf, "s", 6, 64, 0);
233	ASSERT_OK(err, "field_ok");	268	ASSERT_OK(err, "field_ok");
234		269
235	for (i = 1; i < btf__type_cnt(btf); i++) {	270	test_ctx__dump_and_compare(&t,
236	err = btf_dump__dump_type(d, i);
237	ASSERT_OK(err, "dump_type_ok");
238	}
239
240	fflush(dump_buf_file);
241	dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
242	ASSERT_STREQ(dump_buf,
243	"struct s___2 {\n"	271	"struct s___2 {\n"
244	" enum x x;\n"	272	" enum x x;\n"
245	" enum {\n"	273	" enum {\n"
@@ -248,11 +276,53 @@ static void test_btf_dump_incremental(void) · 2 unmodified lines
248	" struct s s;\n"	276	" struct s s;\n"
249	"};\n\n" , "c_dump1");	277	"};\n\n" , "c_dump1");
250		278
251	err_out:	279	test_ctx__free(&t);
252	fclose(dump_buf_file);	280	}
253	free(dump_buf);	281
254	btf_dump__free(d);	282	static void test_btf_dump_type_tags(void)
255	btf__free(btf);	283	{
		284	struct test_ctx t = {};
		285	struct btf *btf;
		286	int id, err;
		287
		288	if (test_ctx__init(&t))
		289	return;
		290
		291	btf = t.btf;
		292
		293	/* Generate BTF corresponding to the following C code:
		294	*
		295	* struct s {
		296	* void __attribute__((btf_type_tag(\"void_tag\"))) *p1;
		297	* void __attribute__((void_attr)) *p2;
		298	* };
		299	*
		300	*/
		301
		302	id = btf__add_type_tag(btf, "void_tag", 0);
		303	ASSERT_EQ(id, 1, "type_tag_id");
		304	id = btf__add_ptr(btf, id);
		305	ASSERT_EQ(id, 2, "void_ptr_id1");
		306
		307	id = btf__add_type_attr(btf, "void_attr", 0);
		308	ASSERT_EQ(id, 3, "type_attr_id");
		309	id = btf__add_ptr(btf, id);
		310	ASSERT_EQ(id, 4, "void_ptr_id2");
		311
		312	id = btf__add_struct(btf, "s", 8);
		313	ASSERT_EQ(id, 5, "struct_id");
		314	err = btf__add_field(btf, "p1", 2, 0, 0);
		315	ASSERT_OK(err, "field_ok1");
		316	err = btf__add_field(btf, "p2", 4, 0, 0);
		317	ASSERT_OK(err, "field_ok2");
		318
		319	test_ctx__dump_and_compare(&t,
		320	"struct s {\n"
		321	" void __attribute__((btf_type_tag(\"void_tag\"))) *p1;\n"
		322	" void __attribute__((void_attr)) *p2;\n"
		323	"};\n\n", "dump_and_compare");
		324
		325	test_ctx__free(&t);
256	}	326	}
257		327
258	#define STRSIZE 4096	328	#define STRSIZE 4096
@@ -874,6 +944,9 @@ void test_btf_dump() { · 615 unmodified lines
874	if (test__start_subtest("btf_dump: incremental"))	944	if (test__start_subtest("btf_dump: incremental"))
875	test_btf_dump_incremental();	945	test_btf_dump_incremental();
876		946
		947	if (test__start_subtest("btf_dump: type_tags"))
		948	test_btf_dump_type_tags();
		949
877	btf = libbpf_find_kernel_btf();	950	btf = libbpf_find_kernel_btf();
878	if (!ASSERT_OK_PTR(btf, "no kernel BTF found"))	951	if (!ASSERT_OK_PTR(btf, "no kernel BTF found"))
879	return;	952	return;

tools/testing/selftests/bpf/prog_tests/cgroup_preorder.c+128−0

@@ -0,0 +1,128 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#include <test_progs.h>
	4	#include "cgroup_helpers.h"
	5	#include "cgroup_preorder.skel.h"
	6
	7	static int run_getsockopt_test(int cg_parent, int cg_child, int sock_fd, bool all_preorder)
	8	{
	9	LIBBPF_OPTS(bpf_prog_attach_opts, opts);
	10	enum bpf_attach_type prog_c_atype, prog_c2_atype, prog_p_atype, prog_p2_atype;
	11	int prog_c_fd, prog_c2_fd, prog_p_fd, prog_p2_fd;
	12	struct cgroup_preorder *skel = NULL;
	13	struct bpf_program *prog;
	14	__u8 *result, buf;
	15	socklen_t optlen;
	16	int err = 0;
	17
	18	skel = cgroup_preorder__open_and_load();
	19	if (!ASSERT_OK_PTR(skel, "cgroup_preorder__open_and_load"))
	20	return 0;
	21
	22	buf = 0x00;
	23	err = setsockopt(sock_fd, SOL_IP, IP_TOS, &buf, 1);
	24	if (!ASSERT_OK(err, "setsockopt"))
	25	goto close_skel;
	26
	27	opts.flags = BPF_F_ALLOW_MULTI;
	28	if (all_preorder)
	29	opts.flags \|= BPF_F_PREORDER;
	30	prog = skel->progs.child;
	31	prog_c_fd = bpf_program__fd(prog);
	32	prog_c_atype = bpf_program__expected_attach_type(prog);
	33	err = bpf_prog_attach_opts(prog_c_fd, cg_child, prog_c_atype, &opts);
	34	if (!ASSERT_OK(err, "bpf_prog_attach_opts-child"))
	35	goto close_skel;
	36
	37	opts.flags = BPF_F_ALLOW_MULTI \| BPF_F_PREORDER;
	38	prog = skel->progs.child_2;
	39	prog_c2_fd = bpf_program__fd(prog);
	40	prog_c2_atype = bpf_program__expected_attach_type(prog);
	41	err = bpf_prog_attach_opts(prog_c2_fd, cg_child, prog_c2_atype, &opts);
	42	if (!ASSERT_OK(err, "bpf_prog_attach_opts-child_2"))
	43	goto detach_child;
	44
	45	optlen = 1;
	46	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	47	if (!ASSERT_OK(err, "getsockopt"))
	48	goto detach_child_2;
	49
	50	result = skel->bss->result;
	51	if (all_preorder)
	52	ASSERT_TRUE(result[0] == 1 && result[1] == 2, "child only");
	53	else
	54	ASSERT_TRUE(result[0] == 2 && result[1] == 1, "child only");
	55
	56	skel->bss->idx = 0;
	57	memset(result, 0, 4);
	58
	59	opts.flags = BPF_F_ALLOW_MULTI;
	60	if (all_preorder)
	61	opts.flags \|= BPF_F_PREORDER;
	62	prog = skel->progs.parent;
	63	prog_p_fd = bpf_program__fd(prog);
	64	prog_p_atype = bpf_program__expected_attach_type(prog);
	65	err = bpf_prog_attach_opts(prog_p_fd, cg_parent, prog_p_atype, &opts);
	66	if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent"))
	67	goto detach_child_2;
	68
	69	opts.flags = BPF_F_ALLOW_MULTI \| BPF_F_PREORDER;
	70	prog = skel->progs.parent_2;
	71	prog_p2_fd = bpf_program__fd(prog);
	72	prog_p2_atype = bpf_program__expected_attach_type(prog);
	73	err = bpf_prog_attach_opts(prog_p2_fd, cg_parent, prog_p2_atype, &opts);
	74	if (!ASSERT_OK(err, "bpf_prog_attach_opts-parent_2"))
	75	goto detach_parent;
	76
	77	err = getsockopt(sock_fd, SOL_IP, IP_TOS, &buf, &optlen);
	78	if (!ASSERT_OK(err, "getsockopt"))
	79	goto detach_parent_2;
	80
	81	if (all_preorder)
	82	ASSERT_TRUE(result[0] == 3 && result[1] == 4 && result[2] == 1 && result[3] == 2,
	83	"parent and child");
	84	else
	85	ASSERT_TRUE(result[0] == 4 && result[1] == 2 && result[2] == 1 && result[3] == 3,
	86	"parent and child");
	87
	88	detach_parent_2:
	89	ASSERT_OK(bpf_prog_detach2(prog_p2_fd, cg_parent, prog_p2_atype),
	90	"bpf_prog_detach2-parent_2");
	91	detach_parent:
	92	ASSERT_OK(bpf_prog_detach2(prog_p_fd, cg_parent, prog_p_atype),
	93	"bpf_prog_detach2-parent");
	94	detach_child_2:
	95	ASSERT_OK(bpf_prog_detach2(prog_c2_fd, cg_child, prog_c2_atype),
	96	"bpf_prog_detach2-child_2");
	97	detach_child:
	98	ASSERT_OK(bpf_prog_detach2(prog_c_fd, cg_child, prog_c_atype),
	99	"bpf_prog_detach2-child");
	100	close_skel:
	101	cgroup_preorder__destroy(skel);
	102	return err;
	103	}
	104
	105	void test_cgroup_preorder(void)
	106	{
	107	int cg_parent = -1, cg_child = -1, sock_fd = -1;
	108
	109	cg_parent = test__join_cgroup("/parent");
	110	if (!ASSERT_GE(cg_parent, 0, "join_cgroup /parent"))
	111	goto out;
	112
	113	cg_child = test__join_cgroup("/parent/child");
	114	if (!ASSERT_GE(cg_child, 0, "join_cgroup /parent/child"))
	115	goto out;
	116
	117	sock_fd = socket(AF_INET, SOCK_STREAM, 0);
	118	if (!ASSERT_GE(sock_fd, 0, "socket"))
	119	goto out;
	120
	121	ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, false), "getsockopt_test_1");
	122	ASSERT_OK(run_getsockopt_test(cg_parent, cg_child, sock_fd, true), "getsockopt_test_2");
	123
	124	out:
	125	close(sock_fd);
	126	close(cg_child);
	127	close(cg_parent);
	128	}

tools/testing/selftests/bpf/prog_tests/cgroup_v1v2.c+9−4

 static int run_test(int cgroup_fd, int server_fd, bool classid)
 {
 	struct connect4_dropper *skel;
-	int fd, err = 0;
+	int fd, err = 0, port;
 	skel = connect4_dropper__open_and_load();
 	if (!ASSERT_OK_PTR(skel, "skel_open"))
 		return -1;
+	port = get_socket_local_port(server_fd);
+	if (!ASSERT_GE(port, 0, "get_socket_local_port"))
+		return -1;
+	skel->bss->port = ntohs(port);
 	skel->links.connect_v4_dropper =
 		bpf_program__attach_cgroup(skel->progs.connect_v4_dropper,
 					   cgroup_fd);
 {
 	struct network_helper_opts opts = {};
 	int server_fd, client_fd, cgroup_fd;
-	static const int port = 60120;
 	/* Step 1: Check base connectivity works without any BPF. */
-	server_fd = start_server(AF_INET, SOCK_STREAM, NULL, port, 0);
+	server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
 	if (!ASSERT_GE(server_fd, 0, "server_fd"))
 		return;
 	client_fd = connect_to_fd_opts(server_fd, &opts);
 	cgroup_fd = test__join_cgroup("/connect_dropper");
 	if (!ASSERT_GE(cgroup_fd, 0, "cgroup_fd"))
 		return;
-	server_fd = start_server(AF_INET, SOCK_STREAM, NULL, port, 0);
+	server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
 	if (!ASSERT_GE(server_fd, 0, "server_fd")) {
 		close(cgroup_fd);
 		return;

tools/testing/selftests/bpf/prog_tests/changes_pkt_data.c+0−107

@@ -1,107 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	#include "bpf/libbpf.h"
3	#include "changes_pkt_data_freplace.skel.h"
4	#include "changes_pkt_data.skel.h"
5	#include <test_progs.h>
6
7	static void print_verifier_log(const char *log)
8	{
9	if (env.verbosity >= VERBOSE_VERY)
10	fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log);
11	}
12
13	static void test_aux(const char *main_prog_name,
14	const char *to_be_replaced,
15	const char *replacement,
16	bool expect_load)
17	{
18	struct changes_pkt_data_freplace *freplace = NULL;
19	struct bpf_program *freplace_prog = NULL;
20	struct bpf_program *main_prog = NULL;
21	LIBBPF_OPTS(bpf_object_open_opts, opts);
22	struct changes_pkt_data *main = NULL;
23	char log[16*1024];
24	int err;
25
26	opts.kernel_log_buf = log;
27	opts.kernel_log_size = sizeof(log);
28	if (env.verbosity >= VERBOSE_SUPER)
29	opts.kernel_log_level = 1 \| 2 \| 4;
30	main = changes_pkt_data__open_opts(&opts);
31	if (!ASSERT_OK_PTR(main, "changes_pkt_data__open"))
32	goto out;
33	main_prog = bpf_object__find_program_by_name(main->obj, main_prog_name);
34	if (!ASSERT_OK_PTR(main_prog, "main_prog"))
35	goto out;
36	bpf_program__set_autoload(main_prog, true);
37	err = changes_pkt_data__load(main);
38	print_verifier_log(log);
39	if (!ASSERT_OK(err, "changes_pkt_data__load"))
40	goto out;
41	freplace = changes_pkt_data_freplace__open_opts(&opts);
42	if (!ASSERT_OK_PTR(freplace, "changes_pkt_data_freplace__open"))
43	goto out;
44	freplace_prog = bpf_object__find_program_by_name(freplace->obj, replacement);
45	if (!ASSERT_OK_PTR(freplace_prog, "freplace_prog"))
46	goto out;
47	bpf_program__set_autoload(freplace_prog, true);
48	bpf_program__set_autoattach(freplace_prog, true);
49	bpf_program__set_attach_target(freplace_prog,
50	bpf_program__fd(main_prog),
51	to_be_replaced);
52	err = changes_pkt_data_freplace__load(freplace);
53	print_verifier_log(log);
54	if (expect_load) {
55	ASSERT_OK(err, "changes_pkt_data_freplace__load");
56	} else {
57	ASSERT_ERR(err, "changes_pkt_data_freplace__load");
58	ASSERT_HAS_SUBSTR(log, "Extension program changes packet data", "error log");
59	}
60
61	out:
62	changes_pkt_data_freplace__destroy(freplace);
63	changes_pkt_data__destroy(main);
64	}
65
66	/* There are two global subprograms in both changes_pkt_data.skel.h:
67	* - one changes packet data;
68	* - another does not.
69	* It is ok to freplace subprograms that change packet data with those
70	* that either do or do not. It is only ok to freplace subprograms
71	* that do not change packet data with those that do not as well.
72	* The below tests check outcomes for each combination of such freplace.
73	* Also test a case when main subprogram itself is replaced and is a single
74	* subprogram in a program.
75	*/
76	void test_changes_pkt_data_freplace(void)
77	{
78	struct {
79	const char *main;
80	const char *to_be_replaced;
81	bool changes;
82	} mains[] = {
83	{ "main_with_subprogs", "changes_pkt_data", true },
84	{ "main_with_subprogs", "does_not_change_pkt_data", false },
85	{ "main_changes", "main_changes", true },
86	{ "main_does_not_change", "main_does_not_change", false },
87	};
88	struct {
89	const char *func;
90	bool changes;
91	} replacements[] = {
92	{ "changes_pkt_data", true },
93	{ "does_not_change_pkt_data", false }
94	};
95	char buf[64];
96
97	for (int i = 0; i < ARRAY_SIZE(mains); ++i) {
98	for (int j = 0; j < ARRAY_SIZE(replacements); ++j) {
99	snprintf(buf, sizeof(buf), "%s_with_%s",
100	mains[i].to_be_replaced, replacements[j].func);
101	if (!test__start_subtest(buf))
102	continue;
103	test_aux(mains[i].main, mains[i].to_be_replaced, replacements[j].func,
104	mains[i].changes \|\| !replacements[j].changes);
105	}
106	}
107	}

tools/testing/selftests/bpf/prog_tests/compute_live_registers.c+9−0

@@ -0,0 +1,9 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include "compute_live_registers.skel.h"
	4	#include "test_progs.h"
	5
	6	void test_compute_live_registers(void)
	7	{
	8	RUN_TESTS(compute_live_registers);
	9	}

tools/testing/selftests/bpf/prog_tests/core_reloc.c+4−2

 #define NESTING_ERR_CASE(name) {					\
 	NESTING_CASE_COMMON(name),					\
 	.fails = true,							\
-	.run_btfgen_fails = true,							\
+	.run_btfgen_fails = true,					\
 }
 #define ARRAYS_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) {	\
-	.a = { [2] = 1 },						\
+	.a = { [2] = 1, [3] = 11 },					\
 	.b = { [1] = { [2] = { [3] = 2 } } },				\
 	.c = { [1] = { .c =  3 } },					\
 	.d = { [0] = { [0] = { .d = 4 } } },				\
 	.input_len = sizeof(struct core_reloc_##name),			\
 	.output = STRUCT_TO_CHAR_PTR(core_reloc_arrays_output) {	\
 		.a2   = 1,						\
+		.a3   = 12,						\
 		.b123 = 2,						\
 		.c1c  = 3,						\
 		.d00d = 4,						\
 	ARRAYS_ERR_CASE(arrays___err_non_array),
 	ARRAYS_ERR_CASE(arrays___err_wrong_val_type),
 	ARRAYS_ERR_CASE(arrays___err_bad_zero_sz_arr),
+	ARRAYS_ERR_CASE(arrays___err_bad_signed_arr_elem_sz),
 	/* enum/ptr/int handling scenarios */
 	PRIMITIVES_CASE(primitives),

tools/testing/selftests/bpf/prog_tests/cpumask.c+4−1

 	"test_global_mask_nested_deep_rcu",
 	"test_global_mask_nested_deep_array_rcu",
 	"test_cpumask_weight",
+	"test_refcount_null_tracking",
+	"test_populate_reject_small_mask",
+	"test_populate_reject_unaligned",
+	"test_populate",
 };
 static void verify_success(const char *prog_name)
 		verify_success(cpumask_success_testcases[i]);
 	}
-	RUN_TESTS(cpumask_success);
 	RUN_TESTS(cpumask_failure);
 }

tools/testing/selftests/bpf/prog_tests/dynptr.c+21−0

@@ -10,6 +10,7 @@ enum test_setup_type { · 9 unmodified lines
10	SETUP_SYSCALL_SLEEP,	10	SETUP_SYSCALL_SLEEP,
11	SETUP_SKB_PROG,	11	SETUP_SKB_PROG,
12	SETUP_SKB_PROG_TP,	12	SETUP_SKB_PROG_TP,
		13	SETUP_XDP_PROG,
13	};	14	};
14		15
15	static struct {	16	static struct {
@@ -18,6 +19,8 @@ static struct { · 2 unmodified lines
18	} success_tests[] = {	19	} success_tests[] = {
19	{"test_read_write", SETUP_SYSCALL_SLEEP},	20	{"test_read_write", SETUP_SYSCALL_SLEEP},
20	{"test_dynptr_data", SETUP_SYSCALL_SLEEP},	21	{"test_dynptr_data", SETUP_SYSCALL_SLEEP},
		22	{"test_dynptr_copy", SETUP_SYSCALL_SLEEP},
		23	{"test_dynptr_copy_xdp", SETUP_XDP_PROG},
21	{"test_ringbuf", SETUP_SYSCALL_SLEEP},	24	{"test_ringbuf", SETUP_SYSCALL_SLEEP},
22	{"test_skb_readonly", SETUP_SKB_PROG},	25	{"test_skb_readonly", SETUP_SKB_PROG},
23	{"test_dynptr_skb_data", SETUP_SKB_PROG},	26	{"test_dynptr_skb_data", SETUP_SKB_PROG},
@@ -120,6 +123,24 @@ static void verify_success(const char *prog_name, enum test_setup_type setup_typ · 96 unmodified lines
120		123
121	break;	124	break;
122	}	125	}
		126	case SETUP_XDP_PROG:
		127	{
		128	char data[5000];
		129	int err, prog_fd;
		130	LIBBPF_OPTS(bpf_test_run_opts, opts,
		131	.data_in = &data,
		132	.data_size_in = sizeof(data),
		133	.repeat = 1,
		134	);
		135
		136	prog_fd = bpf_program__fd(prog);
		137	err = bpf_prog_test_run_opts(prog_fd, &opts);
		138
		139	if (!ASSERT_OK(err, "test_run"))
		140	goto cleanup;
		141
		142	break;
		143	}
123	}	144	}
124		145
125	ASSERT_EQ(skel->bss->err, 0, "err");	146	ASSERT_EQ(skel->bss->err, 0, "err");

tools/testing/selftests/bpf/prog_tests/fd_array.c+2−2

@@ -83,8 +83,8 @@ static inline int bpf_prog_get_map_ids(int prog_fd, __u32 nr_map_ids, __u32 ma · 82 unmodified lines
83	int err;	83	int err;
84		84
85	memset(&info, 0, len);	85	memset(&info, 0, len);
86	info.nr_map_ids = *nr_map_ids,	86	info.nr_map_ids = *nr_map_ids;
87	info.map_ids = ptr_to_u64(map_ids),	87	info.map_ids = ptr_to_u64(map_ids);
88		88
89	err = bpf_prog_get_info_by_fd(prog_fd, &info, &len);	89	err = bpf_prog_get_info_by_fd(prog_fd, &info, &len);
90	if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd"))	90	if (!ASSERT_OK(err, "bpf_prog_get_info_by_fd"))

tools/testing/selftests/bpf/prog_tests/fexit_noreturns.c+9−0

@@ -0,0 +1,9 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <test_progs.h>
	4	#include "fexit_noreturns.skel.h"
	5
	6	void test_fexit_noreturns(void)
	7	{
	8	RUN_TESTS(fexit_noreturns);
	9	}

tools/testing/selftests/bpf/prog_tests/fs_kfuncs.c+152−10

@@ -8,11 +8,12 @@ · 7 unmodified lines
8	#include <unistd.h>	8	#include <unistd.h>
9	#include <test_progs.h>	9	#include <test_progs.h>
10	#include "test_get_xattr.skel.h"	10	#include "test_get_xattr.skel.h"
		11	#include "test_set_remove_xattr.skel.h"
11	#include "test_fsverity.skel.h"	12	#include "test_fsverity.skel.h"
12		13
13	static const char testfile[] = "/tmp/test_progs_fs_kfuncs";	14	static const char testfile[] = "/tmp/test_progs_fs_kfuncs";
14		15
15	static void test_xattr(void)	16	static void test_get_xattr(const char name, const char value, bool allow_access)
16	{	17	{
17	struct test_get_xattr *skel = NULL;	18	struct test_get_xattr *skel = NULL;
18	int fd = -1, err;	19	int fd = -1, err;
@@ -25,7 +26,7 @@ static void test_xattr(void) · 6 unmodified lines
25	close(fd);	26	close(fd);
26	fd = -1;	27	fd = -1;
27		28
28	err = setxattr(testfile, "user.kfuncs", "hello", sizeof("hello"), 0);	29	err = setxattr(testfile, name, value, strlen(value) + 1, 0);
29	if (err && errno == EOPNOTSUPP) {	30	if (err && errno == EOPNOTSUPP) {
30	printf("%s:SKIP:local fs doesn't support xattr (%d)\n"	31	printf("%s:SKIP:local fs doesn't support xattr (%d)\n"
31	"To run this test, make sure /tmp filesystem supports xattr.\n",	32	"To run this test, make sure /tmp filesystem supports xattr.\n",
@@ -48,16 +49,23 @@ static void test_xattr(void) · 16 unmodified lines
48	goto out;	49	goto out;
49		50
50	fd = open(testfile, O_RDONLY, 0644);	51	fd = open(testfile, O_RDONLY, 0644);
		52
51	if (!ASSERT_GE(fd, 0, "open_file"))	53	if (!ASSERT_GE(fd, 0, "open_file"))
52	goto out;	54	goto out;
53		55
54	ASSERT_EQ(skel->bss->found_xattr_from_file, 1, "found_xattr_from_file");
55
56	/* Trigger security_inode_getxattr */	56	/* Trigger security_inode_getxattr */
57	err = getxattr(testfile, "user.kfuncs", v, sizeof(v));	57	err = getxattr(testfile, name, v, sizeof(v));
58	ASSERT_EQ(err, -1, "getxattr_return");	58
59	ASSERT_EQ(errno, EINVAL, "getxattr_errno");	59	if (allow_access) {
60	ASSERT_EQ(skel->bss->found_xattr_from_dentry, 1, "found_xattr_from_dentry");	60	ASSERT_EQ(err, -1, "getxattr_return");
		61	ASSERT_EQ(errno, EINVAL, "getxattr_errno");
		62	ASSERT_EQ(skel->bss->found_xattr_from_file, 1, "found_xattr_from_file");
		63	ASSERT_EQ(skel->bss->found_xattr_from_dentry, 1, "found_xattr_from_dentry");
		64	} else {
		65	ASSERT_EQ(err, strlen(value) + 1, "getxattr_return");
		66	ASSERT_EQ(skel->bss->found_xattr_from_file, 0, "found_xattr_from_file");
		67	ASSERT_EQ(skel->bss->found_xattr_from_dentry, 0, "found_xattr_from_dentry");
		68	}
61		69
62	out:	70	out:
63	close(fd);	71	close(fd);
@@ -65,6 +73,127 @@ out: · 1 unmodified line
65	remove(testfile);	73	remove(testfile);
66	}	74	}
67		75
		76	/* xattr value we will set to security.bpf.foo */
		77	static const char value_foo[] = "hello";
		78
		79	static void read_and_validate_foo(struct test_set_remove_xattr *skel)
		80	{
		81	char value_out[32];
		82	int err;
		83
		84	err = getxattr(testfile, skel->rodata->xattr_foo, value_out, sizeof(value_out));
		85	ASSERT_EQ(err, sizeof(value_foo), "getxattr size foo");
		86	ASSERT_EQ(strncmp(value_out, value_foo, sizeof(value_foo)), 0, "strncmp value_foo");
		87	}
		88
		89	static void set_foo(struct test_set_remove_xattr *skel)
		90	{
		91	ASSERT_OK(setxattr(testfile, skel->rodata->xattr_foo, value_foo, strlen(value_foo) + 1, 0),
		92	"setxattr foo");
		93	}
		94
		95	static void validate_bar_match(struct test_set_remove_xattr *skel)
		96	{
		97	char value_out[32];
		98	int err;
		99
		100	err = getxattr(testfile, skel->rodata->xattr_bar, value_out, sizeof(value_out));
		101	ASSERT_EQ(err, sizeof(skel->data->value_bar), "getxattr size bar");
		102	ASSERT_EQ(strncmp(value_out, skel->data->value_bar, sizeof(skel->data->value_bar)), 0,
		103	"strncmp value_bar");
		104	}
		105
		106	static void validate_bar_removed(struct test_set_remove_xattr *skel)
		107	{
		108	char value_out[32];
		109	int err;
		110
		111	err = getxattr(testfile, skel->rodata->xattr_bar, value_out, sizeof(value_out));
		112	ASSERT_LT(err, 0, "getxattr size bar should fail");
		113	}
		114
		115	static void test_set_remove_xattr(void)
		116	{
		117	struct test_set_remove_xattr *skel = NULL;
		118	int fd = -1, err;
		119
		120	fd = open(testfile, O_CREAT \| O_RDONLY, 0644);
		121	if (!ASSERT_GE(fd, 0, "create_file"))
		122	return;
		123
		124	close(fd);
		125	fd = -1;
		126
		127	skel = test_set_remove_xattr__open_and_load();
		128	if (!ASSERT_OK_PTR(skel, "test_set_remove_xattr__open_and_load"))
		129	return;
		130
		131	/* Set security.bpf.foo to "hello" */
		132	err = setxattr(testfile, skel->rodata->xattr_foo, value_foo, strlen(value_foo) + 1, 0);
		133	if (err && errno == EOPNOTSUPP) {
		134	printf("%s:SKIP:local fs doesn't support xattr (%d)\n"
		135	"To run this test, make sure /tmp filesystem supports xattr.\n",
		136	__func__, errno);
		137	test__skip();
		138	goto out;
		139	}
		140
		141	if (!ASSERT_OK(err, "setxattr"))
		142	goto out;
		143
		144	skel->bss->monitored_pid = getpid();
		145	err = test_set_remove_xattr__attach(skel);
		146	if (!ASSERT_OK(err, "test_set_remove_xattr__attach"))
		147	goto out;
		148
		149	/* First, test not _locked version of the kfuncs with getxattr. */
		150
		151	/* Read security.bpf.foo and trigger test_inode_getxattr. This
		152	* bpf program will set security.bpf.bar to "world".
		153	*/
		154	read_and_validate_foo(skel);
		155	validate_bar_match(skel);
		156
		157	/* Read security.bpf.foo and trigger test_inode_getxattr again.
		158	* This will remove xattr security.bpf.bar.
		159	*/
		160	read_and_validate_foo(skel);
		161	validate_bar_removed(skel);
		162
		163	ASSERT_TRUE(skel->bss->set_security_bpf_bar_success, "set_security_bpf_bar_success");
		164	ASSERT_TRUE(skel->bss->remove_security_bpf_bar_success, "remove_security_bpf_bar_success");
		165	ASSERT_TRUE(skel->bss->set_security_selinux_fail, "set_security_selinux_fail");
		166	ASSERT_TRUE(skel->bss->remove_security_selinux_fail, "remove_security_selinux_fail");
		167
		168	/* Second, test _locked version of the kfuncs, with setxattr */
		169
		170	/* Set security.bpf.foo and trigger test_inode_setxattr. This
		171	* bpf program will set security.bpf.bar to "world".
		172	*/
		173	set_foo(skel);
		174	validate_bar_match(skel);
		175
		176	/* Set security.bpf.foo and trigger test_inode_setxattr again.
		177	* This will remove xattr security.bpf.bar.
		178	*/
		179	set_foo(skel);
		180	validate_bar_removed(skel);
		181
		182	ASSERT_TRUE(skel->bss->locked_set_security_bpf_bar_success,
		183	"locked_set_security_bpf_bar_success");
		184	ASSERT_TRUE(skel->bss->locked_remove_security_bpf_bar_success,
		185	"locked_remove_security_bpf_bar_success");
		186	ASSERT_TRUE(skel->bss->locked_set_security_selinux_fail,
		187	"locked_set_security_selinux_fail");
		188	ASSERT_TRUE(skel->bss->locked_remove_security_selinux_fail,
		189	"locked_remove_security_selinux_fail");
		190
		191	out:
		192	close(fd);
		193	test_set_remove_xattr__destroy(skel);
		194	remove(testfile);
		195	}
		196
68	#ifndef SHA256_DIGEST_SIZE	197	#ifndef SHA256_DIGEST_SIZE
69	#define SHA256_DIGEST_SIZE 32	198	#define SHA256_DIGEST_SIZE 32
70	#endif	199	#endif
@@ -141,8 +270,21 @@ out: · 70 unmodified lines
141		270
142	void test_fs_kfuncs(void)	271	void test_fs_kfuncs(void)
143	{	272	{
144	if (test__start_subtest("xattr"))	273	/* Matches xattr_names in progs/test_get_xattr.c */
145	test_xattr();	274	if (test__start_subtest("user_xattr"))
		275	test_get_xattr("user.kfuncs", "hello", true);
		276
		277	if (test__start_subtest("security_bpf_xattr"))
		278	test_get_xattr("security.bpf.xxx", "hello", true);
		279
		280	if (test__start_subtest("security_bpf_xattr_error"))
		281	test_get_xattr("security.bpf", "hello", false);
		282
		283	if (test__start_subtest("security_selinux_xattr_error"))
		284	test_get_xattr("security.selinux", "hello", false);
		285
		286	if (test__start_subtest("set_remove_xattr"))
		287	test_set_remove_xattr();
146		288
147	if (test__start_subtest("fsverity"))	289	if (test__start_subtest("fsverity"))
148	test_fsverity();	290	test_fsverity();

tools/testing/selftests/bpf/prog_tests/kernel_flag.c+43−0

@@ -0,0 +1,43 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Microsoft */
	3	#include <test_progs.h>
	4	#include "kfunc_call_test.skel.h"
	5	#include "kfunc_call_test.lskel.h"
	6	#include "test_kernel_flag.skel.h"
	7
	8	void test_kernel_flag(void)
	9	{
	10	struct test_kernel_flag *lsm_skel;
	11	struct kfunc_call_test *skel = NULL;
	12	struct kfunc_call_test_lskel *lskel = NULL;
	13	int ret;
	14
	15	lsm_skel = test_kernel_flag__open_and_load();
	16	if (!ASSERT_OK_PTR(lsm_skel, "lsm_skel"))
	17	return;
	18
	19	lsm_skel->bss->monitored_tid = gettid();
	20
	21	ret = test_kernel_flag__attach(lsm_skel);
	22	if (!ASSERT_OK(ret, "test_kernel_flag__attach"))
	23	goto close_prog;
	24
	25	/* Test with skel. This should pass the gatekeeper */
	26	skel = kfunc_call_test__open_and_load();
	27	if (!ASSERT_OK_PTR(skel, "skel"))
	28	goto close_prog;
	29
	30	/* Test with lskel. This should fail due to blocking kernel-based bpf() invocations */
	31	lskel = kfunc_call_test_lskel__open_and_load();
	32	if (!ASSERT_ERR_PTR(lskel, "lskel"))
	33	goto close_prog;
	34
	35	close_prog:
	36	if (skel)
	37	kfunc_call_test__destroy(skel);
	38	if (lskel)
	39	kfunc_call_test_lskel__destroy(lskel);
	40
	41	lsm_skel->bss->monitored_tid = 0;
	42	test_kernel_flag__destroy(lsm_skel);
	43	}

tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c+540−0

@@ -0,0 +1,540 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	#include <netinet/in.h>
	3
	4	#include "network_helpers.h"
	5	#include "test_progs.h"
	6
	7	#define BPF_FILE "test_lwt_ip_encap.bpf.o"
	8
	9	#define NETNS_NAME_SIZE 32
	10	#define NETNS_BASE "ns-lwt-ip-encap"
	11
	12	#define IP4_ADDR_1 "172.16.1.100"
	13	#define IP4_ADDR_2 "172.16.2.100"
	14	#define IP4_ADDR_3 "172.16.3.100"
	15	#define IP4_ADDR_4 "172.16.4.100"
	16	#define IP4_ADDR_5 "172.16.5.100"
	17	#define IP4_ADDR_6 "172.16.6.100"
	18	#define IP4_ADDR_7 "172.16.7.100"
	19	#define IP4_ADDR_8 "172.16.8.100"
	20	#define IP4_ADDR_GRE "172.16.16.100"
	21
	22	#define IP4_ADDR_SRC IP4_ADDR_1
	23	#define IP4_ADDR_DST IP4_ADDR_4
	24
	25	#define IP6_ADDR_1 "fb01::1"
	26	#define IP6_ADDR_2 "fb02::1"
	27	#define IP6_ADDR_3 "fb03::1"
	28	#define IP6_ADDR_4 "fb04::1"
	29	#define IP6_ADDR_5 "fb05::1"
	30	#define IP6_ADDR_6 "fb06::1"
	31	#define IP6_ADDR_7 "fb07::1"
	32	#define IP6_ADDR_8 "fb08::1"
	33	#define IP6_ADDR_GRE "fb10::1"
	34
	35	#define IP6_ADDR_SRC IP6_ADDR_1
	36	#define IP6_ADDR_DST IP6_ADDR_4
	37
	38	/* Setup/topology:
	39	*
	40	* NS1 NS2 NS3
	41	* veth1 <---> veth2 veth3 <---> veth4 (the top route)
	42	* veth5 <---> veth6 veth7 <---> veth8 (the bottom route)
	43	*
	44	* Each vethN gets IP[4\|6]_ADDR_N address.
	45	*
	46	* IP_ADDR_SRC = IP_ADDR_1
	47	* IP_ADDR_DST = IP_ADDR_4
	48	*
	49	* All tests test pings from IP_ADDR__SRC to IP_ADDR_DST.
	50	*
	51	* By default, routes are configured to allow packets to go
	52	* IP_ADDR_1 <=> IP_ADDR_2 <=> IP_ADDR_3 <=> IP_ADDR_4 (the top route).
	53	*
	54	* A GRE device is installed in NS3 with IP*_ADDR_GRE, and
	55	* NS1/NS2 are configured to route packets to IP_ADDR_GRE via IP_ADDR_8
	56	* (the bottom route).
	57	*
	58	* Tests:
	59	*
	60	* 1. Routes NS2->IP*_ADDR_DST are brought down, so the only way a ping
	61	* from IP_ADDR_SRC to IP_ADDR_DST can work is via IP*_ADDR_GRE.
	62	*
	63	* 2a. In an egress test, a bpf LWT_XMIT program is installed on veth1
	64	* that encaps the packets with an IP/GRE header to route to IP*_ADDR_GRE.
	65	*
	66	* ping: SRC->[encap at veth1:egress]->GRE:decap->DST
	67	* ping replies go DST->SRC directly
	68	*
	69	* 2b. In an ingress test, a bpf LWT_IN program is installed on veth2
	70	* that encaps the packets with an IP/GRE header to route to IP*_ADDR_GRE.
	71	*
	72	* ping: SRC->[encap at veth2:ingress]->GRE:decap->DST
	73	* ping replies go DST->SRC directly
	74	*/
	75
	76	static int create_ns(char *name, size_t name_sz)
	77	{
	78	if (!name)
	79	goto fail;
	80
	81	if (!ASSERT_OK(append_tid(name, name_sz), "append TID"))
	82	goto fail;
	83
	84	SYS(fail, "ip netns add %s", name);
	85
	86	/* rp_filter gets confused by what these tests are doing, so disable it */
	87	SYS(fail, "ip netns exec %s sysctl -wq net.ipv4.conf.all.rp_filter=0", name);
	88	SYS(fail, "ip netns exec %s sysctl -wq net.ipv4.conf.default.rp_filter=0", name);
	89	/* Disable IPv6 DAD because it sometimes takes too long and fails tests */
	90	SYS(fail, "ip netns exec %s sysctl -wq net.ipv6.conf.all.accept_dad=0", name);
	91	SYS(fail, "ip netns exec %s sysctl -wq net.ipv6.conf.default.accept_dad=0", name);
	92
	93	return 0;
	94	fail:
	95	return -1;
	96	}
	97
	98	static int set_top_addr(const char ns1, const char ns2, const char *ns3)
	99	{
	100	SYS(fail, "ip -n %s a add %s/24 dev veth1", ns1, IP4_ADDR_1);
	101	SYS(fail, "ip -n %s a add %s/24 dev veth2", ns2, IP4_ADDR_2);
	102	SYS(fail, "ip -n %s a add %s/24 dev veth3", ns2, IP4_ADDR_3);
	103	SYS(fail, "ip -n %s a add %s/24 dev veth4", ns3, IP4_ADDR_4);
	104	SYS(fail, "ip -n %s -6 a add %s/128 dev veth1", ns1, IP6_ADDR_1);
	105	SYS(fail, "ip -n %s -6 a add %s/128 dev veth2", ns2, IP6_ADDR_2);
	106	SYS(fail, "ip -n %s -6 a add %s/128 dev veth3", ns2, IP6_ADDR_3);
	107	SYS(fail, "ip -n %s -6 a add %s/128 dev veth4", ns3, IP6_ADDR_4);
	108
	109	SYS(fail, "ip -n %s link set dev veth1 up", ns1);
	110	SYS(fail, "ip -n %s link set dev veth2 up", ns2);
	111	SYS(fail, "ip -n %s link set dev veth3 up", ns2);
	112	SYS(fail, "ip -n %s link set dev veth4 up", ns3);
	113
	114	return 0;
	115	fail:
	116	return 1;
	117	}
	118
	119	static int set_bottom_addr(const char ns1, const char ns2, const char *ns3)
	120	{
	121	SYS(fail, "ip -n %s a add %s/24 dev veth5", ns1, IP4_ADDR_5);
	122	SYS(fail, "ip -n %s a add %s/24 dev veth6", ns2, IP4_ADDR_6);
	123	SYS(fail, "ip -n %s a add %s/24 dev veth7", ns2, IP4_ADDR_7);
	124	SYS(fail, "ip -n %s a add %s/24 dev veth8", ns3, IP4_ADDR_8);
	125	SYS(fail, "ip -n %s -6 a add %s/128 dev veth5", ns1, IP6_ADDR_5);
	126	SYS(fail, "ip -n %s -6 a add %s/128 dev veth6", ns2, IP6_ADDR_6);
	127	SYS(fail, "ip -n %s -6 a add %s/128 dev veth7", ns2, IP6_ADDR_7);
	128	SYS(fail, "ip -n %s -6 a add %s/128 dev veth8", ns3, IP6_ADDR_8);
	129
	130	SYS(fail, "ip -n %s link set dev veth5 up", ns1);
	131	SYS(fail, "ip -n %s link set dev veth6 up", ns2);
	132	SYS(fail, "ip -n %s link set dev veth7 up", ns2);
	133	SYS(fail, "ip -n %s link set dev veth8 up", ns3);
	134
	135	return 0;
	136	fail:
	137	return 1;
	138	}
	139
	140	static int configure_vrf(const char ns1, const char ns2)
	141	{
	142	if (!ns1 \|\| !ns2)
	143	goto fail;
	144
	145	SYS(fail, "ip -n %s link add red type vrf table 1001", ns1);
	146	SYS(fail, "ip -n %s link set red up", ns1);
	147	SYS(fail, "ip -n %s route add table 1001 unreachable default metric 8192", ns1);
	148	SYS(fail, "ip -n %s -6 route add table 1001 unreachable default metric 8192", ns1);
	149	SYS(fail, "ip -n %s link set veth1 vrf red", ns1);
	150	SYS(fail, "ip -n %s link set veth5 vrf red", ns1);
	151
	152	SYS(fail, "ip -n %s link add red type vrf table 1001", ns2);
	153	SYS(fail, "ip -n %s link set red up", ns2);
	154	SYS(fail, "ip -n %s route add table 1001 unreachable default metric 8192", ns2);
	155	SYS(fail, "ip -n %s -6 route add table 1001 unreachable default metric 8192", ns2);
	156	SYS(fail, "ip -n %s link set veth2 vrf red", ns2);
	157	SYS(fail, "ip -n %s link set veth3 vrf red", ns2);
	158	SYS(fail, "ip -n %s link set veth6 vrf red", ns2);
	159	SYS(fail, "ip -n %s link set veth7 vrf red", ns2);
	160
	161	return 0;
	162	fail:
	163	return -1;
	164	}
	165
	166	static int configure_ns1(const char ns1, const char vrf)
	167	{
	168	struct nstoken *nstoken = NULL;
	169
	170	if (!ns1 \|\| !vrf)
	171	goto fail;
	172
	173	nstoken = open_netns(ns1);
	174	if (!ASSERT_OK_PTR(nstoken, "open ns1"))
	175	goto fail;
	176
	177	/* Top route */
	178	SYS(fail, "ip route add %s/32 dev veth1 %s", IP4_ADDR_2, vrf);
	179	SYS(fail, "ip route add default dev veth1 via %s %s", IP4_ADDR_2, vrf);
	180	SYS(fail, "ip -6 route add %s/128 dev veth1 %s", IP6_ADDR_2, vrf);
	181	SYS(fail, "ip -6 route add default dev veth1 via %s %s", IP6_ADDR_2, vrf);
	182	/* Bottom route */
	183	SYS(fail, "ip route add %s/32 dev veth5 %s", IP4_ADDR_6, vrf);
	184	SYS(fail, "ip route add %s/32 dev veth5 via %s %s", IP4_ADDR_7, IP4_ADDR_6, vrf);
	185	SYS(fail, "ip route add %s/32 dev veth5 via %s %s", IP4_ADDR_8, IP4_ADDR_6, vrf);
	186	SYS(fail, "ip -6 route add %s/128 dev veth5 %s", IP6_ADDR_6, vrf);
	187	SYS(fail, "ip -6 route add %s/128 dev veth5 via %s %s", IP6_ADDR_7, IP6_ADDR_6, vrf);
	188	SYS(fail, "ip -6 route add %s/128 dev veth5 via %s %s", IP6_ADDR_8, IP6_ADDR_6, vrf);
	189
	190	close_netns(nstoken);
	191	return 0;
	192	fail:
	193	close_netns(nstoken);
	194	return -1;
	195	}
	196
	197	static int configure_ns2(const char ns2, const char vrf)
	198	{
	199	struct nstoken *nstoken = NULL;
	200
	201	if (!ns2 \|\| !vrf)
	202	goto fail;
	203
	204	nstoken = open_netns(ns2);
	205	if (!ASSERT_OK_PTR(nstoken, "open ns2"))
	206	goto fail;
	207
	208	SYS(fail, "ip netns exec %s sysctl -wq net.ipv4.ip_forward=1", ns2);
	209	SYS(fail, "ip netns exec %s sysctl -wq net.ipv6.conf.all.forwarding=1", ns2);
	210
	211	/* Top route */
	212	SYS(fail, "ip route add %s/32 dev veth2 %s", IP4_ADDR_1, vrf);
	213	SYS(fail, "ip route add %s/32 dev veth3 %s", IP4_ADDR_4, vrf);
	214	SYS(fail, "ip -6 route add %s/128 dev veth2 %s", IP6_ADDR_1, vrf);
	215	SYS(fail, "ip -6 route add %s/128 dev veth3 %s", IP6_ADDR_4, vrf);
	216	/* Bottom route */
	217	SYS(fail, "ip route add %s/32 dev veth6 %s", IP4_ADDR_5, vrf);
	218	SYS(fail, "ip route add %s/32 dev veth7 %s", IP4_ADDR_8, vrf);
	219	SYS(fail, "ip -6 route add %s/128 dev veth6 %s", IP6_ADDR_5, vrf);
	220	SYS(fail, "ip -6 route add %s/128 dev veth7 %s", IP6_ADDR_8, vrf);
	221
	222	close_netns(nstoken);
	223	return 0;
	224	fail:
	225	close_netns(nstoken);
	226	return -1;
	227	}
	228
	229	static int configure_ns3(const char *ns3)
	230	{
	231	struct nstoken *nstoken = NULL;
	232
	233	if (!ns3)
	234	goto fail;
	235
	236	nstoken = open_netns(ns3);
	237	if (!ASSERT_OK_PTR(nstoken, "open ns3"))
	238	goto fail;
	239
	240	/* Top route */
	241	SYS(fail, "ip route add %s/32 dev veth4", IP4_ADDR_3);
	242	SYS(fail, "ip route add %s/32 dev veth4 via %s", IP4_ADDR_1, IP4_ADDR_3);
	243	SYS(fail, "ip route add %s/32 dev veth4 via %s", IP4_ADDR_2, IP4_ADDR_3);
	244	SYS(fail, "ip -6 route add %s/128 dev veth4", IP6_ADDR_3);
	245	SYS(fail, "ip -6 route add %s/128 dev veth4 via %s", IP6_ADDR_1, IP6_ADDR_3);
	246	SYS(fail, "ip -6 route add %s/128 dev veth4 via %s", IP6_ADDR_2, IP6_ADDR_3);
	247	/* Bottom route */
	248	SYS(fail, "ip route add %s/32 dev veth8", IP4_ADDR_7);
	249	SYS(fail, "ip route add %s/32 dev veth8 via %s", IP4_ADDR_5, IP4_ADDR_7);
	250	SYS(fail, "ip route add %s/32 dev veth8 via %s", IP4_ADDR_6, IP4_ADDR_7);
	251	SYS(fail, "ip -6 route add %s/128 dev veth8", IP6_ADDR_7);
	252	SYS(fail, "ip -6 route add %s/128 dev veth8 via %s", IP6_ADDR_5, IP6_ADDR_7);
	253	SYS(fail, "ip -6 route add %s/128 dev veth8 via %s", IP6_ADDR_6, IP6_ADDR_7);
	254
	255	/* Configure IPv4 GRE device */
	256	SYS(fail, "ip tunnel add gre_dev mode gre remote %s local %s ttl 255",
	257	IP4_ADDR_1, IP4_ADDR_GRE);
	258	SYS(fail, "ip link set gre_dev up");
	259	SYS(fail, "ip a add %s dev gre_dev", IP4_ADDR_GRE);
	260
	261	/* Configure IPv6 GRE device */
	262	SYS(fail, "ip tunnel add gre6_dev mode ip6gre remote %s local %s ttl 255",
	263	IP6_ADDR_1, IP6_ADDR_GRE);
	264	SYS(fail, "ip link set gre6_dev up");
	265	SYS(fail, "ip a add %s dev gre6_dev", IP6_ADDR_GRE);
	266
	267	close_netns(nstoken);
	268	return 0;
	269	fail:
	270	close_netns(nstoken);
	271	return -1;
	272	}
	273
	274	static int setup_network(char ns1, char ns2, char ns3, const char vrf)
	275	{
	276	if (!ns1 \|\| !ns2 \|\| !ns3 \|\| !vrf)
	277	goto fail;
	278
	279	SYS(fail, "ip -n %s link add veth1 type veth peer name veth2 netns %s", ns1, ns2);
	280	SYS(fail, "ip -n %s link add veth3 type veth peer name veth4 netns %s", ns2, ns3);
	281	SYS(fail, "ip -n %s link add veth5 type veth peer name veth6 netns %s", ns1, ns2);
	282	SYS(fail, "ip -n %s link add veth7 type veth peer name veth8 netns %s", ns2, ns3);
	283
	284	if (vrf[0]) {
	285	if (!ASSERT_OK(configure_vrf(ns1, ns2), "configure vrf"))
	286	goto fail;
	287	}
	288	if (!ASSERT_OK(set_top_addr(ns1, ns2, ns3), "set top addresses"))
	289	goto fail;
	290
	291	if (!ASSERT_OK(set_bottom_addr(ns1, ns2, ns3), "set bottom addresses"))
	292	goto fail;
	293
	294	if (!ASSERT_OK(configure_ns1(ns1, vrf), "configure ns1 routes"))
	295	goto fail;
	296
	297	if (!ASSERT_OK(configure_ns2(ns2, vrf), "configure ns2 routes"))
	298	goto fail;
	299
	300	if (!ASSERT_OK(configure_ns3(ns3), "configure ns3 routes"))
	301	goto fail;
	302
	303	/* Link bottom route to the GRE tunnels */
	304	SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s %s",
	305	ns1, IP4_ADDR_GRE, IP4_ADDR_6, vrf);
	306	SYS(fail, "ip -n %s route add %s/32 dev veth7 via %s %s",
	307	ns2, IP4_ADDR_GRE, IP4_ADDR_8, vrf);
	308	SYS(fail, "ip -n %s -6 route add %s/128 dev veth5 via %s %s",
	309	ns1, IP6_ADDR_GRE, IP6_ADDR_6, vrf);
	310	SYS(fail, "ip -n %s -6 route add %s/128 dev veth7 via %s %s",
	311	ns2, IP6_ADDR_GRE, IP6_ADDR_8, vrf);
	312
	313	return 0;
	314	fail:
	315	return -1;
	316	}
	317
	318	static int remove_routes_to_gredev(const char ns1, const char ns2, const char *vrf)
	319	{
	320	SYS(fail, "ip -n %s route del %s dev veth5 %s", ns1, IP4_ADDR_GRE, vrf);
	321	SYS(fail, "ip -n %s route del %s dev veth7 %s", ns2, IP4_ADDR_GRE, vrf);
	322	SYS(fail, "ip -n %s -6 route del %s/128 dev veth5 %s", ns1, IP6_ADDR_GRE, vrf);
	323	SYS(fail, "ip -n %s -6 route del %s/128 dev veth7 %s", ns2, IP6_ADDR_GRE, vrf);
	324
	325	return 0;
	326	fail:
	327	return -1;
	328	}
	329
	330	static int add_unreachable_routes_to_gredev(const char ns1, const char ns2, const char *vrf)
	331	{
	332	SYS(fail, "ip -n %s route add unreachable %s/32 %s", ns1, IP4_ADDR_GRE, vrf);
	333	SYS(fail, "ip -n %s route add unreachable %s/32 %s", ns2, IP4_ADDR_GRE, vrf);
	334	SYS(fail, "ip -n %s -6 route add unreachable %s/128 %s", ns1, IP6_ADDR_GRE, vrf);
	335	SYS(fail, "ip -n %s -6 route add unreachable %s/128 %s", ns2, IP6_ADDR_GRE, vrf);
	336
	337	return 0;
	338	fail:
	339	return -1;
	340	}
	341
	342	#define GSO_SIZE 5000
	343	#define GSO_TCP_PORT 9000
	344	/* This tests the fix from commit ea0371f78799 ("net: fix GSO in bpf_lwt_push_ip_encap") */
	345	static int test_gso_fix(const char ns1, const char ns3, int family)
	346	{
	347	const char *ip_addr = family == AF_INET ? IP4_ADDR_DST : IP6_ADDR_DST;
	348	char gso_packet[GSO_SIZE] = {};
	349	struct nstoken *nstoken = NULL;
	350	int sfd, cfd, afd;
	351	ssize_t bytes;
	352	int ret = -1;
	353
	354	if (!ns1 \|\| !ns3)
	355	return ret;
	356
	357	nstoken = open_netns(ns3);
	358	if (!ASSERT_OK_PTR(nstoken, "open ns3"))
	359	return ret;
	360
	361	sfd = start_server_str(family, SOCK_STREAM, ip_addr, GSO_TCP_PORT, NULL);
	362	if (!ASSERT_OK_FD(sfd, "start server"))
	363	goto close_netns;
	364
	365	close_netns(nstoken);
	366
	367	nstoken = open_netns(ns1);
	368	if (!ASSERT_OK_PTR(nstoken, "open ns1"))
	369	goto close_server;
	370
	371	cfd = connect_to_addr_str(family, SOCK_STREAM, ip_addr, GSO_TCP_PORT, NULL);
	372	if (!ASSERT_OK_FD(cfd, "connect to server"))
	373	goto close_server;
	374
	375	close_netns(nstoken);
	376	nstoken = NULL;
	377
	378	afd = accept(sfd, NULL, NULL);
	379	if (!ASSERT_OK_FD(afd, "accept"))
	380	goto close_client;
	381
	382	/* Send a packet larger than MTU */
	383	bytes = send(cfd, gso_packet, GSO_SIZE, 0);
	384	if (!ASSERT_EQ(bytes, GSO_SIZE, "send packet"))
	385	goto close_accept;
	386
	387	/* Verify we received all expected bytes */
	388	bytes = read(afd, gso_packet, GSO_SIZE);
	389	if (!ASSERT_EQ(bytes, GSO_SIZE, "receive packet"))
	390	goto close_accept;
	391
	392	ret = 0;
	393
	394	close_accept:
	395	close(afd);
	396	close_client:
	397	close(cfd);
	398	close_server:
	399	close(sfd);
	400	close_netns:
	401	close_netns(nstoken);
	402
	403	return ret;
	404	}
	405
	406	static int check_ping_ok(const char *ns1)
	407	{
	408	SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST);
	409	SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST);
	410	return 0;
	411	fail:
	412	return -1;
	413	}
	414
	415	static int check_ping_fails(const char *ns1)
	416	{
	417	int ret;
	418
	419	ret = SYS_NOFAIL("ip netns exec %s ping -c 1 -W1 -I veth1 %s", ns1, IP4_ADDR_DST);
	420	if (!ret)
	421	return -1;
	422
	423	ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST);
	424	if (!ret)
	425	return -1;
	426
	427	return 0;
	428	}
	429
	430	#define EGRESS true
	431	#define INGRESS false
	432	#define IPV4_ENCAP true
	433	#define IPV6_ENCAP false
	434	static void lwt_ip_encap(bool ipv4_encap, bool egress, const char *vrf)
	435	{
	436	char ns1[NETNS_NAME_SIZE] = NETNS_BASE "-1-";
	437	char ns2[NETNS_NAME_SIZE] = NETNS_BASE "-2-";
	438	char ns3[NETNS_NAME_SIZE] = NETNS_BASE "-3-";
	439	char *sec = ipv4_encap ? "encap_gre" : "encap_gre6";
	440
	441	if (!vrf)
	442	return;
	443
	444	if (!ASSERT_OK(create_ns(ns1, NETNS_NAME_SIZE), "create ns1"))
	445	goto out;
	446	if (!ASSERT_OK(create_ns(ns2, NETNS_NAME_SIZE), "create ns2"))
	447	goto out;
	448	if (!ASSERT_OK(create_ns(ns3, NETNS_NAME_SIZE), "create ns3"))
	449	goto out;
	450
	451	if (!ASSERT_OK(setup_network(ns1, ns2, ns3, vrf), "setup network"))
	452	goto out;
	453
	454	/* By default, pings work */
	455	if (!ASSERT_OK(check_ping_ok(ns1), "ping OK"))
	456	goto out;
	457
	458	/* Remove NS2->DST routes, ping fails */
	459	SYS(out, "ip -n %s route del %s/32 dev veth3 %s", ns2, IP4_ADDR_DST, vrf);
	460	SYS(out, "ip -n %s -6 route del %s/128 dev veth3 %s", ns2, IP6_ADDR_DST, vrf);
	461	if (!ASSERT_OK(check_ping_fails(ns1), "ping expected fail"))
	462	goto out;
	463
	464	/* Install replacement routes (LWT/eBPF), pings succeed */
	465	if (egress) {
	466	SYS(out, "ip -n %s route add %s encap bpf xmit obj %s sec %s dev veth1 %s",
	467	ns1, IP4_ADDR_DST, BPF_FILE, sec, vrf);
	468	SYS(out, "ip -n %s -6 route add %s encap bpf xmit obj %s sec %s dev veth1 %s",
	469	ns1, IP6_ADDR_DST, BPF_FILE, sec, vrf);
	470	} else {
	471	SYS(out, "ip -n %s route add %s encap bpf in obj %s sec %s dev veth2 %s",
	472	ns2, IP4_ADDR_DST, BPF_FILE, sec, vrf);
	473	SYS(out, "ip -n %s -6 route add %s encap bpf in obj %s sec %s dev veth2 %s",
	474	ns2, IP6_ADDR_DST, BPF_FILE, sec, vrf);
	475	}
	476
	477	if (!ASSERT_OK(check_ping_ok(ns1), "ping OK"))
	478	goto out;
	479
	480	/* Skip GSO tests with VRF: VRF routing needs properly assigned
	481	* source IP/device, which is easy to do with ping but hard with TCP.
	482	*/
	483	if (egress && !vrf[0]) {
	484	if (!ASSERT_OK(test_gso_fix(ns1, ns3, AF_INET), "test GSO"))
	485	goto out;
	486	}
	487
	488	/* Negative test: remove routes to GRE devices: ping fails */
	489	if (!ASSERT_OK(remove_routes_to_gredev(ns1, ns2, vrf), "remove routes to gredev"))
	490	goto out;
	491	if (!ASSERT_OK(check_ping_fails(ns1), "ping expected fail"))
	492	goto out;
	493
	494	/* Another negative test */
	495	if (!ASSERT_OK(add_unreachable_routes_to_gredev(ns1, ns2, vrf),
	496	"add unreachable routes"))
	497	goto out;
	498	ASSERT_OK(check_ping_fails(ns1), "ping expected fail");
	499
	500	out:
	501	SYS_NOFAIL("ip netns del %s", ns1);
	502	SYS_NOFAIL("ip netns del %s", ns2);
	503	SYS_NOFAIL("ip netns del %s", ns3);
	504	}
	505
	506	void test_lwt_ip_encap_vrf_ipv6(void)
	507	{
	508	if (test__start_subtest("egress"))
	509	lwt_ip_encap(IPV6_ENCAP, EGRESS, "vrf red");
	510
	511	if (test__start_subtest("ingress"))
	512	lwt_ip_encap(IPV6_ENCAP, INGRESS, "vrf red");
	513	}
	514
	515	void test_lwt_ip_encap_vrf_ipv4(void)
	516	{
	517	if (test__start_subtest("egress"))
	518	lwt_ip_encap(IPV4_ENCAP, EGRESS, "vrf red");
	519
	520	if (test__start_subtest("ingress"))
	521	lwt_ip_encap(IPV4_ENCAP, INGRESS, "vrf red");
	522	}
	523
	524	void test_lwt_ip_encap_ipv6(void)
	525	{
	526	if (test__start_subtest("egress"))
	527	lwt_ip_encap(IPV6_ENCAP, EGRESS, "");
	528
	529	if (test__start_subtest("ingress"))
	530	lwt_ip_encap(IPV6_ENCAP, INGRESS, "");
	531	}
	532
	533	void test_lwt_ip_encap_ipv4(void)
	534	{
	535	if (test__start_subtest("egress"))
	536	lwt_ip_encap(IPV4_ENCAP, EGRESS, "");
	537
	538	if (test__start_subtest("ingress"))
	539	lwt_ip_encap(IPV4_ENCAP, INGRESS, "");
	540	}

tools/testing/selftests/bpf/prog_tests/lwt_seg6local.c+176−0

@@ -0,0 +1,176 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2
	3	/* Connects 6 network namespaces through veths.
	4	* Each NS may have different IPv6 global scope addresses :
	5	*
	6	* NS1 NS2 NS3 NS4 NS5 NS6
	7	* lo veth1 <-> veth2 veth3 <-> veth4 veth5 <-> veth6 lo veth7 <-> veth8 veth9 <-> veth10 lo
	8	* fb00 ::1 ::12 ::21 ::34 ::43 ::56 ::65 ::78 ::87 ::910 ::109 ::6
	9	* fd00 ::4
	10	* fc42 ::1
	11	*
	12	* All IPv6 packets going to fb00::/16 through NS2 will be encapsulated in a
	13	* IPv6 header with a Segment Routing Header, with segments :
	14	* fd00::1 -> fd00::2 -> fd00::3 -> fd00::4
	15	*
	16	* 3 fd00::/16 IPv6 addresses are binded to seg6local End.BPF actions :
	17	* - fd00::1 : add a TLV, change the flags and apply a End.X action to fc42::1
	18	* - fd00::2 : remove the TLV, change the flags, add a tag
	19	* - fd00::3 : apply an End.T action to fd00::4, through routing table 117
	20	*
	21	* fd00::4 is a simple Segment Routing node decapsulating the inner IPv6 packet.
	22	* Each End.BPF action will validate the operations applied on the SRH by the
	23	* previous BPF program in the chain, otherwise the packet is dropped.
	24	*
	25	* An UDP datagram is sent from fb00::1 to fb00::6. The test succeeds if this
	26	* datagram can be read on NS6 when binding to fb00::6.
	27	*/
	28
	29	#include "network_helpers.h"
	30	#include "test_progs.h"
	31
	32	#define NETNS_BASE "lwt-seg6local-"
	33	#define BPF_FILE "test_lwt_seg6local.bpf.o"
	34
	35	static void cleanup(void)
	36	{
	37	int ns;
	38
	39	for (ns = 1; ns < 7; ns++)
	40	SYS_NOFAIL("ip netns del %s%d", NETNS_BASE, ns);
	41	}
	42
	43	static int setup(void)
	44	{
	45	int ns;
	46
	47	for (ns = 1; ns < 7; ns++)
	48	SYS(fail, "ip netns add %s%d", NETNS_BASE, ns);
	49
	50	SYS(fail, "ip -n %s6 link set dev lo up", NETNS_BASE);
	51
	52	for (ns = 1; ns < 6; ns++) {
	53	int local_id = ns * 2 - 1;
	54	int peer_id = ns * 2;
	55	int next_ns = ns + 1;
	56
	57	SYS(fail, "ip -n %s%d link add veth%d type veth peer name veth%d netns %s%d",
	58	NETNS_BASE, ns, local_id, peer_id, NETNS_BASE, next_ns);
	59
	60	SYS(fail, "ip -n %s%d link set dev veth%d up", NETNS_BASE, ns, local_id);
	61	SYS(fail, "ip -n %s%d link set dev veth%d up", NETNS_BASE, next_ns, peer_id);
	62
	63	/* All link scope addresses to veths */
	64	SYS(fail, "ip -n %s%d -6 addr add fb00::%d%d/16 dev veth%d scope link",
	65	NETNS_BASE, ns, local_id, peer_id, local_id);
	66	SYS(fail, "ip -n %s%d -6 addr add fb00::%d%d/16 dev veth%d scope link",
	67	NETNS_BASE, next_ns, peer_id, local_id, peer_id);
	68	}
	69
	70
	71	SYS(fail, "ip -n %s5 -6 route add fb00::109 table 117 dev veth9 scope link", NETNS_BASE);
	72
	73	SYS(fail, "ip -n %s1 -6 addr add fb00::1/16 dev lo", NETNS_BASE);
	74	SYS(fail, "ip -n %s1 -6 route add fb00::6 dev veth1 via fb00::21", NETNS_BASE);
	75
	76	SYS(fail, "ip -n %s2 -6 route add fb00::6 encap bpf in obj %s sec encap_srh dev veth2",
	77	NETNS_BASE, BPF_FILE);
	78	SYS(fail, "ip -n %s2 -6 route add fd00::1 dev veth3 via fb00::43 scope link", NETNS_BASE);
	79
	80	SYS(fail, "ip -n %s3 -6 route add fc42::1 dev veth5 via fb00::65", NETNS_BASE);
	81	SYS(fail,
	82	"ip -n %s3 -6 route add fd00::1 encap seg6local action End.BPF endpoint obj %s sec add_egr_x dev veth4",
	83	NETNS_BASE, BPF_FILE);
	84
	85	SYS(fail,
	86	"ip -n %s4 -6 route add fd00::2 encap seg6local action End.BPF endpoint obj %s sec pop_egr dev veth6",
	87	NETNS_BASE, BPF_FILE);
	88	SYS(fail, "ip -n %s4 -6 addr add fc42::1 dev lo", NETNS_BASE);
	89	SYS(fail, "ip -n %s4 -6 route add fd00::3 dev veth7 via fb00::87", NETNS_BASE);
	90
	91	SYS(fail, "ip -n %s5 -6 route add fd00::4 table 117 dev veth9 via fb00::109", NETNS_BASE);
	92	SYS(fail,
	93	"ip -n %s5 -6 route add fd00::3 encap seg6local action End.BPF endpoint obj %s sec inspect_t dev veth8",
	94	NETNS_BASE, BPF_FILE);
	95
	96	SYS(fail, "ip -n %s6 -6 addr add fb00::6/16 dev lo", NETNS_BASE);
	97	SYS(fail, "ip -n %s6 -6 addr add fd00::4/16 dev lo", NETNS_BASE);
	98
	99	for (ns = 1; ns < 6; ns++)
	100	SYS(fail, "ip netns exec %s%d sysctl -wq net.ipv6.conf.all.forwarding=1",
	101	NETNS_BASE, ns);
	102
	103	SYS(fail, "ip netns exec %s6 sysctl -wq net.ipv6.conf.all.seg6_enabled=1", NETNS_BASE);
	104	SYS(fail, "ip netns exec %s6 sysctl -wq net.ipv6.conf.lo.seg6_enabled=1", NETNS_BASE);
	105	SYS(fail, "ip netns exec %s6 sysctl -wq net.ipv6.conf.veth10.seg6_enabled=1", NETNS_BASE);
	106
	107	return 0;
	108	fail:
	109	return -1;
	110	}
	111
	112	#define SERVER_PORT 7330
	113	#define CLIENT_PORT 2121
	114	void test_lwt_seg6local(void)
	115	{
	116	struct sockaddr_in6 server_addr = {};
	117	const char *ns1 = NETNS_BASE "1";
	118	const char *ns6 = NETNS_BASE "6";
	119	struct nstoken *nstoken = NULL;
	120	const char *foobar = "foobar";
	121	ssize_t bytes;
	122	int sfd, cfd;
	123	char buf[7];
	124
	125	if (!ASSERT_OK(setup(), "setup"))
	126	goto out;
	127
	128	nstoken = open_netns(ns6);
	129	if (!ASSERT_OK_PTR(nstoken, "open ns6"))
	130	goto out;
	131
	132	sfd = start_server_str(AF_INET6, SOCK_DGRAM, "fb00::6", SERVER_PORT, NULL);
	133	if (!ASSERT_OK_FD(sfd, "start server"))
	134	goto close_netns;
	135
	136	close_netns(nstoken);
	137
	138	nstoken = open_netns(ns1);
	139	if (!ASSERT_OK_PTR(nstoken, "open ns1"))
	140	goto close_server;
	141
	142	cfd = start_server_str(AF_INET6, SOCK_DGRAM, "fb00::1", CLIENT_PORT, NULL);
	143	if (!ASSERT_OK_FD(cfd, "start client"))
	144	goto close_server;
	145
	146	close_netns(nstoken);
	147	nstoken = NULL;
	148
	149	/* Send a packet larger than MTU */
	150	server_addr.sin6_family = AF_INET6;
	151	server_addr.sin6_port = htons(SERVER_PORT);
	152	if (!ASSERT_EQ(inet_pton(AF_INET6, "fb00::6", &server_addr.sin6_addr), 1,
	153	"build target addr"))
	154	goto close_client;
	155
	156	bytes = sendto(cfd, foobar, sizeof(foobar), 0,
	157	(struct sockaddr *)&server_addr, sizeof(server_addr));
	158	if (!ASSERT_EQ(bytes, sizeof(foobar), "send packet"))
	159	goto close_client;
	160
	161	/* Verify we received all expected bytes */
	162	bytes = read(sfd, buf, sizeof(buf));
	163	if (!ASSERT_EQ(bytes, sizeof(buf), "receive packet"))
	164	goto close_client;
	165	ASSERT_STREQ(buf, foobar, "check udp packet");
	166
	167	close_client:
	168	close(cfd);
	169	close_server:
	170	close(sfd);
	171	close_netns:
	172	close_netns(nstoken);
	173
	174	out:
	175	cleanup();
	176	}

tools/testing/selftests/bpf/prog_tests/netns_cookie.c+14−7

 	skel->links.get_netns_cookie_sockops = bpf_program__attach_cgroup(
 		skel->progs.get_netns_cookie_sockops, cgroup_fd);
-	if (!ASSERT_OK_PTR(skel->links.get_netns_cookie_sockops, "prog_attach"))
+	if (!ASSERT_OK_PTR(skel->links.get_netns_cookie_sockops, "prog_attach_sockops"))
 		goto done;
 	verdict = bpf_program__fd(skel->progs.get_netns_cookie_sk_msg);
 	map = bpf_map__fd(skel->maps.sock_map);
 	err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
-	if (!ASSERT_OK(err, "prog_attach"))
+	if (!ASSERT_OK(err, "prog_attach_sk_msg"))
 		goto done;
 	tc_fd = bpf_program__fd(skel->progs.get_netns_cookie_tcx);
 	err = bpf_prog_attach_opts(tc_fd, loopback, BPF_TCX_INGRESS, &opta);
-	if (!ASSERT_OK(err, "prog_attach"))
+	if (!ASSERT_OK(err, "prog_attach_tcx"))
 		goto done;
+	skel->links.get_netns_cookie_cgroup_skb = bpf_program__attach_cgroup(
+		skel->progs.get_netns_cookie_cgroup_skb, cgroup_fd);
+	if (!ASSERT_OK_PTR(skel->links.get_netns_cookie_cgroup_skb, "prog_attach_cgroup_skb"))
+		goto cleanup_tc;
 	server_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
 	if (CHECK(server_fd < 0, "start_server", "errno %d\n", errno))
 		goto cleanup_tc;
 	if (!ASSERT_OK(err, "getsockopt"))
 		goto cleanup_tc;
-	ASSERT_EQ(val, cookie_expected_value, "cookie_value");
+	ASSERT_EQ(val, cookie_expected_value, "cookie_value_sockops");
 	err = bpf_map_lookup_elem(bpf_map__fd(skel->maps.sk_msg_netns_cookies),
 				  &client_fd, &val);
 	if (!ASSERT_OK(err, "map_lookup(sk_msg_netns_cookies)"))
 		goto cleanup_tc;
-	ASSERT_EQ(val, cookie_expected_value, "cookie_value");
+	ASSERT_EQ(val, cookie_expected_value, "cookie_value_sk_msg");
-	ASSERT_EQ(skel->bss->tcx_init_netns_cookie, cookie_expected_value, "cookie_value");
+	ASSERT_EQ(skel->bss->tcx_init_netns_cookie, cookie_expected_value, "cookie_value_init_tcx");
-	ASSERT_EQ(skel->bss->tcx_netns_cookie, cookie_expected_value, "cookie_value");
+	ASSERT_EQ(skel->bss->tcx_netns_cookie, cookie_expected_value, "cookie_value_tcx");
+	ASSERT_EQ(skel->bss->cgroup_skb_init_netns_cookie, cookie_expected_value, "cookie_value_init_cgroup_skb");
+	ASSERT_EQ(skel->bss->cgroup_skb_netns_cookie, cookie_expected_value, "cookie_value_cgroup_skb");
 cleanup_tc:
 	err = bpf_prog_detach_opts(tc_fd, loopback, BPF_TCX_INGRESS, &optd);

tools/testing/selftests/bpf/prog_tests/ns_current_pid_tgid.c+17−30

@@ -200,41 +200,28 @@ static void test_ns_current_pid_tgid_new_ns(int (fn)(void ), void *arg) · 199 unmodified lines
200	return;	200	return;
201	}	201	}
202		202
203	static void test_in_netns(int (fn)(void ), void *arg)
204	{
205	struct nstoken *nstoken = NULL;
206
207	SYS(cleanup, "ip netns add ns_current_pid_tgid");
208	SYS(cleanup, "ip -net ns_current_pid_tgid link set dev lo up");
209
210	nstoken = open_netns("ns_current_pid_tgid");
211	if (!ASSERT_OK_PTR(nstoken, "open_netns"))
212	goto cleanup;
213
214	test_ns_current_pid_tgid_new_ns(fn, arg);
215
216	cleanup:
217	if (nstoken)
218	close_netns(nstoken);
219	SYS_NOFAIL("ip netns del ns_current_pid_tgid");
220	}
221
222	/* TODO: use a different tracepoint */	203	/* TODO: use a different tracepoint */
223	void serial_test_ns_current_pid_tgid(void)	204	void serial_test_current_pid_tgid(void)
224	{	205	{
225	if (test__start_subtest("root_ns_tp"))	206	if (test__start_subtest("root_ns_tp"))
226	test_current_pid_tgid_tp(NULL);	207	test_current_pid_tgid_tp(NULL);
227	if (test__start_subtest("new_ns_tp"))	208	if (test__start_subtest("new_ns_tp"))
228	test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_tp, NULL);	209	test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_tp, NULL);
229	if (test__start_subtest("new_ns_cgrp")) {	210	}
230	int cgroup_fd = -1;	211
231		212	void test_ns_current_pid_tgid_cgrp(void)
232	cgroup_fd = test__join_cgroup("/sock_addr");	213	{
233	if (ASSERT_GE(cgroup_fd, 0, "join_cgroup")) {	214	int cgroup_fd = test__join_cgroup("/sock_addr");
234	test_in_netns(test_current_pid_tgid_cgrp, &cgroup_fd);	215
235	close(cgroup_fd);	216	if (ASSERT_OK_FD(cgroup_fd, "join_cgroup")) {
236	}	217	test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_cgrp, &cgroup_fd);
		218	close(cgroup_fd);
237	}	219	}
238	if (test__start_subtest("new_ns_sk_msg"))
239	test_in_netns(test_current_pid_tgid_sk_msg, NULL);
240	}	220	}
		221
		222	void test_ns_current_pid_tgid_sk_msg(void)
		223	{
		224	test_ns_current_pid_tgid_new_ns(test_current_pid_tgid_sk_msg, NULL);
		225	}
		226
		227

tools/testing/selftests/bpf/prog_tests/prepare.c+99−0

@@ -0,0 +1,99 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta */
	3
	4	#include <test_progs.h>
	5	#include <network_helpers.h>
	6	#include "prepare.skel.h"
	7
	8	static bool check_prepared(struct bpf_object *obj)
	9	{
	10	bool is_prepared = true;
	11	const struct bpf_map *map;
	12
	13	bpf_object__for_each_map(map, obj) {
	14	if (bpf_map__fd(map) < 0)
	15	is_prepared = false;
	16	}
	17
	18	return is_prepared;
	19	}
	20
	21	static void test_prepare_no_load(void)
	22	{
	23	struct prepare *skel;
	24	int err;
	25	LIBBPF_OPTS(bpf_test_run_opts, topts,
	26	.data_in = &pkt_v4,
	27	.data_size_in = sizeof(pkt_v4),
	28	);
	29
	30	skel = prepare__open();
	31	if (!ASSERT_OK_PTR(skel, "prepare__open"))
	32	return;
	33
	34	if (!ASSERT_FALSE(check_prepared(skel->obj), "not check_prepared"))
	35	goto cleanup;
	36
	37	err = bpf_object__prepare(skel->obj);
	38
	39	if (!ASSERT_TRUE(check_prepared(skel->obj), "check_prepared"))
	40	goto cleanup;
	41
	42	if (!ASSERT_OK(err, "bpf_object__prepare"))
	43	goto cleanup;
	44
	45	cleanup:
	46	prepare__destroy(skel);
	47	}
	48
	49	static void test_prepare_load(void)
	50	{
	51	struct prepare *skel;
	52	int err, prog_fd;
	53	LIBBPF_OPTS(bpf_test_run_opts, topts,
	54	.data_in = &pkt_v4,
	55	.data_size_in = sizeof(pkt_v4),
	56	);
	57
	58	skel = prepare__open();
	59	if (!ASSERT_OK_PTR(skel, "prepare__open"))
	60	return;
	61
	62	if (!ASSERT_FALSE(check_prepared(skel->obj), "not check_prepared"))
	63	goto cleanup;
	64
	65	err = bpf_object__prepare(skel->obj);
	66	if (!ASSERT_OK(err, "bpf_object__prepare"))
	67	goto cleanup;
	68
	69	err = prepare__load(skel);
	70	if (!ASSERT_OK(err, "prepare__load"))
	71	goto cleanup;
	72
	73	if (!ASSERT_TRUE(check_prepared(skel->obj), "check_prepared"))
	74	goto cleanup;
	75
	76	prog_fd = bpf_program__fd(skel->progs.program);
	77	if (!ASSERT_GE(prog_fd, 0, "prog_fd"))
	78	goto cleanup;
	79
	80	err = bpf_prog_test_run_opts(prog_fd, &topts);
	81	if (!ASSERT_OK(err, "test_run_opts err"))
	82	goto cleanup;
	83
	84	if (!ASSERT_OK(topts.retval, "test_run_opts retval"))
	85	goto cleanup;
	86
	87	ASSERT_EQ(skel->bss->err, 0, "err");
	88
	89	cleanup:
	90	prepare__destroy(skel);
	91	}
	92
	93	void test_prepare(void)
	94	{
	95	if (test__start_subtest("prepare_load"))
	96	test_prepare_load();
	97	if (test__start_subtest("prepare_no_load"))
	98	test_prepare_no_load();
	99	}

tools/testing/selftests/bpf/prog_tests/pro_epilogue.c+2−0

 #include "epilogue_tailcall.skel.h"
 #include "pro_epilogue_goto_start.skel.h"
 #include "epilogue_exit.skel.h"
+#include "pro_epilogue_with_kfunc.skel.h"
 struct st_ops_args {
 	__u64 a;
 	RUN_TESTS(pro_epilogue);
 	RUN_TESTS(pro_epilogue_goto_start);
 	RUN_TESTS(epilogue_exit);
+	RUN_TESTS(pro_epilogue_with_kfunc);
 	if (test__start_subtest("tailcall"))
 		test_tailcall();
 }

tools/testing/selftests/bpf/prog_tests/rcu_read_lock.c+3−0

 	"nested_rcu_region",
 	"rcu_read_lock_global_subprog_lock",
 	"rcu_read_lock_global_subprog_unlock",
+	"rcu_read_lock_sleepable_helper_global_subprog",
+	"rcu_read_lock_sleepable_kfunc_global_subprog",
+	"rcu_read_lock_sleepable_global_subprog_indirect",
 };
 static void test_inproper_region(void)

tools/testing/selftests/bpf/prog_tests/read_vsyscall.c+1−0

@@ -24,6 +24,7 @@ struct read_ret_desc { · 23 unmodified lines
24	{ .name = "copy_from_user", .ret = -EFAULT },	24	{ .name = "copy_from_user", .ret = -EFAULT },
25	{ .name = "copy_from_user_task", .ret = -EFAULT },	25	{ .name = "copy_from_user_task", .ret = -EFAULT },
26	{ .name = "copy_from_user_str", .ret = -EFAULT },	26	{ .name = "copy_from_user_str", .ret = -EFAULT },
		27	{ .name = "copy_from_user_task_str", .ret = -EFAULT },
27	};	28	};
28		29
29	void test_read_vsyscall(void)	30	void test_read_vsyscall(void)

tools/testing/selftests/bpf/prog_tests/setget_sockopt.c+1−1

@@ -202,7 +202,7 @@ err_out: · 201 unmodified lines
202	void test_setget_sockopt(void)	202	void test_setget_sockopt(void)
203	{	203	{
204	cg_fd = test__join_cgroup(CG_NAME);	204	cg_fd = test__join_cgroup(CG_NAME);
205	if (cg_fd < 0)	205	if (!ASSERT_OK_FD(cg_fd, "join cgroup"))
206	return;	206	return;
207		207
208	if (create_netns())	208	if (create_netns())

tools/testing/selftests/bpf/prog_tests/spin_lock.c+3−0

 	{ "lock_id_mismatch_innermapval_mapval", "bpf_spin_unlock of different lock" },
 	{ "lock_global_subprog_call1", "global function calls are not allowed while holding a lock" },
 	{ "lock_global_subprog_call2", "global function calls are not allowed while holding a lock" },
+	{ "lock_global_sleepable_helper_subprog", "global function calls are not allowed while holding a lock" },
+	{ "lock_global_sleepable_kfunc_subprog", "global function calls are not allowed while holding a lock" },
+	{ "lock_global_sleepable_subprog_indirect", "global function calls are not allowed while holding a lock" },
 };
 static int match_regex(const char *pattern, const char *string)

tools/testing/selftests/bpf/prog_tests/summarization.c+144−0

@@ -0,0 +1,144 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include "bpf/libbpf.h"
	3	#include "summarization_freplace.skel.h"
	4	#include "summarization.skel.h"
	5	#include <test_progs.h>
	6
	7	static void print_verifier_log(const char *log)
	8	{
	9	if (env.verbosity >= VERBOSE_VERY)
	10	fprintf(stdout, "VERIFIER LOG:\n=============\n%s=============\n", log);
	11	}
	12
	13	static void test_aux(const char *main_prog_name,
	14	const char *to_be_replaced,
	15	const char *replacement,
	16	bool expect_load,
	17	const char *err_msg)
	18	{
	19	struct summarization_freplace *freplace = NULL;
	20	struct bpf_program *freplace_prog = NULL;
	21	struct bpf_program *main_prog = NULL;
	22	LIBBPF_OPTS(bpf_object_open_opts, opts);
	23	struct summarization *main = NULL;
	24	char log[16*1024];
	25	int err;
	26
	27	opts.kernel_log_buf = log;
	28	opts.kernel_log_size = sizeof(log);
	29	if (env.verbosity >= VERBOSE_SUPER)
	30	opts.kernel_log_level = 1 \| 2 \| 4;
	31	main = summarization__open_opts(&opts);
	32	if (!ASSERT_OK_PTR(main, "summarization__open"))
	33	goto out;
	34	main_prog = bpf_object__find_program_by_name(main->obj, main_prog_name);
	35	if (!ASSERT_OK_PTR(main_prog, "main_prog"))
	36	goto out;
	37	bpf_program__set_autoload(main_prog, true);
	38	err = summarization__load(main);
	39	print_verifier_log(log);
	40	if (!ASSERT_OK(err, "summarization__load"))
	41	goto out;
	42	freplace = summarization_freplace__open_opts(&opts);
	43	if (!ASSERT_OK_PTR(freplace, "summarization_freplace__open"))
	44	goto out;
	45	freplace_prog = bpf_object__find_program_by_name(freplace->obj, replacement);
	46	if (!ASSERT_OK_PTR(freplace_prog, "freplace_prog"))
	47	goto out;
	48	bpf_program__set_autoload(freplace_prog, true);
	49	bpf_program__set_autoattach(freplace_prog, true);
	50	bpf_program__set_attach_target(freplace_prog,
	51	bpf_program__fd(main_prog),
	52	to_be_replaced);
	53	err = summarization_freplace__load(freplace);
	54	print_verifier_log(log);
	55
	56	/* The might_sleep extension doesn't work yet as sleepable calls are not
	57	* allowed, but preserve the check in case it's supported later and then
	58	* this particular combination can be enabled.
	59	*/
	60	if (!strcmp("might_sleep", replacement) && err) {
	61	ASSERT_HAS_SUBSTR(log, "helper call might sleep in a non-sleepable prog", "error log");
	62	ASSERT_EQ(err, -EINVAL, "err");
	63	test__skip();
	64	goto out;
	65	}
	66
	67	if (expect_load) {
	68	ASSERT_OK(err, "summarization_freplace__load");
	69	} else {
	70	ASSERT_ERR(err, "summarization_freplace__load");
	71	ASSERT_HAS_SUBSTR(log, err_msg, "error log");
	72	}
	73
	74	out:
	75	summarization_freplace__destroy(freplace);
	76	summarization__destroy(main);
	77	}
	78
	79	/* There are two global subprograms in both summarization.skel.h:
	80	* - one changes packet data;
	81	* - another does not.
	82	* It is ok to freplace subprograms that change packet data with those
	83	* that either do or do not. It is only ok to freplace subprograms
	84	* that do not change packet data with those that do not as well.
	85	* The below tests check outcomes for each combination of such freplace.
	86	* Also test a case when main subprogram itself is replaced and is a single
	87	* subprogram in a program.
	88	*
	89	* This holds for might_sleep programs. It is ok to replace might_sleep with
	90	* might_sleep and with does_not_sleep, but does_not_sleep cannot be replaced
	91	* with might_sleep.
	92	*/
	93	void test_summarization_freplace(void)
	94	{
	95	struct {
	96	const char *main;
	97	const char *to_be_replaced;
	98	bool has_side_effect;
	99	} mains[2][4] = {
	100	{
	101	{ "main_changes_with_subprogs", "changes_pkt_data", true },
	102	{ "main_changes_with_subprogs", "does_not_change_pkt_data", false },
	103	{ "main_changes", "main_changes", true },
	104	{ "main_does_not_change", "main_does_not_change", false },
	105	},
	106	{
	107	{ "main_might_sleep_with_subprogs", "might_sleep", true },
	108	{ "main_might_sleep_with_subprogs", "does_not_sleep", false },
	109	{ "main_might_sleep", "main_might_sleep", true },
	110	{ "main_does_not_sleep", "main_does_not_sleep", false },
	111	},
	112	};
	113	const char *pkt_err = "Extension program changes packet data";
	114	const char *slp_err = "Extension program may sleep";
	115	struct {
	116	const char *func;
	117	bool has_side_effect;
	118	const char *err_msg;
	119	} replacements[2][2] = {
	120	{
	121	{ "changes_pkt_data", true, pkt_err },
	122	{ "does_not_change_pkt_data", false, pkt_err },
	123	},
	124	{
	125	{ "might_sleep", true, slp_err },
	126	{ "does_not_sleep", false, slp_err },
	127	},
	128	};
	129	char buf[64];
	130
	131	for (int t = 0; t < 2; t++) {
	132	for (int i = 0; i < ARRAY_SIZE(mains); ++i) {
	133	for (int j = 0; j < ARRAY_SIZE(replacements); ++j) {
	134	snprintf(buf, sizeof(buf), "%s_with_%s",
	135	mains[t][i].to_be_replaced, replacements[t][j].func);
	136	if (!test__start_subtest(buf))
	137	continue;
	138	test_aux(mains[t][i].main, mains[t][i].to_be_replaced, replacements[t][j].func,
	139	mains[t][i].has_side_effect \|\| !replacements[t][j].has_side_effect,
	140	replacements[t][j].err_msg);
	141	}
	142	}
	143	}
	144	}

tools/testing/selftests/bpf/prog_tests/tailcalls.c+1−0

@@ -1600,6 +1600,7 @@ static void test_tailcall_bpf2bpf_freplace(void) · 1599 unmodified lines
1600	goto out;	1600	goto out;
1601		1601
1602	err = bpf_link__destroy(freplace_link);	1602	err = bpf_link__destroy(freplace_link);
		1603	freplace_link = NULL;
1603	if (!ASSERT_OK(err, "destroy link"))	1604	if (!ASSERT_OK(err, "destroy link"))
1604	goto out;	1605	goto out;
1605		1606

tools/testing/selftests/bpf/prog_tests/tc_links.c+14−14

@@ -13,7 +13,7 @@ · 12 unmodified lines
13	#include "netlink_helpers.h"	13	#include "netlink_helpers.h"
14	#include "tc_helpers.h"	14	#include "tc_helpers.h"
15		15
16	void serial_test_tc_links_basic(void)	16	void test_ns_tc_links_basic(void)
17	{	17	{
18	LIBBPF_OPTS(bpf_prog_query_opts, optq);	18	LIBBPF_OPTS(bpf_prog_query_opts, optq);
19	LIBBPF_OPTS(bpf_tcx_opts, optl);	19	LIBBPF_OPTS(bpf_tcx_opts, optl);
@@ -260,7 +260,7 @@ cleanup: · 240 unmodified lines
260	assert_mprog_count(target, 0);	260	assert_mprog_count(target, 0);
261	}	261	}
262		262
263	void serial_test_tc_links_before(void)	263	void test_ns_tc_links_before(void)
264	{	264	{
265	test_tc_links_before_target(BPF_TCX_INGRESS);	265	test_tc_links_before_target(BPF_TCX_INGRESS);
266	test_tc_links_before_target(BPF_TCX_EGRESS);	266	test_tc_links_before_target(BPF_TCX_EGRESS);
@@ -414,7 +414,7 @@ cleanup: · 147 unmodified lines
414	assert_mprog_count(target, 0);	414	assert_mprog_count(target, 0);
415	}	415	}
416		416
417	void serial_test_tc_links_after(void)	417	void test_ns_tc_links_after(void)
418	{	418	{
419	test_tc_links_after_target(BPF_TCX_INGRESS);	419	test_tc_links_after_target(BPF_TCX_INGRESS);
420	test_tc_links_after_target(BPF_TCX_EGRESS);	420	test_tc_links_after_target(BPF_TCX_EGRESS);
@@ -514,7 +514,7 @@ cleanup: · 93 unmodified lines
514	assert_mprog_count(target, 0);	514	assert_mprog_count(target, 0);
515	}	515	}
516		516
517	void serial_test_tc_links_revision(void)	517	void test_ns_tc_links_revision(void)
518	{	518	{
519	test_tc_links_revision_target(BPF_TCX_INGRESS);	519	test_tc_links_revision_target(BPF_TCX_INGRESS);
520	test_tc_links_revision_target(BPF_TCX_EGRESS);	520	test_tc_links_revision_target(BPF_TCX_EGRESS);
@@ -618,7 +618,7 @@ cleanup: · 97 unmodified lines
618	assert_mprog_count(target, 0);	618	assert_mprog_count(target, 0);
619	}	619	}
620		620
621	void serial_test_tc_links_chain_classic(void)	621	void test_ns_tc_links_chain_classic(void)
622	{	622	{
623	test_tc_chain_classic(BPF_TCX_INGRESS, false);	623	test_tc_chain_classic(BPF_TCX_INGRESS, false);
624	test_tc_chain_classic(BPF_TCX_EGRESS, false);	624	test_tc_chain_classic(BPF_TCX_EGRESS, false);
@@ -846,7 +846,7 @@ cleanup: · 221 unmodified lines
846	assert_mprog_count(target, 0);	846	assert_mprog_count(target, 0);
847	}	847	}
848		848
849	void serial_test_tc_links_replace(void)	849	void test_ns_tc_links_replace(void)
850	{	850	{
851	test_tc_links_replace_target(BPF_TCX_INGRESS);	851	test_tc_links_replace_target(BPF_TCX_INGRESS);
852	test_tc_links_replace_target(BPF_TCX_EGRESS);	852	test_tc_links_replace_target(BPF_TCX_EGRESS);
@@ -1158,7 +1158,7 @@ cleanup: · 305 unmodified lines
1158	assert_mprog_count(target, 0);	1158	assert_mprog_count(target, 0);
1159	}	1159	}
1160		1160
1161	void serial_test_tc_links_invalid(void)	1161	void test_ns_tc_links_invalid(void)
1162	{	1162	{
1163	test_tc_links_invalid_target(BPF_TCX_INGRESS);	1163	test_tc_links_invalid_target(BPF_TCX_INGRESS);
1164	test_tc_links_invalid_target(BPF_TCX_EGRESS);	1164	test_tc_links_invalid_target(BPF_TCX_EGRESS);
@@ -1314,7 +1314,7 @@ cleanup: · 149 unmodified lines
1314	assert_mprog_count(target, 0);	1314	assert_mprog_count(target, 0);
1315	}	1315	}
1316		1316
1317	void serial_test_tc_links_prepend(void)	1317	void test_ns_tc_links_prepend(void)
1318	{	1318	{
1319	test_tc_links_prepend_target(BPF_TCX_INGRESS);	1319	test_tc_links_prepend_target(BPF_TCX_INGRESS);
1320	test_tc_links_prepend_target(BPF_TCX_EGRESS);	1320	test_tc_links_prepend_target(BPF_TCX_EGRESS);
@@ -1470,7 +1470,7 @@ cleanup: · 149 unmodified lines
1470	assert_mprog_count(target, 0);	1470	assert_mprog_count(target, 0);
1471	}	1471	}
1472		1472
1473	void serial_test_tc_links_append(void)	1473	void test_ns_tc_links_append(void)
1474	{	1474	{
1475	test_tc_links_append_target(BPF_TCX_INGRESS);	1475	test_tc_links_append_target(BPF_TCX_INGRESS);
1476	test_tc_links_append_target(BPF_TCX_EGRESS);	1476	test_tc_links_append_target(BPF_TCX_EGRESS);
@@ -1568,7 +1568,7 @@ cleanup: · 91 unmodified lines
1568	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");	1568	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
1569	}	1569	}
1570		1570
1571	void serial_test_tc_links_dev_cleanup(void)	1571	void test_ns_tc_links_dev_cleanup(void)
1572	{	1572	{
1573	test_tc_links_dev_cleanup_target(BPF_TCX_INGRESS);	1573	test_tc_links_dev_cleanup_target(BPF_TCX_INGRESS);
1574	test_tc_links_dev_cleanup_target(BPF_TCX_EGRESS);	1574	test_tc_links_dev_cleanup_target(BPF_TCX_EGRESS);
@@ -1672,7 +1672,7 @@ cleanup: · 97 unmodified lines
1672	test_tc_link__destroy(skel);	1672	test_tc_link__destroy(skel);
1673	}	1673	}
1674		1674
1675	void serial_test_tc_links_chain_mixed(void)	1675	void test_ns_tc_links_chain_mixed(void)
1676	{	1676	{
1677	test_tc_chain_mixed(BPF_TCX_INGRESS);	1677	test_tc_chain_mixed(BPF_TCX_INGRESS);
1678	test_tc_chain_mixed(BPF_TCX_EGRESS);	1678	test_tc_chain_mixed(BPF_TCX_EGRESS);
@@ -1782,7 +1782,7 @@ cleanup: · 103 unmodified lines
1782	assert_mprog_count(target, 0);	1782	assert_mprog_count(target, 0);
1783	}	1783	}
1784		1784
1785	void serial_test_tc_links_ingress(void)	1785	void test_ns_tc_links_ingress(void)
1786	{	1786	{
1787	test_tc_links_ingress(BPF_TCX_INGRESS, true, true);	1787	test_tc_links_ingress(BPF_TCX_INGRESS, true, true);
1788	test_tc_links_ingress(BPF_TCX_INGRESS, true, false);	1788	test_tc_links_ingress(BPF_TCX_INGRESS, true, false);
@@ -1823,7 +1823,7 @@ static int qdisc_replace(int ifindex, const char *kind, bool block) · 34 unmodified lines
1823	return err;	1823	return err;
1824	}	1824	}
1825		1825
1826	void serial_test_tc_links_dev_chain0(void)	1826	void test_ns_tc_links_dev_chain0(void)
1827	{	1827	{
1828	int err, ifindex;	1828	int err, ifindex;
1829		1829
@@ -1955,7 +1955,7 @@ cleanup: · 125 unmodified lines
1955	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");	1955	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
1956	}	1956	}
1957		1957
1958	void serial_test_tc_links_dev_mixed(void)	1958	void test_ns_tc_links_dev_mixed(void)
1959	{	1959	{
1960	test_tc_links_dev_mixed(BPF_TCX_INGRESS);	1960	test_tc_links_dev_mixed(BPF_TCX_INGRESS);
1961	test_tc_links_dev_mixed(BPF_TCX_EGRESS);	1961	test_tc_links_dev_mixed(BPF_TCX_EGRESS);

tools/testing/selftests/bpf/prog_tests/tc_opts.c+20−20

@@ -10,7 +10,7 @@ · 9 unmodified lines
10	#include "test_tc_link.skel.h"	10	#include "test_tc_link.skel.h"
11	#include "tc_helpers.h"	11	#include "tc_helpers.h"
12		12
13	void serial_test_tc_opts_basic(void)	13	void test_ns_tc_opts_basic(void)
14	{	14	{
15	LIBBPF_OPTS(bpf_prog_attach_opts, opta);	15	LIBBPF_OPTS(bpf_prog_attach_opts, opta);
16	LIBBPF_OPTS(bpf_prog_detach_opts, optd);	16	LIBBPF_OPTS(bpf_prog_detach_opts, optd);
@@ -254,7 +254,7 @@ cleanup: · 237 unmodified lines
254	test_tc_link__destroy(skel);	254	test_tc_link__destroy(skel);
255	}	255	}
256		256
257	void serial_test_tc_opts_before(void)	257	void test_ns_tc_opts_before(void)
258	{	258	{
259	test_tc_opts_before_target(BPF_TCX_INGRESS);	259	test_tc_opts_before_target(BPF_TCX_INGRESS);
260	test_tc_opts_before_target(BPF_TCX_EGRESS);	260	test_tc_opts_before_target(BPF_TCX_EGRESS);
@@ -445,7 +445,7 @@ cleanup: · 184 unmodified lines
445	test_tc_link__destroy(skel);	445	test_tc_link__destroy(skel);
446	}	446	}
447		447
448	void serial_test_tc_opts_after(void)	448	void test_ns_tc_opts_after(void)
449	{	449	{
450	test_tc_opts_after_target(BPF_TCX_INGRESS);	450	test_tc_opts_after_target(BPF_TCX_INGRESS);
451	test_tc_opts_after_target(BPF_TCX_EGRESS);	451	test_tc_opts_after_target(BPF_TCX_EGRESS);
@@ -554,7 +554,7 @@ cleanup: · 102 unmodified lines
554	test_tc_link__destroy(skel);	554	test_tc_link__destroy(skel);
555	}	555	}
556		556
557	void serial_test_tc_opts_revision(void)	557	void test_ns_tc_opts_revision(void)
558	{	558	{
559	test_tc_opts_revision_target(BPF_TCX_INGRESS);	559	test_tc_opts_revision_target(BPF_TCX_INGRESS);
560	test_tc_opts_revision_target(BPF_TCX_EGRESS);	560	test_tc_opts_revision_target(BPF_TCX_EGRESS);
@@ -655,7 +655,7 @@ cleanup: · 94 unmodified lines
655	assert_mprog_count(target, 0);	655	assert_mprog_count(target, 0);
656	}	656	}
657		657
658	void serial_test_tc_opts_chain_classic(void)	658	void test_ns_tc_opts_chain_classic(void)
659	{	659	{
660	test_tc_chain_classic(BPF_TCX_INGRESS, false);	660	test_tc_chain_classic(BPF_TCX_INGRESS, false);
661	test_tc_chain_classic(BPF_TCX_EGRESS, false);	661	test_tc_chain_classic(BPF_TCX_EGRESS, false);
@@ -864,7 +864,7 @@ cleanup: · 202 unmodified lines
864	test_tc_link__destroy(skel);	864	test_tc_link__destroy(skel);
865	}	865	}
866		866
867	void serial_test_tc_opts_replace(void)	867	void test_ns_tc_opts_replace(void)
868	{	868	{
869	test_tc_opts_replace_target(BPF_TCX_INGRESS);	869	test_tc_opts_replace_target(BPF_TCX_INGRESS);
870	test_tc_opts_replace_target(BPF_TCX_EGRESS);	870	test_tc_opts_replace_target(BPF_TCX_EGRESS);
@@ -1017,7 +1017,7 @@ cleanup: · 146 unmodified lines
1017	test_tc_link__destroy(skel);	1017	test_tc_link__destroy(skel);
1018	}	1018	}
1019		1019
1020	void serial_test_tc_opts_invalid(void)	1020	void test_ns_tc_opts_invalid(void)
1021	{	1021	{
1022	test_tc_opts_invalid_target(BPF_TCX_INGRESS);	1022	test_tc_opts_invalid_target(BPF_TCX_INGRESS);
1023	test_tc_opts_invalid_target(BPF_TCX_EGRESS);	1023	test_tc_opts_invalid_target(BPF_TCX_EGRESS);
@@ -1157,7 +1157,7 @@ cleanup: · 133 unmodified lines
1157	test_tc_link__destroy(skel);	1157	test_tc_link__destroy(skel);
1158	}	1158	}
1159		1159
1160	void serial_test_tc_opts_prepend(void)	1160	void test_ns_tc_opts_prepend(void)
1161	{	1161	{
1162	test_tc_opts_prepend_target(BPF_TCX_INGRESS);	1162	test_tc_opts_prepend_target(BPF_TCX_INGRESS);
1163	test_tc_opts_prepend_target(BPF_TCX_EGRESS);	1163	test_tc_opts_prepend_target(BPF_TCX_EGRESS);
@@ -1297,7 +1297,7 @@ cleanup: · 133 unmodified lines
1297	test_tc_link__destroy(skel);	1297	test_tc_link__destroy(skel);
1298	}	1298	}
1299		1299
1300	void serial_test_tc_opts_append(void)	1300	void test_ns_tc_opts_append(void)
1301	{	1301	{
1302	test_tc_opts_append_target(BPF_TCX_INGRESS);	1302	test_tc_opts_append_target(BPF_TCX_INGRESS);
1303	test_tc_opts_append_target(BPF_TCX_EGRESS);	1303	test_tc_opts_append_target(BPF_TCX_EGRESS);
@@ -1387,7 +1387,7 @@ cleanup: · 83 unmodified lines
1387	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");	1387	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
1388	}	1388	}
1389		1389
1390	void serial_test_tc_opts_dev_cleanup(void)	1390	void test_ns_tc_opts_dev_cleanup(void)
1391	{	1391	{
1392	test_tc_opts_dev_cleanup_target(BPF_TCX_INGRESS);	1392	test_tc_opts_dev_cleanup_target(BPF_TCX_INGRESS);
1393	test_tc_opts_dev_cleanup_target(BPF_TCX_EGRESS);	1393	test_tc_opts_dev_cleanup_target(BPF_TCX_EGRESS);
@@ -1563,7 +1563,7 @@ cleanup: · 169 unmodified lines
1563	assert_mprog_count(target, 0);	1563	assert_mprog_count(target, 0);
1564	}	1564	}
1565		1565
1566	void serial_test_tc_opts_mixed(void)	1566	void test_ns_tc_opts_mixed(void)
1567	{	1567	{
1568	test_tc_opts_mixed_target(BPF_TCX_INGRESS);	1568	test_tc_opts_mixed_target(BPF_TCX_INGRESS);
1569	test_tc_opts_mixed_target(BPF_TCX_EGRESS);	1569	test_tc_opts_mixed_target(BPF_TCX_EGRESS);
@@ -1642,7 +1642,7 @@ cleanup: · 72 unmodified lines
1642	assert_mprog_count(target, 0);	1642	assert_mprog_count(target, 0);
1643	}	1643	}
1644		1644
1645	void serial_test_tc_opts_demixed(void)	1645	void test_ns_tc_opts_demixed(void)
1646	{	1646	{
1647	test_tc_opts_demixed_target(BPF_TCX_INGRESS);	1647	test_tc_opts_demixed_target(BPF_TCX_INGRESS);
1648	test_tc_opts_demixed_target(BPF_TCX_EGRESS);	1648	test_tc_opts_demixed_target(BPF_TCX_EGRESS);
@@ -1813,7 +1813,7 @@ cleanup: · 164 unmodified lines
1813	test_tc_link__destroy(skel);	1813	test_tc_link__destroy(skel);
1814	}	1814	}
1815		1815
1816	void serial_test_tc_opts_detach(void)	1816	void test_ns_tc_opts_detach(void)
1817	{	1817	{
1818	test_tc_opts_detach_target(BPF_TCX_INGRESS);	1818	test_tc_opts_detach_target(BPF_TCX_INGRESS);
1819	test_tc_opts_detach_target(BPF_TCX_EGRESS);	1819	test_tc_opts_detach_target(BPF_TCX_EGRESS);
@@ -2020,7 +2020,7 @@ cleanup: · 200 unmodified lines
2020	test_tc_link__destroy(skel);	2020	test_tc_link__destroy(skel);
2021	}	2021	}
2022		2022
2023	void serial_test_tc_opts_detach_before(void)	2023	void test_ns_tc_opts_detach_before(void)
2024	{	2024	{
2025	test_tc_opts_detach_before_target(BPF_TCX_INGRESS);	2025	test_tc_opts_detach_before_target(BPF_TCX_INGRESS);
2026	test_tc_opts_detach_before_target(BPF_TCX_EGRESS);	2026	test_tc_opts_detach_before_target(BPF_TCX_EGRESS);
@@ -2236,7 +2236,7 @@ cleanup: · 209 unmodified lines
2236	test_tc_link__destroy(skel);	2236	test_tc_link__destroy(skel);
2237	}	2237	}
2238		2238
2239	void serial_test_tc_opts_detach_after(void)	2239	void test_ns_tc_opts_detach_after(void)
2240	{	2240	{
2241	test_tc_opts_detach_after_target(BPF_TCX_INGRESS);	2241	test_tc_opts_detach_after_target(BPF_TCX_INGRESS);
2242	test_tc_opts_detach_after_target(BPF_TCX_EGRESS);	2242	test_tc_opts_detach_after_target(BPF_TCX_EGRESS);
@@ -2265,7 +2265,7 @@ static void test_tc_opts_delete_empty(int target, bool chain_tc_old) · 22 unmodified lines
2265	assert_mprog_count(target, 0);	2265	assert_mprog_count(target, 0);
2266	}	2266	}
2267		2267
2268	void serial_test_tc_opts_delete_empty(void)	2268	void test_ns_tc_opts_delete_empty(void)
2269	{	2269	{
2270	test_tc_opts_delete_empty(BPF_TCX_INGRESS, false);	2270	test_tc_opts_delete_empty(BPF_TCX_INGRESS, false);
2271	test_tc_opts_delete_empty(BPF_TCX_EGRESS, false);	2271	test_tc_opts_delete_empty(BPF_TCX_EGRESS, false);
@@ -2372,7 +2372,7 @@ cleanup: · 100 unmodified lines
2372	test_tc_link__destroy(skel);	2372	test_tc_link__destroy(skel);
2373	}	2373	}
2374		2374
2375	void serial_test_tc_opts_chain_mixed(void)	2375	void test_ns_tc_opts_chain_mixed(void)
2376	{	2376	{
2377	test_tc_chain_mixed(BPF_TCX_INGRESS);	2377	test_tc_chain_mixed(BPF_TCX_INGRESS);
2378	test_tc_chain_mixed(BPF_TCX_EGRESS);	2378	test_tc_chain_mixed(BPF_TCX_EGRESS);
@@ -2446,7 +2446,7 @@ cleanup: · 67 unmodified lines
2446	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");	2446	ASSERT_EQ(if_nametoindex("tcx_opts2"), 0, "dev2_removed");
2447	}	2447	}
2448		2448
2449	void serial_test_tc_opts_max(void)	2449	void test_ns_tc_opts_max(void)
2450	{	2450	{
2451	test_tc_opts_max_target(BPF_TCX_INGRESS, 0, false);	2451	test_tc_opts_max_target(BPF_TCX_INGRESS, 0, false);
2452	test_tc_opts_max_target(BPF_TCX_EGRESS, 0, false);	2452	test_tc_opts_max_target(BPF_TCX_EGRESS, 0, false);
@@ -2748,7 +2748,7 @@ cleanup: · 295 unmodified lines
2748	test_tc_link__destroy(skel);	2748	test_tc_link__destroy(skel);
2749	}	2749	}
2750		2750
2751	void serial_test_tc_opts_query(void)	2751	void test_ns_tc_opts_query(void)
2752	{	2752	{
2753	test_tc_opts_query_target(BPF_TCX_INGRESS);	2753	test_tc_opts_query_target(BPF_TCX_INGRESS);
2754	test_tc_opts_query_target(BPF_TCX_EGRESS);	2754	test_tc_opts_query_target(BPF_TCX_EGRESS);
@@ -2807,7 +2807,7 @@ cleanup: · 52 unmodified lines
2807	test_tc_link__destroy(skel);	2807	test_tc_link__destroy(skel);
2808	}	2808	}
2809		2809
2810	void serial_test_tc_opts_query_attach(void)	2810	void test_ns_tc_opts_query_attach(void)
2811	{	2811	{
2812	test_tc_opts_query_attach_target(BPF_TCX_INGRESS);	2812	test_tc_opts_query_attach_target(BPF_TCX_INGRESS);
2813	test_tc_opts_query_attach_target(BPF_TCX_EGRESS);	2813	test_tc_opts_query_attach_target(BPF_TCX_EGRESS);

tools/testing/selftests/bpf/prog_tests/test_struct_ops_kptr_return.c+16−0

@@ -0,0 +1,16 @@
	1	#include <test_progs.h>
	2
	3	#include "struct_ops_kptr_return.skel.h"
	4	#include "struct_ops_kptr_return_fail__wrong_type.skel.h"
	5	#include "struct_ops_kptr_return_fail__invalid_scalar.skel.h"
	6	#include "struct_ops_kptr_return_fail__nonzero_offset.skel.h"
	7	#include "struct_ops_kptr_return_fail__local_kptr.skel.h"
	8
	9	void test_struct_ops_kptr_return(void)
	10	{
	11	RUN_TESTS(struct_ops_kptr_return);
	12	RUN_TESTS(struct_ops_kptr_return_fail__wrong_type);
	13	RUN_TESTS(struct_ops_kptr_return_fail__invalid_scalar);
	14	RUN_TESTS(struct_ops_kptr_return_fail__nonzero_offset);
	15	RUN_TESTS(struct_ops_kptr_return_fail__local_kptr);
	16	}

tools/testing/selftests/bpf/prog_tests/test_struct_ops_refcounted.c+14−0

@@ -0,0 +1,14 @@
	1	#include <test_progs.h>
	2
	3	#include "struct_ops_refcounted.skel.h"
	4	#include "struct_ops_refcounted_fail__ref_leak.skel.h"
	5	#include "struct_ops_refcounted_fail__global_subprog.skel.h"
	6	#include "struct_ops_refcounted_fail__tail_call.skel.h"
	7
	8	void test_struct_ops_refcounted(void)
	9	{
	10	RUN_TESTS(struct_ops_refcounted);
	11	RUN_TESTS(struct_ops_refcounted_fail__ref_leak);
	12	RUN_TESTS(struct_ops_refcounted_fail__global_subprog);
	13	RUN_TESTS(struct_ops_refcounted_fail__tail_call);
	14	}

tools/testing/selftests/bpf/prog_tests/test_tunnel.c+530−103

@@ -71,6 +71,8 @@ · 70 unmodified lines
71	#define IP4_ADDR2_VETH1 "172.16.1.20"	71	#define IP4_ADDR2_VETH1 "172.16.1.20"
72	#define IP4_ADDR_TUNL_DEV0 "10.1.1.100"	72	#define IP4_ADDR_TUNL_DEV0 "10.1.1.100"
73	#define IP4_ADDR_TUNL_DEV1 "10.1.1.200"	73	#define IP4_ADDR_TUNL_DEV1 "10.1.1.200"
		74	#define IP6_ADDR_TUNL_DEV0 "fc80::100"
		75	#define IP6_ADDR_TUNL_DEV1 "fc80::200"
74		76
75	#define IP6_ADDR_VETH0 "::11"	77	#define IP6_ADDR_VETH0 "::11"
76	#define IP6_ADDR1_VETH1 "::22"	78	#define IP6_ADDR1_VETH1 "::22"
@@ -98,6 +100,27 @@ · 21 unmodified lines
98	#define XFRM_SPI_IN_TO_OUT 0x1	100	#define XFRM_SPI_IN_TO_OUT 0x1
99	#define XFRM_SPI_OUT_TO_IN 0x2	101	#define XFRM_SPI_OUT_TO_IN 0x2
100		102
		103	#define GRE_TUNL_DEV0 "gre00"
		104	#define GRE_TUNL_DEV1 "gre11"
		105
		106	#define IP6GRE_TUNL_DEV0 "ip6gre00"
		107	#define IP6GRE_TUNL_DEV1 "ip6gre11"
		108
		109	#define ERSPAN_TUNL_DEV0 "erspan00"
		110	#define ERSPAN_TUNL_DEV1 "erspan11"
		111
		112	#define IP6ERSPAN_TUNL_DEV0 "ip6erspan00"
		113	#define IP6ERSPAN_TUNL_DEV1 "ip6erspan11"
		114
		115	#define GENEVE_TUNL_DEV0 "geneve00"
		116	#define GENEVE_TUNL_DEV1 "geneve11"
		117
		118	#define IP6GENEVE_TUNL_DEV0 "ip6geneve00"
		119	#define IP6GENEVE_TUNL_DEV1 "ip6geneve11"
		120
		121	#define IP6TNL_TUNL_DEV0 "ip6tnl00"
		122	#define IP6TNL_TUNL_DEV1 "ip6tnl11"
		123
101	#define PING_ARGS "-i 0.01 -c 3 -w 10 -q"	124	#define PING_ARGS "-i 0.01 -c 3 -w 10 -q"
102		125
103	static int config_device(void)	126	static int config_device(void)
@@ -216,6 +239,18 @@ fail: · 112 unmodified lines
216	return -1;	239	return -1;
217	}	240	}
218		241
		242	static int set_ipv4_addr(const char dev0, const char dev1)
		243	{
		244	SYS(fail, "ip -n at_ns0 link set dev %s up", dev0);
		245	SYS(fail, "ip -n at_ns0 addr add dev %s %s/24", dev0, IP4_ADDR_TUNL_DEV0);
		246	SYS(fail, "ip link set dev %s up", dev1);
		247	SYS(fail, "ip addr add dev %s %s/24", dev1, IP4_ADDR_TUNL_DEV1);
		248
		249	return 0;
		250	fail:
		251	return 1;
		252	}
		253
219	static int add_ipip_tunnel(enum ipip_encap encap)	254	static int add_ipip_tunnel(enum ipip_encap encap)
220	{	255	{
221	int err;	256	int err;
@@ -356,6 +391,99 @@ static void delete_xfrm_tunnel(void) · 134 unmodified lines
356	IP4_ADDR1_VETH1, IP4_ADDR_VETH0, XFRM_SPI_OUT_TO_IN);	391	IP4_ADDR1_VETH1, IP4_ADDR_VETH0, XFRM_SPI_OUT_TO_IN);
357	}	392	}
358		393
		394	static int add_ipv4_tunnel(const char dev0, const char dev1,
		395	const char type, const char opt)
		396	{
		397	if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
		398	return -1;
		399
		400	SYS(fail, "ip -n at_ns0 link add dev %s type %s %s local %s remote %s",
		401	dev0, type, opt, IP4_ADDR_VETH0, IP4_ADDR1_VETH1);
		402
		403	SYS(fail, "ip link add dev %s type %s external", dev1, type);
		404
		405	return set_ipv4_addr(dev0, dev1);
		406	fail:
		407	return -1;
		408	}
		409
		410	static void delete_tunnel(const char dev0, const char dev1)
		411	{
		412	if (!dev0 \|\| !dev1)
		413	return;
		414
		415	SYS_NOFAIL("ip netns exec at_ns0 ip link delete dev %s", dev0);
		416	SYS_NOFAIL("ip link delete dev %s", dev1);
		417	}
		418
		419	static int set_ipv6_addr(const char dev0, const char dev1)
		420	{
		421	/* disable IPv6 DAD because it might take too long and fail tests */
		422	SYS(fail, "ip -n at_ns0 addr add %s/96 dev veth0 nodad", IP6_ADDR_VETH0);
		423	SYS(fail, "ip -n at_ns0 link set dev veth0 up");
		424	SYS(fail, "ip addr add %s/96 dev veth1 nodad", IP6_ADDR1_VETH1);
		425	SYS(fail, "ip link set dev veth1 up");
		426
		427	SYS(fail, "ip -n at_ns0 addr add dev %s %s/24", dev0, IP4_ADDR_TUNL_DEV0);
		428	SYS(fail, "ip -n at_ns0 addr add dev %s %s/96 nodad", dev0, IP6_ADDR_TUNL_DEV0);
		429	SYS(fail, "ip -n at_ns0 link set dev %s up", dev0);
		430
		431	SYS(fail, "ip addr add dev %s %s/24", dev1, IP4_ADDR_TUNL_DEV1);
		432	SYS(fail, "ip addr add dev %s %s/96 nodad", dev1, IP6_ADDR_TUNL_DEV1);
		433	SYS(fail, "ip link set dev %s up", dev1);
		434	return 0;
		435	fail:
		436	return 1;
		437	}
		438
		439	static int add_ipv6_tunnel(const char dev0, const char dev1,
		440	const char type, const char opt)
		441	{
		442	if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
		443	return -1;
		444
		445	SYS(fail, "ip -n at_ns0 link add dev %s type %s %s local %s remote %s",
		446	dev0, type, opt, IP6_ADDR_VETH0, IP6_ADDR1_VETH1);
		447
		448	SYS(fail, "ip link add dev %s type %s external", dev1, type);
		449
		450	return set_ipv6_addr(dev0, dev1);
		451	fail:
		452	return -1;
		453	}
		454
		455	static int add_geneve_tunnel(const char dev0, const char dev1,
		456	const char type, const char opt)
		457	{
		458	if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
		459	return -1;
		460
		461	SYS(fail, "ip -n at_ns0 link add dev %s type %s id 2 %s remote %s",
		462	dev0, type, opt, IP4_ADDR1_VETH1);
		463
		464	SYS(fail, "ip link add dev %s type %s %s external", dev1, type, opt);
		465
		466	return set_ipv4_addr(dev0, dev1);
		467	fail:
		468	return -1;
		469	}
		470
		471	static int add_ip6geneve_tunnel(const char dev0, const char dev1,
		472	const char type, const char opt)
		473	{
		474	if (!type \|\| !opt \|\| !dev0 \|\| !dev1)
		475	return -1;
		476
		477	SYS(fail, "ip -n at_ns0 link add dev %s type %s id 22 %s remote %s",
		478	dev0, type, opt, IP6_ADDR1_VETH1);
		479
		480	SYS(fail, "ip link add dev %s type %s %s external", dev1, type, opt);
		481
		482	return set_ipv6_addr(dev0, dev1);
		483	fail:
		484	return -1;
		485	}
		486
359	static int test_ping(int family, const char *addr)	487	static int test_ping(int family, const char *addr)
360	{	488	{
361	SYS(fail, "%s %s %s > /dev/null", ping_command(family), PING_ARGS, addr);	489	SYS(fail, "%s %s %s > /dev/null", ping_command(family), PING_ARGS, addr);
@@ -364,32 +492,76 @@ fail: · 2 unmodified lines
364	return -1;	492	return -1;
365	}	493	}
366		494
367	static int attach_tc_prog(struct bpf_tc_hook *hook, int igr_fd, int egr_fd)	495	static void ping_dev0(void)
368	{	496	{
		497	/* ping from root namespace test */
		498	test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);
		499	}
		500
		501	static void ping_dev1(void)
		502	{
		503	struct nstoken *nstoken;
		504
		505	/* ping from at_ns0 namespace test */
		506	nstoken = open_netns("at_ns0");
		507	if (!ASSERT_OK_PTR(nstoken, "setns"))
		508	return;
		509
		510	test_ping(AF_INET, IP4_ADDR_TUNL_DEV1);
		511	close_netns(nstoken);
		512	}
		513
		514	static void ping6_veth0(void)
		515	{
		516	test_ping(AF_INET6, IP6_ADDR_VETH0);
		517	}
		518
		519	static void ping6_dev0(void)
		520	{
		521	test_ping(AF_INET6, IP6_ADDR_TUNL_DEV0);
		522	}
		523
		524	static void ping6_dev1(void)
		525	{
		526	struct nstoken *nstoken;
		527
		528	/* ping from at_ns0 namespace test */
		529	nstoken = open_netns("at_ns0");
		530	if (!ASSERT_OK_PTR(nstoken, "setns"))
		531	return;
		532
		533	test_ping(AF_INET, IP6_ADDR_TUNL_DEV1);
		534	close_netns(nstoken);
		535	}
		536
		537	static int attach_tc_prog(int ifindex, int igr_fd, int egr_fd)
		538	{
		539	DECLARE_LIBBPF_OPTS(bpf_tc_hook, hook, .ifindex = ifindex,
		540	.attach_point = BPF_TC_INGRESS \| BPF_TC_EGRESS);
369	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts1, .handle = 1,	541	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts1, .handle = 1,
370	.priority = 1, .prog_fd = igr_fd);	542	.priority = 1, .prog_fd = igr_fd);
371	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts2, .handle = 1,	543	DECLARE_LIBBPF_OPTS(bpf_tc_opts, opts2, .handle = 1,
372	.priority = 1, .prog_fd = egr_fd);	544	.priority = 1, .prog_fd = egr_fd);
373	int ret;	545	int ret;
374		546
375	ret = bpf_tc_hook_create(hook);	547	ret = bpf_tc_hook_create(&hook);
376	if (!ASSERT_OK(ret, "create tc hook"))	548	if (!ASSERT_OK(ret, "create tc hook"))
377	return ret;	549	return ret;
378		550
379	if (igr_fd >= 0) {	551	if (igr_fd >= 0) {
380	hook->attach_point = BPF_TC_INGRESS;	552	hook.attach_point = BPF_TC_INGRESS;
381	ret = bpf_tc_attach(hook, &opts1);	553	ret = bpf_tc_attach(&hook, &opts1);
382	if (!ASSERT_OK(ret, "bpf_tc_attach")) {	554	if (!ASSERT_OK(ret, "bpf_tc_attach")) {
383	bpf_tc_hook_destroy(hook);	555	bpf_tc_hook_destroy(&hook);
384	return ret;	556	return ret;
385	}	557	}
386	}	558	}
387		559
388	if (egr_fd >= 0) {	560	if (egr_fd >= 0) {
389	hook->attach_point = BPF_TC_EGRESS;	561	hook.attach_point = BPF_TC_EGRESS;
390	ret = bpf_tc_attach(hook, &opts2);	562	ret = bpf_tc_attach(&hook, &opts2);
391	if (!ASSERT_OK(ret, "bpf_tc_attach")) {	563	if (!ASSERT_OK(ret, "bpf_tc_attach")) {
392	bpf_tc_hook_destroy(hook);	564	bpf_tc_hook_destroy(&hook);
393	return ret;	565	return ret;
394	}	566	}
395	}	567	}
@@ -397,6 +569,50 @@ static int attach_tc_prog(struct bpf_tc_hook *hook, int igr_fd, int egr_fd) · 1 unmodified line
397	return 0;	569	return 0;
398	}	570	}
399		571
		572	static int generic_attach(const char *dev, int igr_fd, int egr_fd)
		573	{
		574	int ifindex;
		575
		576	if (!ASSERT_OK_FD(igr_fd, "check ingress fd"))
		577	return -1;
		578	if (!ASSERT_OK_FD(egr_fd, "check egress fd"))
		579	return -1;
		580
		581	ifindex = if_nametoindex(dev);
		582	if (!ASSERT_NEQ(ifindex, 0, "get ifindex"))
		583	return -1;
		584
		585	return attach_tc_prog(ifindex, igr_fd, egr_fd);
		586	}
		587
		588	static int generic_attach_igr(const char *dev, int igr_fd)
		589	{
		590	int ifindex;
		591
		592	if (!ASSERT_OK_FD(igr_fd, "check ingress fd"))
		593	return -1;
		594
		595	ifindex = if_nametoindex(dev);
		596	if (!ASSERT_NEQ(ifindex, 0, "get ifindex"))
		597	return -1;
		598
		599	return attach_tc_prog(ifindex, igr_fd, -1);
		600	}
		601
		602	static int generic_attach_egr(const char *dev, int egr_fd)
		603	{
		604	int ifindex;
		605
		606	if (!ASSERT_OK_FD(egr_fd, "check egress fd"))
		607	return -1;
		608
		609	ifindex = if_nametoindex(dev);
		610	if (!ASSERT_NEQ(ifindex, 0, "get ifindex"))
		611	return -1;
		612
		613	return attach_tc_prog(ifindex, -1, egr_fd);
		614	}
		615
400	static void test_vxlan_tunnel(void)	616	static void test_vxlan_tunnel(void)
401	{	617	{
402	struct test_tunnel_kern *skel = NULL;	618	struct test_tunnel_kern *skel = NULL;
@@ -404,11 +620,9 @@ static void test_vxlan_tunnel(void) · 1 unmodified line
404	int local_ip_map_fd = -1;	620	int local_ip_map_fd = -1;
405	int set_src_prog_fd, get_src_prog_fd;	621	int set_src_prog_fd, get_src_prog_fd;
406	int set_dst_prog_fd;	622	int set_dst_prog_fd;
407	int key = 0, ifindex = -1;	623	int key = 0;
408	uint local_ip;	624	uint local_ip;
409	int err;	625	int err;
410	DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
411	.attach_point = BPF_TC_INGRESS);
412		626
413	/* add vxlan tunnel */	627	/* add vxlan tunnel */
414	err = add_vxlan_tunnel();	628	err = add_vxlan_tunnel();
@@ -419,42 +633,22 @@ static void test_vxlan_tunnel(void) · 4 unmodified lines
419	skel = test_tunnel_kern__open_and_load();	633	skel = test_tunnel_kern__open_and_load();
420	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))	634	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
421	goto done;	635	goto done;
422	ifindex = if_nametoindex(VXLAN_TUNL_DEV1);
423	if (!ASSERT_NEQ(ifindex, 0, "vxlan11 ifindex"))
424	goto done;
425	tc_hook.ifindex = ifindex;
426	get_src_prog_fd = bpf_program__fd(skel->progs.vxlan_get_tunnel_src);	636	get_src_prog_fd = bpf_program__fd(skel->progs.vxlan_get_tunnel_src);
427	set_src_prog_fd = bpf_program__fd(skel->progs.vxlan_set_tunnel_src);	637	set_src_prog_fd = bpf_program__fd(skel->progs.vxlan_set_tunnel_src);
428	if (!ASSERT_GE(get_src_prog_fd, 0, "bpf_program__fd"))	638	if (generic_attach(VXLAN_TUNL_DEV1, get_src_prog_fd, set_src_prog_fd))
429	goto done;
430	if (!ASSERT_GE(set_src_prog_fd, 0, "bpf_program__fd"))
431	goto done;
432	if (attach_tc_prog(&tc_hook, get_src_prog_fd, set_src_prog_fd))
433	goto done;	639	goto done;
434		640
435	/* load and attach bpf prog to veth dev tc hook point */	641	/* load and attach bpf prog to veth dev tc hook point */
436	ifindex = if_nametoindex("veth1");
437	if (!ASSERT_NEQ(ifindex, 0, "veth1 ifindex"))
438	goto done;
439	tc_hook.ifindex = ifindex;
440	set_dst_prog_fd = bpf_program__fd(skel->progs.veth_set_outer_dst);	642	set_dst_prog_fd = bpf_program__fd(skel->progs.veth_set_outer_dst);
441	if (!ASSERT_GE(set_dst_prog_fd, 0, "bpf_program__fd"))	643	if (generic_attach_igr("veth1", set_dst_prog_fd))
442	goto done;
443	if (attach_tc_prog(&tc_hook, set_dst_prog_fd, -1))
444	goto done;	644	goto done;
445		645
446	/* load and attach prog set_md to tunnel dev tc hook point at_ns0 */	646	/* load and attach prog set_md to tunnel dev tc hook point at_ns0 */
447	nstoken = open_netns("at_ns0");	647	nstoken = open_netns("at_ns0");
448	if (!ASSERT_OK_PTR(nstoken, "setns src"))	648	if (!ASSERT_OK_PTR(nstoken, "setns src"))
449	goto done;	649	goto done;
450	ifindex = if_nametoindex(VXLAN_TUNL_DEV0);
451	if (!ASSERT_NEQ(ifindex, 0, "vxlan00 ifindex"))
452	goto done;
453	tc_hook.ifindex = ifindex;
454	set_dst_prog_fd = bpf_program__fd(skel->progs.vxlan_set_tunnel_dst);	650	set_dst_prog_fd = bpf_program__fd(skel->progs.vxlan_set_tunnel_dst);
455	if (!ASSERT_GE(set_dst_prog_fd, 0, "bpf_program__fd"))	651	if (generic_attach_egr(VXLAN_TUNL_DEV0, set_dst_prog_fd))
456	goto done;
457	if (attach_tc_prog(&tc_hook, -1, set_dst_prog_fd))
458	goto done;	652	goto done;
459	close_netns(nstoken);	653	close_netns(nstoken);
460		654
@@ -468,9 +662,7 @@ static void test_vxlan_tunnel(void) · 7 unmodified lines
468	goto done;	662	goto done;
469		663
470	/* ping test */	664	/* ping test */
471	err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);	665	ping_dev0();
472	if (!ASSERT_OK(err, "test_ping"))
473	goto done;
474		666
475	done:	667	done:
476	/* delete vxlan tunnel */	668	/* delete vxlan tunnel */
@@ -488,11 +680,9 @@ static void test_ip6vxlan_tunnel(void) · 11 unmodified lines
488	int local_ip_map_fd = -1;	680	int local_ip_map_fd = -1;
489	int set_src_prog_fd, get_src_prog_fd;	681	int set_src_prog_fd, get_src_prog_fd;
490	int set_dst_prog_fd;	682	int set_dst_prog_fd;
491	int key = 0, ifindex = -1;	683	int key = 0;
492	uint local_ip;	684	uint local_ip;
493	int err;	685	int err;
494	DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
495	.attach_point = BPF_TC_INGRESS);
496		686
497	/* add vxlan tunnel */	687	/* add vxlan tunnel */
498	err = add_ip6vxlan_tunnel();	688	err = add_ip6vxlan_tunnel();
@@ -503,31 +693,17 @@ static void test_ip6vxlan_tunnel(void) · 4 unmodified lines
503	skel = test_tunnel_kern__open_and_load();	693	skel = test_tunnel_kern__open_and_load();
504	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))	694	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
505	goto done;	695	goto done;
506	ifindex = if_nametoindex(IP6VXLAN_TUNL_DEV1);
507	if (!ASSERT_NEQ(ifindex, 0, "ip6vxlan11 ifindex"))
508	goto done;
509	tc_hook.ifindex = ifindex;
510	get_src_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_get_tunnel_src);	696	get_src_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_get_tunnel_src);
511	set_src_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_set_tunnel_src);	697	set_src_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_set_tunnel_src);
512	if (!ASSERT_GE(set_src_prog_fd, 0, "bpf_program__fd"))	698	if (generic_attach(IP6VXLAN_TUNL_DEV1, get_src_prog_fd, set_src_prog_fd))
513	goto done;
514	if (!ASSERT_GE(get_src_prog_fd, 0, "bpf_program__fd"))
515	goto done;
516	if (attach_tc_prog(&tc_hook, get_src_prog_fd, set_src_prog_fd))
517	goto done;	699	goto done;
518		700
519	/* load and attach prog set_md to tunnel dev tc hook point at_ns0 */	701	/* load and attach prog set_md to tunnel dev tc hook point at_ns0 */
520	nstoken = open_netns("at_ns0");	702	nstoken = open_netns("at_ns0");
521	if (!ASSERT_OK_PTR(nstoken, "setns src"))	703	if (!ASSERT_OK_PTR(nstoken, "setns src"))
522	goto done;	704	goto done;
523	ifindex = if_nametoindex(IP6VXLAN_TUNL_DEV0);
524	if (!ASSERT_NEQ(ifindex, 0, "ip6vxlan00 ifindex"))
525	goto done;
526	tc_hook.ifindex = ifindex;
527	set_dst_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_set_tunnel_dst);	705	set_dst_prog_fd = bpf_program__fd(skel->progs.ip6vxlan_set_tunnel_dst);
528	if (!ASSERT_GE(set_dst_prog_fd, 0, "bpf_program__fd"))	706	if (generic_attach_egr(IP6VXLAN_TUNL_DEV0, set_dst_prog_fd))
529	goto done;
530	if (attach_tc_prog(&tc_hook, -1, set_dst_prog_fd))
531	goto done;	707	goto done;
532	close_netns(nstoken);	708	close_netns(nstoken);
533		709
@@ -541,9 +717,7 @@ static void test_ip6vxlan_tunnel(void) · 7 unmodified lines
541	goto done;	717	goto done;
542		718
543	/* ping test */	719	/* ping test */
544	err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);	720	ping_dev0();
545	if (!ASSERT_OK(err, "test_ping"))
546	goto done;
547		721
548	done:	722	done:
549	/* delete ipv6 vxlan tunnel */	723	/* delete ipv6 vxlan tunnel */
@@ -557,12 +731,8 @@ done: · 7 unmodified lines
557	static void test_ipip_tunnel(enum ipip_encap encap)	731	static void test_ipip_tunnel(enum ipip_encap encap)
558	{	732	{
559	struct test_tunnel_kern *skel = NULL;	733	struct test_tunnel_kern *skel = NULL;
560	struct nstoken *nstoken;
561	int set_src_prog_fd, get_src_prog_fd;	734	int set_src_prog_fd, get_src_prog_fd;
562	int ifindex = -1;
563	int err;	735	int err;
564	DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
565	.attach_point = BPF_TC_INGRESS);
566		736
567	/* add ipip tunnel */	737	/* add ipip tunnel */
568	err = add_ipip_tunnel(encap);	738	err = add_ipip_tunnel(encap);
@@ -573,10 +743,6 @@ static void test_ipip_tunnel(enum ipip_encap encap) · 4 unmodified lines
573	skel = test_tunnel_kern__open_and_load();	743	skel = test_tunnel_kern__open_and_load();
574	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))	744	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
575	goto done;	745	goto done;
576	ifindex = if_nametoindex(IPIP_TUNL_DEV1);
577	if (!ASSERT_NEQ(ifindex, 0, "ipip11 ifindex"))
578	goto done;
579	tc_hook.ifindex = ifindex;
580		746
581	switch (encap) {	747	switch (encap) {
582	case FOU:	748	case FOU:
@@ -598,26 +764,11 @@ static void test_ipip_tunnel(enum ipip_encap encap) · 15 unmodified lines
598	skel->progs.ipip_set_tunnel);	764	skel->progs.ipip_set_tunnel);
599	}	765	}
600		766
601	if (!ASSERT_GE(set_src_prog_fd, 0, "bpf_program__fd"))	767	if (generic_attach(IPIP_TUNL_DEV1, get_src_prog_fd, set_src_prog_fd))
602	goto done;
603	if (!ASSERT_GE(get_src_prog_fd, 0, "bpf_program__fd"))
604	goto done;
605	if (attach_tc_prog(&tc_hook, get_src_prog_fd, set_src_prog_fd))
606	goto done;	768	goto done;
607		769
608	/* ping from root namespace test */	770	ping_dev0();
609	err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV0);	771	ping_dev1();
610	if (!ASSERT_OK(err, "test_ping"))
611	goto done;
612
613	/* ping from at_ns0 namespace test */
614	nstoken = open_netns("at_ns0");
615	if (!ASSERT_OK_PTR(nstoken, "setns"))
616	goto done;
617	err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV1);
618	if (!ASSERT_OK(err, "test_ping"))
619	goto done;
620	close_netns(nstoken);
621		772
622	done:	773	done:
623	/* delete ipip tunnel */	774	/* delete ipip tunnel */
@@ -628,11 +779,8 @@ done: · 4 unmodified lines
628		779
629	static void test_xfrm_tunnel(void)	780	static void test_xfrm_tunnel(void)
630	{	781	{
631	DECLARE_LIBBPF_OPTS(bpf_tc_hook, tc_hook,
632	.attach_point = BPF_TC_INGRESS);
633	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);	782	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
634	struct test_tunnel_kern *skel = NULL;	783	struct test_tunnel_kern *skel = NULL;
635	struct nstoken *nstoken;
636	int xdp_prog_fd;	784	int xdp_prog_fd;
637	int tc_prog_fd;	785	int tc_prog_fd;
638	int ifindex;	786	int ifindex;
@@ -646,19 +794,16 @@ static void test_xfrm_tunnel(void) · 7 unmodified lines
646	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))	794	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
647	goto done;	795	goto done;
648		796
649	ifindex = if_nametoindex("veth1");
650	if (!ASSERT_NEQ(ifindex, 0, "veth1 ifindex"))
651	goto done;
652		797
653	/* attach tc prog to tunnel dev */	798	/* attach tc prog to tunnel dev */
654	tc_hook.ifindex = ifindex;
655	tc_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state);	799	tc_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state);
656	if (!ASSERT_GE(tc_prog_fd, 0, "bpf_program__fd"))	800	if (generic_attach_igr("veth1", tc_prog_fd))
657	goto done;
658	if (attach_tc_prog(&tc_hook, tc_prog_fd, -1))
659	goto done;	801	goto done;
660		802
661	/* attach xdp prog to tunnel dev */	803	/* attach xdp prog to tunnel dev */
		804	ifindex = if_nametoindex("veth1");
		805	if (!ASSERT_NEQ(ifindex, 0, "veth1 ifindex"))
		806	goto done;
662	xdp_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state_xdp);	807	xdp_prog_fd = bpf_program__fd(skel->progs.xfrm_get_state_xdp);
663	if (!ASSERT_GE(xdp_prog_fd, 0, "bpf_program__fd"))	808	if (!ASSERT_GE(xdp_prog_fd, 0, "bpf_program__fd"))
664	goto done;	809	goto done;
@@ -666,14 +811,7 @@ static void test_xfrm_tunnel(void) · 1 unmodified line
666	if (!ASSERT_OK(err, "bpf_xdp_attach"))	811	if (!ASSERT_OK(err, "bpf_xdp_attach"))
667	goto done;	812	goto done;
668		813
669	/* ping from at_ns0 namespace test */	814	ping_dev1();
670	nstoken = open_netns("at_ns0");
671	if (!ASSERT_OK_PTR(nstoken, "setns"))
672	goto done;
673	err = test_ping(AF_INET, IP4_ADDR_TUNL_DEV1);
674	close_netns(nstoken);
675	if (!ASSERT_OK(err, "test_ping"))
676	goto done;
677		815
678	if (!ASSERT_EQ(skel->bss->xfrm_reqid, 1, "req_id"))	816	if (!ASSERT_EQ(skel->bss->xfrm_reqid, 1, "req_id"))
679	goto done;	817	goto done;
@@ -690,6 +828,281 @@ done: · 10 unmodified lines
690	test_tunnel_kern__destroy(skel);	828	test_tunnel_kern__destroy(skel);
691	}	829	}
692		830
		831	enum gre_test {
		832	GRE,
		833	GRE_NOKEY,
		834	GRETAP,
		835	GRETAP_NOKEY,
		836	};
		837
		838	static void test_gre_tunnel(enum gre_test test)
		839	{
		840	struct test_tunnel_kern *skel;
		841	int set_fd, get_fd;
		842	int err;
		843
		844	skel = test_tunnel_kern__open_and_load();
		845	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
		846	return;
		847
		848	switch (test) {
		849	case GRE:
		850	err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gre", "seq");
		851	set_fd = bpf_program__fd(skel->progs.gre_set_tunnel_no_key);
		852	get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
		853	break;
		854	case GRE_NOKEY:
		855	err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gre", "seq key 2");
		856	set_fd = bpf_program__fd(skel->progs.gre_set_tunnel);
		857	get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
		858	break;
		859	case GRETAP:
		860	err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gretap", "seq");
		861	set_fd = bpf_program__fd(skel->progs.gre_set_tunnel_no_key);
		862	get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
		863	break;
		864	case GRETAP_NOKEY:
		865	err = add_ipv4_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1, "gretap", "seq key 2");
		866	set_fd = bpf_program__fd(skel->progs.gre_set_tunnel);
		867	get_fd = bpf_program__fd(skel->progs.gre_get_tunnel);
		868	break;
		869	}
		870	if (!ASSERT_OK(err, "add tunnel"))
		871	goto done;
		872
		873	if (generic_attach(GRE_TUNL_DEV1, get_fd, set_fd))
		874	goto done;
		875
		876	ping_dev0();
		877	ping_dev1();
		878
		879	done:
		880	delete_tunnel(GRE_TUNL_DEV0, GRE_TUNL_DEV1);
		881	test_tunnel_kern__destroy(skel);
		882	}
		883
		884	enum ip6gre_test {
		885	IP6GRE,
		886	IP6GRETAP
		887	};
		888
		889	static void test_ip6gre_tunnel(enum ip6gre_test test)
		890	{
		891	struct test_tunnel_kern *skel;
		892	int set_fd, get_fd;
		893	int err;
		894
		895	skel = test_tunnel_kern__open_and_load();
		896	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
		897	return;
		898
		899	switch (test) {
		900	case IP6GRE:
		901	err = add_ipv6_tunnel(IP6GRE_TUNL_DEV0, IP6GRE_TUNL_DEV1,
		902	"ip6gre", "flowlabel 0xbcdef key 2");
		903	break;
		904	case IP6GRETAP:
		905	err = add_ipv6_tunnel(IP6GRE_TUNL_DEV0, IP6GRE_TUNL_DEV1,
		906	"ip6gretap", "flowlabel 0xbcdef key 2");
		907	break;
		908	}
		909	if (!ASSERT_OK(err, "add tunnel"))
		910	goto done;
		911
		912	set_fd = bpf_program__fd(skel->progs.ip6gretap_set_tunnel);
		913	get_fd = bpf_program__fd(skel->progs.ip6gretap_get_tunnel);
		914	if (generic_attach(IP6GRE_TUNL_DEV1, get_fd, set_fd))
		915	goto done;
		916
		917	ping6_veth0();
		918	ping6_dev1();
		919	ping_dev0();
		920	ping_dev1();
		921	done:
		922	delete_tunnel(IP6GRE_TUNL_DEV0, IP6GRE_TUNL_DEV1);
		923	test_tunnel_kern__destroy(skel);
		924	}
		925
		926	enum erspan_test {
		927	V1,
		928	V2
		929	};
		930
		931	static void test_erspan_tunnel(enum erspan_test test)
		932	{
		933	struct test_tunnel_kern *skel;
		934	int set_fd, get_fd;
		935	int err;
		936
		937	skel = test_tunnel_kern__open_and_load();
		938	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
		939	return;
		940
		941	switch (test) {
		942	case V1:
		943	err = add_ipv4_tunnel(ERSPAN_TUNL_DEV0, ERSPAN_TUNL_DEV1,
		944	"erspan", "seq key 2 erspan_ver 1 erspan 123");
		945	break;
		946	case V2:
		947	err = add_ipv4_tunnel(ERSPAN_TUNL_DEV0, ERSPAN_TUNL_DEV1,
		948	"erspan",
		949	"seq key 2 erspan_ver 2 erspan_dir egress erspan_hwid 3");
		950	break;
		951	}
		952	if (!ASSERT_OK(err, "add tunnel"))
		953	goto done;
		954
		955	set_fd = bpf_program__fd(skel->progs.erspan_set_tunnel);
		956	get_fd = bpf_program__fd(skel->progs.erspan_get_tunnel);
		957	if (generic_attach(ERSPAN_TUNL_DEV1, get_fd, set_fd))
		958	goto done;
		959
		960	ping_dev0();
		961	ping_dev1();
		962	done:
		963	delete_tunnel(ERSPAN_TUNL_DEV0, ERSPAN_TUNL_DEV1);
		964	test_tunnel_kern__destroy(skel);
		965	}
		966
		967	static void test_ip6erspan_tunnel(enum erspan_test test)
		968	{
		969	struct test_tunnel_kern *skel;
		970	int set_fd, get_fd;
		971	int err;
		972
		973	skel = test_tunnel_kern__open_and_load();
		974	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
		975	return;
		976
		977	switch (test) {
		978	case V1:
		979	err = add_ipv6_tunnel(IP6ERSPAN_TUNL_DEV0, IP6ERSPAN_TUNL_DEV1,
		980	"ip6erspan", "seq key 2 erspan_ver 1 erspan 123");
		981	break;
		982	case V2:
		983	err = add_ipv6_tunnel(IP6ERSPAN_TUNL_DEV0, IP6ERSPAN_TUNL_DEV1,
		984	"ip6erspan",
		985	"seq key 2 erspan_ver 2 erspan_dir egress erspan_hwid 7");
		986	break;
		987	}
		988	if (!ASSERT_OK(err, "add tunnel"))
		989	goto done;
		990
		991	set_fd = bpf_program__fd(skel->progs.ip4ip6erspan_set_tunnel);
		992	get_fd = bpf_program__fd(skel->progs.ip4ip6erspan_get_tunnel);
		993	if (generic_attach(IP6ERSPAN_TUNL_DEV1, get_fd, set_fd))
		994	goto done;
		995
		996	ping6_veth0();
		997	ping_dev1();
		998	done:
		999	delete_tunnel(IP6ERSPAN_TUNL_DEV0, IP6ERSPAN_TUNL_DEV1);
		1000	test_tunnel_kern__destroy(skel);
		1001	}
		1002
		1003	static void test_geneve_tunnel(void)
		1004	{
		1005	struct test_tunnel_kern *skel;
		1006	int set_fd, get_fd;
		1007	int err;
		1008
		1009	skel = test_tunnel_kern__open_and_load();
		1010	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
		1011	return;
		1012
		1013	err = add_geneve_tunnel(GENEVE_TUNL_DEV0, GENEVE_TUNL_DEV1,
		1014	"geneve", "dstport 6081");
		1015	if (!ASSERT_OK(err, "add tunnel"))
		1016	goto done;
		1017
		1018	set_fd = bpf_program__fd(skel->progs.geneve_set_tunnel);
		1019	get_fd = bpf_program__fd(skel->progs.geneve_get_tunnel);
		1020	if (generic_attach(GENEVE_TUNL_DEV1, get_fd, set_fd))
		1021	goto done;
		1022
		1023	ping_dev0();
		1024	ping_dev1();
		1025	done:
		1026	delete_tunnel(GENEVE_TUNL_DEV0, GENEVE_TUNL_DEV1);
		1027	test_tunnel_kern__destroy(skel);
		1028	}
		1029
		1030	static void test_ip6geneve_tunnel(void)
		1031	{
		1032	struct test_tunnel_kern *skel;
		1033	int set_fd, get_fd;
		1034	int err;
		1035
		1036	skel = test_tunnel_kern__open_and_load();
		1037	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
		1038	return;
		1039
		1040	err = add_ip6geneve_tunnel(IP6GENEVE_TUNL_DEV0, IP6GENEVE_TUNL_DEV1,
		1041	"geneve", "");
		1042	if (!ASSERT_OK(err, "add tunnel"))
		1043	goto done;
		1044
		1045	set_fd = bpf_program__fd(skel->progs.ip6geneve_set_tunnel);
		1046	get_fd = bpf_program__fd(skel->progs.ip6geneve_get_tunnel);
		1047	if (generic_attach(IP6GENEVE_TUNL_DEV1, get_fd, set_fd))
		1048	goto done;
		1049
		1050	ping_dev0();
		1051	ping_dev1();
		1052	done:
		1053	delete_tunnel(IP6GENEVE_TUNL_DEV0, IP6GENEVE_TUNL_DEV1);
		1054	test_tunnel_kern__destroy(skel);
		1055	}
		1056
		1057	enum ip6tnl_test {
		1058	IPIP6,
		1059	IP6IP6
		1060	};
		1061
		1062	static void test_ip6tnl_tunnel(enum ip6tnl_test test)
		1063	{
		1064	struct test_tunnel_kern *skel;
		1065	int set_fd, get_fd;
		1066	int err;
		1067
		1068	skel = test_tunnel_kern__open_and_load();
		1069	if (!ASSERT_OK_PTR(skel, "test_tunnel_kern__open_and_load"))
		1070	return;
		1071
		1072	err = add_ipv6_tunnel(IP6TNL_TUNL_DEV0, IP6TNL_TUNL_DEV1, "ip6tnl", "");
		1073	if (!ASSERT_OK(err, "add tunnel"))
		1074	goto done;
		1075
		1076	switch (test) {
		1077	case IPIP6:
		1078	set_fd = bpf_program__fd(skel->progs.ipip6_set_tunnel);
		1079	get_fd = bpf_program__fd(skel->progs.ipip6_get_tunnel);
		1080	break;
		1081	case IP6IP6:
		1082	set_fd = bpf_program__fd(skel->progs.ip6ip6_set_tunnel);
		1083	get_fd = bpf_program__fd(skel->progs.ip6ip6_get_tunnel);
		1084	break;
		1085	}
		1086	if (generic_attach(IP6TNL_TUNL_DEV1, get_fd, set_fd))
		1087	goto done;
		1088
		1089	ping6_veth0();
		1090	switch (test) {
		1091	case IPIP6:
		1092	ping_dev0();
		1093	ping_dev1();
		1094	break;
		1095	case IP6IP6:
		1096	ping6_dev0();
		1097	ping6_dev1();
		1098	break;
		1099	}
		1100
		1101	done:
		1102	delete_tunnel(IP6TNL_TUNL_DEV0, IP6TNL_TUNL_DEV1);
		1103	test_tunnel_kern__destroy(skel);
		1104	}
		1105
693	#define RUN_TEST(name, ...) \	1106	#define RUN_TEST(name, ...) \
694	({ \	1107	({ \
695	if (test__start_subtest(#name)) { \	1108	if (test__start_subtest(#name)) { \
@@ -707,6 +1120,20 @@ static void test_tunnel_run_tests(void arg) · 11 unmodified lines
707	RUN_TEST(ipip_tunnel, FOU);	1120	RUN_TEST(ipip_tunnel, FOU);
708	RUN_TEST(ipip_tunnel, GUE);	1121	RUN_TEST(ipip_tunnel, GUE);
709	RUN_TEST(xfrm_tunnel);	1122	RUN_TEST(xfrm_tunnel);
		1123	RUN_TEST(gre_tunnel, GRE);
		1124	RUN_TEST(gre_tunnel, GRE_NOKEY);
		1125	RUN_TEST(gre_tunnel, GRETAP);
		1126	RUN_TEST(gre_tunnel, GRETAP_NOKEY);
		1127	RUN_TEST(ip6gre_tunnel, IP6GRE);
		1128	RUN_TEST(ip6gre_tunnel, IP6GRETAP);
		1129	RUN_TEST(erspan_tunnel, V1);
		1130	RUN_TEST(erspan_tunnel, V2);
		1131	RUN_TEST(ip6erspan_tunnel, V1);
		1132	RUN_TEST(ip6erspan_tunnel, V2);
		1133	RUN_TEST(geneve_tunnel);
		1134	RUN_TEST(ip6geneve_tunnel);
		1135	RUN_TEST(ip6tnl_tunnel, IPIP6);
		1136	RUN_TEST(ip6tnl_tunnel, IP6IP6);
710		1137
711	return NULL;	1138	return NULL;
712	}	1139	}

tools/testing/selftests/bpf/prog_tests/test_veristat.c+139−0

@@ -0,0 +1,139 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#include <test_progs.h>
	4	#include <string.h>
	5	#include <stdio.h>
	6
	7	#define __CHECK_STR(str, name) \
	8	do { \
	9	if (!ASSERT_HAS_SUBSTR(fix->output, (str), (name))) \
	10	goto out; \
	11	} while (0)
	12
	13	struct fixture {
	14	char tmpfile[80];
	15	int fd;
	16	char *output;
	17	size_t sz;
	18	char veristat[80];
	19	};
	20
	21	static struct fixture *init_fixture(void)
	22	{
	23	struct fixture *fix = malloc(sizeof(struct fixture));
	24
	25	/* for no_alu32 and cpuv4 veristat is in parent folder */
	26	if (access("./veristat", F_OK) == 0)
	27	strcpy(fix->veristat, "./veristat");
	28	else if (access("../veristat", F_OK) == 0)
	29	strcpy(fix->veristat, "../veristat");
	30	else
	31	PRINT_FAIL("Can't find veristat binary");
	32
	33	snprintf(fix->tmpfile, sizeof(fix->tmpfile), "/tmp/test_veristat.XXXXXX");
	34	fix->fd = mkstemp(fix->tmpfile);
	35	fix->sz = 1000000;
	36	fix->output = malloc(fix->sz);
	37	return fix;
	38	}
	39
	40	static void teardown_fixture(struct fixture *fix)
	41	{
	42	free(fix->output);
	43	close(fix->fd);
	44	remove(fix->tmpfile);
	45	free(fix);
	46	}
	47
	48	static void test_set_global_vars_succeeds(void)
	49	{
	50	struct fixture *fix = init_fixture();
	51
	52	SYS(out,
	53	"%s set_global_vars.bpf.o"\
	54	" -G \"var_s64 = 0xf000000000000001\" "\
	55	" -G \"var_u64 = 0xfedcba9876543210\" "\
	56	" -G \"var_s32 = -0x80000000\" "\
	57	" -G \"var_u32 = 0x76543210\" "\
	58	" -G \"var_s16 = -32768\" "\
	59	" -G \"var_u16 = 60652\" "\
	60	" -G \"var_s8 = -128\" "\
	61	" -G \"var_u8 = 255\" "\
	62	" -G \"var_ea = EA2\" "\
	63	" -G \"var_eb = EB2\" "\
	64	" -G \"var_ec = EC2\" "\
	65	" -G \"var_b = 1\" "\
	66	"-vl2 > %s", fix->veristat, fix->tmpfile);
	67
	68	read(fix->fd, fix->output, fix->sz);
	69	__CHECK_STR("_w=0xf000000000000001 ", "var_s64 = 0xf000000000000001");
	70	__CHECK_STR("_w=0xfedcba9876543210 ", "var_u64 = 0xfedcba9876543210");
	71	__CHECK_STR("_w=0x80000000 ", "var_s32 = -0x80000000");
	72	__CHECK_STR("_w=0x76543210 ", "var_u32 = 0x76543210");
	73	__CHECK_STR("_w=0x8000 ", "var_s16 = -32768");
	74	__CHECK_STR("_w=0xecec ", "var_u16 = 60652");
	75	__CHECK_STR("_w=128 ", "var_s8 = -128");
	76	__CHECK_STR("_w=255 ", "var_u8 = 255");
	77	__CHECK_STR("_w=11 ", "var_ea = EA2");
	78	__CHECK_STR("_w=12 ", "var_eb = EB2");
	79	__CHECK_STR("_w=13 ", "var_ec = EC2");
	80	__CHECK_STR("_w=1 ", "var_b = 1");
	81
	82	out:
	83	teardown_fixture(fix);
	84	}
	85
	86	static void test_set_global_vars_from_file_succeeds(void)
	87	{
	88	struct fixture *fix = init_fixture();
	89	char input_file[80];
	90	const char *vars = "var_s16 = -32768\nvar_u16 = 60652";
	91	int fd;
	92
	93	snprintf(input_file, sizeof(input_file), "/tmp/veristat_input.XXXXXX");
	94	fd = mkstemp(input_file);
	95	if (!ASSERT_GE(fd, 0, "valid fd"))
	96	goto out;
	97
	98	write(fd, vars, strlen(vars));
	99	syncfs(fd);
	100	SYS(out, "%s set_global_vars.bpf.o -G \"@%s\" -vl2 > %s",
	101	fix->veristat, input_file, fix->tmpfile);
	102	read(fix->fd, fix->output, fix->sz);
	103	__CHECK_STR("_w=0x8000 ", "var_s16 = -32768");
	104	__CHECK_STR("_w=0xecec ", "var_u16 = 60652");
	105
	106	out:
	107	close(fd);
	108	remove(input_file);
	109	teardown_fixture(fix);
	110	}
	111
	112	static void test_set_global_vars_out_of_range(void)
	113	{
	114	struct fixture *fix = init_fixture();
	115
	116	SYS_FAIL(out,
	117	"%s set_global_vars.bpf.o -G \"var_s32 = 2147483648\" -vl2 2> %s",
	118	fix->veristat, fix->tmpfile);
	119
	120	read(fix->fd, fix->output, fix->sz);
	121	__CHECK_STR("is out of range [-2147483648; 2147483647]", "out of range");
	122
	123	out:
	124	teardown_fixture(fix);
	125	}
	126
	127	void test_veristat(void)
	128	{
	129	if (test__start_subtest("set_global_vars_succeeds"))
	130	test_set_global_vars_succeeds();
	131
	132	if (test__start_subtest("set_global_vars_out_of_range"))
	133	test_set_global_vars_out_of_range();
	134
	135	if (test__start_subtest("set_global_vars_from_file_succeeds"))
	136	test_set_global_vars_from_file_succeeds();
	137	}
	138
	139	#undef __CHECK_STR

tools/testing/selftests/bpf/prog_tests/test_xdp_veth.c+512−126

@@ -3,17 +3,50 @@ · 2 unmodified lines
3	/* Create 3 namespaces with 3 veth peers, and forward packets in-between using	3	/* Create 3 namespaces with 3 veth peers, and forward packets in-between using
4	* native XDP	4	* native XDP
5	*	5	*
6	* XDP_TX	6	* Network topology:
7	* NS1(veth11) NS2(veth22) NS3(veth33)	7	* ---------- ---------- ----------
8	* \| \| \|	8	* \| NS1 \| \| NS2 \| \| NS3 \|
9	* \| \| \|	9	* \| veth11 \| \| veth22 \| \| veth33 \|
10	* (veth1, (veth2, (veth3,	10	* ----\|----- -----\|---- -----\|----
11	* id:111) id:122) id:133)	11	* \| \| \|
12	* ^ \| ^ \| ^ \|	12	* ----\|------------------\|----------------\|----
13	* \| \| XDP_REDIRECT \| \| XDP_REDIRECT \| \|	13	* \| veth1 veth2 veth3 \|
14	* \| ------------------ ------------------ \|	14	* \| \|
15	* -----------------------------------------	15	* \| NSO \|
16	* XDP_REDIRECT	16	* ---------------------------------------------
		17	*
		18	* Test cases:
		19	* - [test_xdp_veth_redirect] : ping veth33 from veth11
		20	*
		21	* veth11 veth22 veth33
		22	* (XDP_PASS) (XDP_TX) (XDP_PASS)
		23	* \| \| \|
		24	* \| \| \|
		25	* veth1 veth2 veth3
		26	* (XDP_REDIRECT) (XDP_REDIRECT) (XDP_REDIRECT)
		27	* ^ \| ^ \| ^ \|
		28	* \| \| \| \| \| \|
		29	* \| ------------------ ------------------ \|
		30	* -----------------------------------------
		31	*
		32	* - [test_xdp_veth_broadcast_redirect]: broadcast from veth11
		33	* - IPv4 ping : BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS
		34	* -> echo request received by all except veth11
		35	* - IPv4 ping : BPF_F_BROADCAST
		36	* -> echo request received by all veth
		37	* - [test_xdp_veth_egress]:
		38	* - all src mac should be the magic mac
		39	*
		40	* veth11 veth22 veth33
		41	* (XDP_PASS) (XDP_PASS) (XDP_PASS)
		42	* \| \| \|
		43	* \| \| \|
		44	* veth1 veth2 veth3
		45	* (XDP_REDIRECT) (XDP_REDIRECT) (XDP_REDIRECT)
		46	* \| ^ ^
		47	* \| \| \|
		48	* ----------------------------------------
		49	*
17	*/	50	*/
18		51
19	#define _GNU_SOURCE	52	#define _GNU_SOURCE
@@ -22,192 +55,545 @@ · 2 unmodified lines
22	#include "network_helpers.h"	55	#include "network_helpers.h"
23	#include "xdp_dummy.skel.h"	56	#include "xdp_dummy.skel.h"
24	#include "xdp_redirect_map.skel.h"	57	#include "xdp_redirect_map.skel.h"
		58	#include "xdp_redirect_multi_kern.skel.h"
25	#include "xdp_tx.skel.h"	59	#include "xdp_tx.skel.h"
		60	#include <uapi/linux/if_link.h>
26		61
27	#define VETH_PAIRS_COUNT 3	62	#define VETH_PAIRS_COUNT 3
28	#define NS_SUFFIX_LEN 6	63	#define VETH_NAME_MAX_LEN 32
29	#define VETH_NAME_MAX_LEN 16	64	#define IP_MAX_LEN 16
30	#define IP_SRC "10.1.1.11"	65	#define IP_SRC "10.1.1.11"
31	#define IP_DST "10.1.1.33"	66	#define IP_DST "10.1.1.33"
32	#define IP_CMD_MAX_LEN 128	67	#define IP_NEIGH "10.1.1.253"
33		68	#define PROG_NAME_MAX_LEN 128
34	struct skeletons {	69	#define NS_NAME_MAX_LEN 32
35	struct xdp_dummy *xdp_dummy;
36	struct xdp_tx *xdp_tx;
37	struct xdp_redirect_map *xdp_redirect_maps;
38	};
39		70
40	struct veth_configuration {	71	struct veth_configuration {
41	char local_veth[VETH_NAME_MAX_LEN]; /* Interface in main namespace */	72	char local_veth[VETH_NAME_MAX_LEN]; /* Interface in main namespace */
42	char remote_veth[VETH_NAME_MAX_LEN]; /* Peer interface in dedicated namespace*/	73	char remote_veth[VETH_NAME_MAX_LEN]; /* Peer interface in dedicated namespace*/
43	const char namespace; / Namespace for the remote veth */	74	char namespace[NS_NAME_MAX_LEN]; /* Namespace for the remote veth */
44	char next_veth[VETH_NAME_MAX_LEN]; /* Local interface to redirect traffic to */	75	int next_veth; /* Local interface to redirect traffic to */
45	char remote_addr; / IP address of the remote veth */	76	char remote_addr[IP_MAX_LEN]; /* IP address of the remote veth */
46	};	77	};
47		78
48	static struct veth_configuration config[VETH_PAIRS_COUNT] = {	79	struct net_configuration {
49	{	80	char ns0_name[NS_NAME_MAX_LEN];
50	.local_veth = "veth1",	81	struct veth_configuration veth_cfg[VETH_PAIRS_COUNT];
51	.remote_veth = "veth11",	82	};
52	.next_veth = "veth2",	83
53	.remote_addr = IP_SRC,	84	static const struct net_configuration default_config = {
54	.namespace = "ns-veth11"	85	.ns0_name = "ns0-",
55	},
56	{
57	.local_veth = "veth2",
58	.remote_veth = "veth22",
59	.next_veth = "veth3",
60	.remote_addr = NULL,
61	.namespace = "ns-veth22"
62	},
63	{	86	{
64	.local_veth = "veth3",	87	{
65	.remote_veth = "veth33",	88	.local_veth = "veth1-",
66	.next_veth = "veth1",	89	.remote_veth = "veth11",
67	.remote_addr = IP_DST,	90	.next_veth = 1,
68	.namespace = "ns-veth33"	91	.remote_addr = IP_SRC,
		92	.namespace = "ns-veth11-"
		93	},
		94	{
		95	.local_veth = "veth2-",
		96	.remote_veth = "veth22",
		97	.next_veth = 2,
		98	.remote_addr = "",
		99	.namespace = "ns-veth22-"
		100	},
		101	{
		102	.local_veth = "veth3-",
		103	.remote_veth = "veth33",
		104	.next_veth = 0,
		105	.remote_addr = IP_DST,
		106	.namespace = "ns-veth33-"
		107	}
69	}	108	}
70	};	109	};
71		110
72	static int attach_programs_to_veth_pair(struct skeletons *skeletons, int index)	111	struct prog_configuration {
		112	char local_name[PROG_NAME_MAX_LEN]; /* BPF prog to attach to local_veth */
		113	char remote_name[PROG_NAME_MAX_LEN]; /* BPF prog to attach to remote_veth */
		114	u32 local_flags; /* XDP flags to use on local_veth */
		115	u32 remote_flags; /* XDP flags to use on remote_veth */
		116	};
		117
		118	static int attach_programs_to_veth_pair(struct bpf_object **objs, size_t nb_obj,
		119	struct net_configuration *net_config,
		120	struct prog_configuration *prog, int index)
73	{	121	{
74	struct bpf_program local_prog, remote_prog;	122	struct bpf_program local_prog, remote_prog;
75	struct bpf_link local_link, remote_link;
76	struct nstoken *nstoken;	123	struct nstoken *nstoken;
77	struct bpf_link *link;	124	int interface, ret, i;
78	int interface;	125
79		126	for (i = 0; i < nb_obj; i++) {
80	switch (index) {	127	local_prog = bpf_object__find_program_by_name(objs[i], prog[index].local_name);
81	case 0:	128	if (local_prog)
82	local_prog = skeletons->xdp_redirect_maps->progs.xdp_redirect_map_0;	129	break;
83	local_link = &skeletons->xdp_redirect_maps->links.xdp_redirect_map_0;
84	remote_prog = skeletons->xdp_dummy->progs.xdp_dummy_prog;
85	remote_link = &skeletons->xdp_dummy->links.xdp_dummy_prog;
86	break;
87	case 1:
88	local_prog = skeletons->xdp_redirect_maps->progs.xdp_redirect_map_1;
89	local_link = &skeletons->xdp_redirect_maps->links.xdp_redirect_map_1;
90	remote_prog = skeletons->xdp_tx->progs.xdp_tx;
91	remote_link = &skeletons->xdp_tx->links.xdp_tx;
92	break;
93	case 2:
94	local_prog = skeletons->xdp_redirect_maps->progs.xdp_redirect_map_2;
95	local_link = &skeletons->xdp_redirect_maps->links.xdp_redirect_map_2;
96	remote_prog = skeletons->xdp_dummy->progs.xdp_dummy_prog;
97	remote_link = &skeletons->xdp_dummy->links.xdp_dummy_prog;
98	break;
99	}	130	}
100	interface = if_nametoindex(config[index].local_veth);	131	if (!ASSERT_OK_PTR(local_prog, "find local program"))
		132	return -1;
		133
		134	for (i = 0; i < nb_obj; i++) {
		135	remote_prog = bpf_object__find_program_by_name(objs[i], prog[index].remote_name);
		136	if (remote_prog)
		137	break;
		138	}
		139	if (!ASSERT_OK_PTR(remote_prog, "find remote program"))
		140	return -1;
		141
		142	interface = if_nametoindex(net_config->veth_cfg[index].local_veth);
101	if (!ASSERT_NEQ(interface, 0, "non zero interface index"))	143	if (!ASSERT_NEQ(interface, 0, "non zero interface index"))
102	return -1;	144	return -1;
103	link = bpf_program__attach_xdp(local_prog, interface);	145
104	if (!ASSERT_OK_PTR(link, "attach xdp program to local veth"))	146	ret = bpf_xdp_attach(interface, bpf_program__fd(local_prog),
		147	prog[index].local_flags, NULL);
		148	if (!ASSERT_OK(ret, "attach xdp program to local veth"))
105	return -1;	149	return -1;
106	*local_link = link;	150
107	nstoken = open_netns(config[index].namespace);	151	nstoken = open_netns(net_config->veth_cfg[index].namespace);
108	if (!ASSERT_OK_PTR(nstoken, "switch to remote veth namespace"))	152	if (!ASSERT_OK_PTR(nstoken, "switch to remote veth namespace"))
109	return -1;	153	return -1;
110	interface = if_nametoindex(config[index].remote_veth);	154
		155	interface = if_nametoindex(net_config->veth_cfg[index].remote_veth);
111	if (!ASSERT_NEQ(interface, 0, "non zero interface index")) {	156	if (!ASSERT_NEQ(interface, 0, "non zero interface index")) {
112	close_netns(nstoken);	157	close_netns(nstoken);
113	return -1;	158	return -1;
114	}	159	}
115	link = bpf_program__attach_xdp(remote_prog, interface);	160
116	*remote_link = link;	161	ret = bpf_xdp_attach(interface, bpf_program__fd(remote_prog),
117	close_netns(nstoken);	162	prog[index].remote_flags, NULL);
118	if (!ASSERT_OK_PTR(link, "attach xdp program to remote veth"))	163	if (!ASSERT_OK(ret, "attach xdp program to remote veth")) {
		164	close_netns(nstoken);
119	return -1;	165	return -1;
		166	}
120		167
		168	close_netns(nstoken);
121	return 0;	169	return 0;
122	}	170	}
123		171
124	static int configure_network(struct skeletons *skeletons)	172	static int create_network(struct net_configuration *net_config)
125	{	173	{
126	int interface_id;	174	struct nstoken *nstoken = NULL;
127	int map_fd;	175	int i, err;
128	int err;	176
129	int i = 0;	177	memcpy(net_config, &default_config, sizeof(struct net_configuration));
		178
		179	/* Create unique namespaces */
		180	err = append_tid(net_config->ns0_name, NS_NAME_MAX_LEN);
		181	if (!ASSERT_OK(err, "append TID to ns0 name"))
		182	goto fail;
		183	SYS(fail, "ip netns add %s", net_config->ns0_name);
130		184
131	/* First create and configure all interfaces */
132	for (i = 0; i < VETH_PAIRS_COUNT; i++) {	185	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
133	SYS(fail, "ip netns add %s", config[i].namespace);	186	err = append_tid(net_config->veth_cfg[i].namespace, NS_NAME_MAX_LEN);
134	SYS(fail, "ip link add %s type veth peer name %s netns %s",	187	if (!ASSERT_OK(err, "append TID to ns name"))
135	config[i].local_veth, config[i].remote_veth, config[i].namespace);	188	goto fail;
136	SYS(fail, "ip link set dev %s up", config[i].local_veth);	189	SYS(fail, "ip netns add %s", net_config->veth_cfg[i].namespace);
137	if (config[i].remote_addr)
138	SYS(fail, "ip -n %s addr add %s/24 dev %s", config[i].namespace,
139	config[i].remote_addr, config[i].remote_veth);
140	SYS(fail, "ip -n %s link set dev %s up", config[i].namespace,
141	config[i].remote_veth);
142	}	190	}
143		191
144	/* Then configure the redirect map and attach programs to interfaces */	192	/* Create interfaces */
145	map_fd = bpf_map__fd(skeletons->xdp_redirect_maps->maps.tx_port);	193	nstoken = open_netns(net_config->ns0_name);
146	if (!ASSERT_GE(map_fd, 0, "open redirect map"))	194	if (!nstoken)
147	goto fail;	195	goto fail;
		196
148	for (i = 0; i < VETH_PAIRS_COUNT; i++) {	197	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
149	interface_id = if_nametoindex(config[i].next_veth);	198	SYS(fail, "ip link add %s type veth peer name %s netns %s",
150	if (!ASSERT_NEQ(interface_id, 0, "non zero interface index"))	199	net_config->veth_cfg[i].local_veth, net_config->veth_cfg[i].remote_veth,
151	goto fail;	200	net_config->veth_cfg[i].namespace);
152	err = bpf_map_update_elem(map_fd, &i, &interface_id, BPF_ANY);	201	SYS(fail, "ip link set dev %s up", net_config->veth_cfg[i].local_veth);
153	if (!ASSERT_OK(err, "configure interface redirection through map"))	202	if (net_config->veth_cfg[i].remote_addr[0])
154	goto fail;	203	SYS(fail, "ip -n %s addr add %s/24 dev %s",
155	if (attach_programs_to_veth_pair(skeletons, i))	204	net_config->veth_cfg[i].namespace,
156	goto fail;	205	net_config->veth_cfg[i].remote_addr,
		206	net_config->veth_cfg[i].remote_veth);
		207	SYS(fail, "ip -n %s link set dev %s up", net_config->veth_cfg[i].namespace,
		208	net_config->veth_cfg[i].remote_veth);
157	}	209	}
158		210
		211	close_netns(nstoken);
159	return 0;	212	return 0;
160		213
161	fail:	214	fail:
		215	close_netns(nstoken);
162	return -1;	216	return -1;
163	}	217	}
164		218
165	static void cleanup_network(void)	219	static void cleanup_network(struct net_configuration *net_config)
166	{	220	{
167	int i;	221	int i;
168		222
169	/* Deleting namespaces is enough to automatically remove veth pairs as well	223	SYS_NOFAIL("ip netns del %s", net_config->ns0_name);
170	*/
171	for (i = 0; i < VETH_PAIRS_COUNT; i++)	224	for (i = 0; i < VETH_PAIRS_COUNT; i++)
172	SYS_NOFAIL("ip netns del %s", config[i].namespace);	225	SYS_NOFAIL("ip netns del %s", net_config->veth_cfg[i].namespace);
173	}	226	}
174		227
175	static int check_ping(struct skeletons *skeletons)	228	#define VETH_REDIRECT_SKEL_NB 3
		229	static void xdp_veth_redirect(u32 flags)
176	{	230	{
		231	struct prog_configuration ping_config[VETH_PAIRS_COUNT] = {
		232	{
		233	.local_name = "xdp_redirect_map_0",
		234	.remote_name = "xdp_dummy_prog",
		235	.local_flags = flags,
		236	.remote_flags = flags,
		237	},
		238	{
		239	.local_name = "xdp_redirect_map_1",
		240	.remote_name = "xdp_tx",
		241	.local_flags = flags,
		242	.remote_flags = flags,
		243	},
		244	{
		245	.local_name = "xdp_redirect_map_2",
		246	.remote_name = "xdp_dummy_prog",
		247	.local_flags = flags,
		248	.remote_flags = flags,
		249	}
		250	};
		251	struct bpf_object *bpf_objs[VETH_REDIRECT_SKEL_NB];
		252	struct xdp_redirect_map *xdp_redirect_map;
		253	struct net_configuration net_config;
		254	struct nstoken *nstoken = NULL;
		255	struct xdp_dummy *xdp_dummy;
		256	struct xdp_tx *xdp_tx;
		257	int map_fd;
		258	int i;
		259
		260	xdp_dummy = xdp_dummy__open_and_load();
		261	if (!ASSERT_OK_PTR(xdp_dummy, "xdp_dummy__open_and_load"))
		262	return;
		263
		264	xdp_tx = xdp_tx__open_and_load();
		265	if (!ASSERT_OK_PTR(xdp_tx, "xdp_tx__open_and_load"))
		266	goto destroy_xdp_dummy;
		267
		268	xdp_redirect_map = xdp_redirect_map__open_and_load();
		269	if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load"))
		270	goto destroy_xdp_tx;
		271
		272	if (!ASSERT_OK(create_network(&net_config), "create network"))
		273	goto destroy_xdp_redirect_map;
		274
		275	/* Then configure the redirect map and attach programs to interfaces */
		276	map_fd = bpf_map__fd(xdp_redirect_map->maps.tx_port);
		277	if (!ASSERT_OK_FD(map_fd, "open redirect map"))
		278	goto destroy_xdp_redirect_map;
		279
		280	bpf_objs[0] = xdp_dummy->obj;
		281	bpf_objs[1] = xdp_tx->obj;
		282	bpf_objs[2] = xdp_redirect_map->obj;
		283
		284	nstoken = open_netns(net_config.ns0_name);
		285	if (!ASSERT_OK_PTR(nstoken, "open NS0"))
		286	goto destroy_xdp_redirect_map;
		287
		288	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
		289	int next_veth = net_config.veth_cfg[i].next_veth;
		290	int interface_id;
		291	int err;
		292
		293	interface_id = if_nametoindex(net_config.veth_cfg[next_veth].local_veth);
		294	if (!ASSERT_NEQ(interface_id, 0, "non zero interface index"))
		295	goto destroy_xdp_redirect_map;
		296	err = bpf_map_update_elem(map_fd, &i, &interface_id, BPF_ANY);
		297	if (!ASSERT_OK(err, "configure interface redirection through map"))
		298	goto destroy_xdp_redirect_map;
		299	if (attach_programs_to_veth_pair(bpf_objs, VETH_REDIRECT_SKEL_NB,
		300	&net_config, ping_config, i))
		301	goto destroy_xdp_redirect_map;
		302	}
		303
177	/* Test: if all interfaces are properly configured, we must be able to ping	304	/* Test: if all interfaces are properly configured, we must be able to ping
178	* veth33 from veth11	305	* veth33 from veth11
179	*/	306	*/
180	return SYS_NOFAIL("ip netns exec %s ping -c 1 -W 1 %s > /dev/null",	307	ASSERT_OK(SYS_NOFAIL("ip netns exec %s ping -c 1 -W 1 %s > /dev/null",
181	config[0].namespace, IP_DST);	308	net_config.veth_cfg[0].namespace, IP_DST), "ping");
		309
		310	destroy_xdp_redirect_map:
		311	close_netns(nstoken);
		312	xdp_redirect_map__destroy(xdp_redirect_map);
		313	destroy_xdp_tx:
		314	xdp_tx__destroy(xdp_tx);
		315	destroy_xdp_dummy:
		316	xdp_dummy__destroy(xdp_dummy);
		317
		318	cleanup_network(&net_config);
182	}	319	}
183		320
184	void test_xdp_veth_redirect(void)	321	#define BROADCAST_REDIRECT_SKEL_NB 2
		322	static void xdp_veth_broadcast_redirect(u32 attach_flags, u64 redirect_flags)
185	{	323	{
186	struct skeletons skeletons = {};	324	struct prog_configuration prog_cfg[VETH_PAIRS_COUNT] = {
		325	{
		326	.local_name = "xdp_redirect_map_multi_prog",
		327	.remote_name = "xdp_count_0",
		328	.local_flags = attach_flags,
		329	.remote_flags = attach_flags,
		330	},
		331	{
		332	.local_name = "xdp_redirect_map_multi_prog",
		333	.remote_name = "xdp_count_1",
		334	.local_flags = attach_flags,
		335	.remote_flags = attach_flags,
		336	},
		337	{
		338	.local_name = "xdp_redirect_map_multi_prog",
		339	.remote_name = "xdp_count_2",
		340	.local_flags = attach_flags,
		341	.remote_flags = attach_flags,
		342	}
		343	};
		344	struct bpf_object *bpf_objs[BROADCAST_REDIRECT_SKEL_NB];
		345	struct xdp_redirect_multi_kern *xdp_redirect_multi_kern;
		346	struct xdp_redirect_map *xdp_redirect_map;
		347	struct bpf_devmap_val devmap_val = {};
		348	struct net_configuration net_config;
		349	struct nstoken *nstoken = NULL;
		350	u16 protocol = ETH_P_IP;
		351	int group_map;
		352	int flags_map;
		353	int cnt_map;
		354	u64 cnt = 0;
		355	int i, err;
187		356
188	skeletons.xdp_dummy = xdp_dummy__open_and_load();	357	xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load();
189	if (!ASSERT_OK_PTR(skeletons.xdp_dummy, "xdp_dummy__open_and_load"))	358	if (!ASSERT_OK_PTR(xdp_redirect_multi_kern, "xdp_redirect_multi_kern__open_and_load"))
190	return;	359	return;
191		360
192	skeletons.xdp_tx = xdp_tx__open_and_load();	361	xdp_redirect_map = xdp_redirect_map__open_and_load();
193	if (!ASSERT_OK_PTR(skeletons.xdp_tx, "xdp_tx__open_and_load"))	362	if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load"))
		363	goto destroy_xdp_redirect_multi_kern;
		364
		365	if (!ASSERT_OK(create_network(&net_config), "create network"))
		366	goto destroy_xdp_redirect_map;
		367
		368	group_map = bpf_map__fd(xdp_redirect_multi_kern->maps.map_all);
		369	if (!ASSERT_OK_FD(group_map, "open map_all"))
		370	goto destroy_xdp_redirect_map;
		371
		372	flags_map = bpf_map__fd(xdp_redirect_multi_kern->maps.redirect_flags);
		373	if (!ASSERT_OK_FD(group_map, "open map_all"))
		374	goto destroy_xdp_redirect_map;
		375
		376	err = bpf_map_update_elem(flags_map, &protocol, &redirect_flags, BPF_NOEXIST);
		377	if (!ASSERT_OK(err, "init IP count"))
		378	goto destroy_xdp_redirect_map;
		379
		380	cnt_map = bpf_map__fd(xdp_redirect_map->maps.rxcnt);
		381	if (!ASSERT_OK_FD(cnt_map, "open rxcnt map"))
		382	goto destroy_xdp_redirect_map;
		383
		384	bpf_objs[0] = xdp_redirect_multi_kern->obj;
		385	bpf_objs[1] = xdp_redirect_map->obj;
		386
		387	nstoken = open_netns(net_config.ns0_name);
		388	if (!ASSERT_OK_PTR(nstoken, "open NS0"))
		389	goto destroy_xdp_redirect_map;
		390
		391	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
		392	int ifindex = if_nametoindex(net_config.veth_cfg[i].local_veth);
		393
		394	if (attach_programs_to_veth_pair(bpf_objs, BROADCAST_REDIRECT_SKEL_NB,
		395	&net_config, prog_cfg, i))
		396	goto destroy_xdp_redirect_map;
		397
		398	SYS(destroy_xdp_redirect_map,
		399	"ip -n %s neigh add %s lladdr 00:00:00:00:00:01 dev %s",
		400	net_config.veth_cfg[i].namespace, IP_NEIGH, net_config.veth_cfg[i].remote_veth);
		401
		402	devmap_val.ifindex = ifindex;
		403	err = bpf_map_update_elem(group_map, &ifindex, &devmap_val, 0);
		404	if (!ASSERT_OK(err, "bpf_map_update_elem"))
		405	goto destroy_xdp_redirect_map;
		406
		407	}
		408
		409	SYS_NOFAIL("ip netns exec %s ping %s -i 0.1 -c 4 -W1 > /dev/null ",
		410	net_config.veth_cfg[0].namespace, IP_NEIGH);
		411
		412	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
		413	err = bpf_map_lookup_elem(cnt_map, &i, &cnt);
		414	if (!ASSERT_OK(err, "get IP cnt"))
		415	goto destroy_xdp_redirect_map;
		416
		417	if (redirect_flags & BPF_F_EXCLUDE_INGRESS)
		418	/* veth11 shouldn't receive the ICMP requests;
		419	* others should
		420	*/
		421	ASSERT_EQ(cnt, i ? 4 : 0, "compare IP cnt");
		422	else
		423	/* All remote veth should receive the ICMP requests */
		424	ASSERT_EQ(cnt, 4, "compare IP cnt");
		425	}
		426
		427	destroy_xdp_redirect_map:
		428	close_netns(nstoken);
		429	xdp_redirect_map__destroy(xdp_redirect_map);
		430	destroy_xdp_redirect_multi_kern:
		431	xdp_redirect_multi_kern__destroy(xdp_redirect_multi_kern);
		432
		433	cleanup_network(&net_config);
		434	}
		435
		436	#define VETH_EGRESS_SKEL_NB 3
		437	static void xdp_veth_egress(u32 flags)
		438	{
		439	struct prog_configuration prog_cfg[VETH_PAIRS_COUNT] = {
		440	{
		441	.local_name = "xdp_redirect_map_all_prog",
		442	.remote_name = "xdp_dummy_prog",
		443	.local_flags = flags,
		444	.remote_flags = flags,
		445	},
		446	{
		447	.local_name = "xdp_redirect_map_all_prog",
		448	.remote_name = "store_mac_1",
		449	.local_flags = flags,
		450	.remote_flags = flags,
		451	},
		452	{
		453	.local_name = "xdp_redirect_map_all_prog",
		454	.remote_name = "store_mac_2",
		455	.local_flags = flags,
		456	.remote_flags = flags,
		457	}
		458	};
		459	const char magic_mac[6] = { 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF};
		460	struct xdp_redirect_multi_kern *xdp_redirect_multi_kern;
		461	struct bpf_object *bpf_objs[VETH_EGRESS_SKEL_NB];
		462	struct xdp_redirect_map *xdp_redirect_map;
		463	struct bpf_devmap_val devmap_val = {};
		464	struct net_configuration net_config;
		465	int mac_map, egress_map, res_map;
		466	struct nstoken *nstoken = NULL;
		467	struct xdp_dummy *xdp_dummy;
		468	int err;
		469	int i;
		470
		471	xdp_dummy = xdp_dummy__open_and_load();
		472	if (!ASSERT_OK_PTR(xdp_dummy, "xdp_dummy__open_and_load"))
		473	return;
		474
		475	xdp_redirect_multi_kern = xdp_redirect_multi_kern__open_and_load();
		476	if (!ASSERT_OK_PTR(xdp_redirect_multi_kern, "xdp_redirect_multi_kern__open_and_load"))
194	goto destroy_xdp_dummy;	477	goto destroy_xdp_dummy;
195		478
196	skeletons.xdp_redirect_maps = xdp_redirect_map__open_and_load();	479	xdp_redirect_map = xdp_redirect_map__open_and_load();
197	if (!ASSERT_OK_PTR(skeletons.xdp_redirect_maps, "xdp_redirect_map__open_and_load"))	480	if (!ASSERT_OK_PTR(xdp_redirect_map, "xdp_redirect_map__open_and_load"))
198	goto destroy_xdp_tx;	481	goto destroy_xdp_redirect_multi_kern;
199		482
200	if (configure_network(&skeletons))	483	if (!ASSERT_OK(create_network(&net_config), "create network"))
201	goto destroy_xdp_redirect_map;	484	goto destroy_xdp_redirect_map;
202		485
203	ASSERT_OK(check_ping(&skeletons), "ping");	486	mac_map = bpf_map__fd(xdp_redirect_multi_kern->maps.mac_map);
		487	if (!ASSERT_OK_FD(mac_map, "open mac_map"))
		488	goto destroy_xdp_redirect_map;
		489
		490	egress_map = bpf_map__fd(xdp_redirect_multi_kern->maps.map_egress);
		491	if (!ASSERT_OK_FD(egress_map, "open map_egress"))
		492	goto destroy_xdp_redirect_map;
		493
		494	devmap_val.bpf_prog.fd = bpf_program__fd(xdp_redirect_multi_kern->progs.xdp_devmap_prog);
		495
		496	bpf_objs[0] = xdp_dummy->obj;
		497	bpf_objs[1] = xdp_redirect_multi_kern->obj;
		498	bpf_objs[2] = xdp_redirect_map->obj;
		499
		500	nstoken = open_netns(net_config.ns0_name);
		501	if (!ASSERT_OK_PTR(nstoken, "open NS0"))
		502	goto destroy_xdp_redirect_map;
		503
		504	for (i = 0; i < VETH_PAIRS_COUNT; i++) {
		505	int ifindex = if_nametoindex(net_config.veth_cfg[i].local_veth);
		506
		507	SYS(destroy_xdp_redirect_map,
		508	"ip -n %s neigh add %s lladdr 00:00:00:00:00:01 dev %s",
		509	net_config.veth_cfg[i].namespace, IP_NEIGH, net_config.veth_cfg[i].remote_veth);
		510
		511	if (attach_programs_to_veth_pair(bpf_objs, VETH_REDIRECT_SKEL_NB,
		512	&net_config, prog_cfg, i))
		513	goto destroy_xdp_redirect_map;
		514
		515	err = bpf_map_update_elem(mac_map, &ifindex, magic_mac, 0);
		516	if (!ASSERT_OK(err, "bpf_map_update_elem"))
		517	goto destroy_xdp_redirect_map;
		518
		519	devmap_val.ifindex = ifindex;
		520	err = bpf_map_update_elem(egress_map, &ifindex, &devmap_val, 0);
		521	if (!ASSERT_OK(err, "bpf_map_update_elem"))
		522	goto destroy_xdp_redirect_map;
		523	}
		524
		525	SYS_NOFAIL("ip netns exec %s ping %s -i 0.1 -c 4 -W1 > /dev/null ",
		526	net_config.veth_cfg[0].namespace, IP_NEIGH);
		527
		528	res_map = bpf_map__fd(xdp_redirect_map->maps.rx_mac);
		529	if (!ASSERT_OK_FD(res_map, "open rx_map"))
		530	goto destroy_xdp_redirect_map;
		531
		532	for (i = 0; i < 2; i++) {
		533	u32 key = i;
		534	u64 res;
		535
		536	err = bpf_map_lookup_elem(res_map, &key, &res);
		537	if (!ASSERT_OK(err, "get MAC res"))
		538	goto destroy_xdp_redirect_map;
		539
		540	ASSERT_STRNEQ((const char *)&res, magic_mac, ETH_ALEN, "compare mac");
		541	}
204		542
205	destroy_xdp_redirect_map:	543	destroy_xdp_redirect_map:
206	xdp_redirect_map__destroy(skeletons.xdp_redirect_maps);	544	close_netns(nstoken);
207	destroy_xdp_tx:	545	xdp_redirect_map__destroy(xdp_redirect_map);
208	xdp_tx__destroy(skeletons.xdp_tx);	546	destroy_xdp_redirect_multi_kern:
		547	xdp_redirect_multi_kern__destroy(xdp_redirect_multi_kern);
209	destroy_xdp_dummy:	548	destroy_xdp_dummy:
210	xdp_dummy__destroy(skeletons.xdp_dummy);	549	xdp_dummy__destroy(xdp_dummy);
		550
		551	cleanup_network(&net_config);
		552	}
		553
		554	void test_xdp_veth_redirect(void)
		555	{
		556	if (test__start_subtest("0"))
		557	xdp_veth_redirect(0);
		558
		559	if (test__start_subtest("DRV_MODE"))
		560	xdp_veth_redirect(XDP_FLAGS_DRV_MODE);
		561
		562	if (test__start_subtest("SKB_MODE"))
		563	xdp_veth_redirect(XDP_FLAGS_SKB_MODE);
		564	}
		565
		566	void test_xdp_veth_broadcast_redirect(void)
		567	{
		568	if (test__start_subtest("0/BROADCAST"))
		569	xdp_veth_broadcast_redirect(0, BPF_F_BROADCAST);
		570
		571	if (test__start_subtest("0/(BROADCAST \| EXCLUDE_INGRESS)"))
		572	xdp_veth_broadcast_redirect(0, BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);
		573
		574	if (test__start_subtest("DRV_MODE/BROADCAST"))
		575	xdp_veth_broadcast_redirect(XDP_FLAGS_DRV_MODE, BPF_F_BROADCAST);
		576
		577	if (test__start_subtest("DRV_MODE/(BROADCAST \| EXCLUDE_INGRESS)"))
		578	xdp_veth_broadcast_redirect(XDP_FLAGS_DRV_MODE,
		579	BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);
		580
		581	if (test__start_subtest("SKB_MODE/BROADCAST"))
		582	xdp_veth_broadcast_redirect(XDP_FLAGS_SKB_MODE, BPF_F_BROADCAST);
		583
		584	if (test__start_subtest("SKB_MODE/(BROADCAST \| EXCLUDE_INGRESS)"))
		585	xdp_veth_broadcast_redirect(XDP_FLAGS_SKB_MODE,
		586	BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);
		587	}
		588
		589	void test_xdp_veth_egress(void)
		590	{
		591	if (test__start_subtest("0/egress"))
		592	xdp_veth_egress(0);
		593
		594	if (test__start_subtest("DRV_MODE/egress"))
		595	xdp_veth_egress(XDP_FLAGS_DRV_MODE);
211		596
212	cleanup_network();	597	if (test__start_subtest("SKB_MODE/egress"))
		598	xdp_veth_egress(XDP_FLAGS_SKB_MODE);
213	}	599	}

tools/testing/selftests/bpf/prog_tests/token.c+96−1

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	#include "priv_prog.skel.h"	19	#include "priv_prog.skel.h"
20	#include "dummy_st_ops_success.skel.h"	20	#include "dummy_st_ops_success.skel.h"
21	#include "token_lsm.skel.h"	21	#include "token_lsm.skel.h"
		22	#include "priv_freplace_prog.skel.h"
22		23
23	static inline int sys_mount(const char dev_name, const char dir_name,	24	static inline int sys_mount(const char dev_name, const char dir_name,
24	const char *type, unsigned long flags,	25	const char *type, unsigned long flags,
@@ -788,6 +789,84 @@ static int userns_obj_priv_prog(int mnt_fd, struct token_lsm *lsm_skel) · 763 unmodified lines
788	return 0;	789	return 0;
789	}	790	}
790		791
		792	static int userns_obj_priv_freplace_setup(int mnt_fd, struct priv_freplace_prog **fr_skel,
		793	struct priv_prog *skel, int tgt_fd)
		794	{
		795	LIBBPF_OPTS(bpf_object_open_opts, opts);
		796	int err;
		797	char buf[256];
		798
		799	/* use bpf_token_path to provide BPF FS path */
		800	snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd);
		801	opts.bpf_token_path = buf;
		802	*skel = priv_prog__open_opts(&opts);
		803	if (!ASSERT_OK_PTR(*skel, "priv_prog__open_opts"))
		804	return -EINVAL;
		805	err = priv_prog__load(*skel);
		806	if (!ASSERT_OK(err, "priv_prog__load"))
		807	return -EINVAL;
		808
		809	*fr_skel = priv_freplace_prog__open_opts(&opts);
		810	if (!ASSERT_OK_PTR(*skel, "priv_freplace_prog__open_opts"))
		811	return -EINVAL;
		812
		813	tgt_fd = bpf_program__fd((skel)->progs.xdp_prog1);
		814	return 0;
		815	}
		816
		817	/* Verify that freplace works from user namespace, because bpf token is loaded
		818	* in bpf_object__prepare
		819	*/
		820	static int userns_obj_priv_freplace_prog(int mnt_fd, struct token_lsm *lsm_skel)
		821	{
		822	struct priv_freplace_prog *fr_skel = NULL;
		823	struct priv_prog *skel = NULL;
		824	int err, tgt_fd;
		825
		826	err = userns_obj_priv_freplace_setup(mnt_fd, &fr_skel, &skel, &tgt_fd);
		827	if (!ASSERT_OK(err, "setup"))
		828	goto out;
		829
		830	err = bpf_object__prepare(fr_skel->obj);
		831	if (!ASSERT_OK(err, "freplace__prepare"))
		832	goto out;
		833
		834	err = bpf_program__set_attach_target(fr_skel->progs.new_xdp_prog2, tgt_fd, "xdp_prog1");
		835	if (!ASSERT_OK(err, "set_attach_target"))
		836	goto out;
		837
		838	err = priv_freplace_prog__load(fr_skel);
		839	ASSERT_OK(err, "priv_freplace_prog__load");
		840
		841	out:
		842	priv_freplace_prog__destroy(fr_skel);
		843	priv_prog__destroy(skel);
		844	return err;
		845	}
		846
		847	/* Verify that replace fails to set attach target from user namespace without bpf token */
		848	static int userns_obj_priv_freplace_prog_fail(int mnt_fd, struct token_lsm *lsm_skel)
		849	{
		850	struct priv_freplace_prog *fr_skel = NULL;
		851	struct priv_prog *skel = NULL;
		852	int err, tgt_fd;
		853
		854	err = userns_obj_priv_freplace_setup(mnt_fd, &fr_skel, &skel, &tgt_fd);
		855	if (!ASSERT_OK(err, "setup"))
		856	goto out;
		857
		858	err = bpf_program__set_attach_target(fr_skel->progs.new_xdp_prog2, tgt_fd, "xdp_prog1");
		859	if (ASSERT_ERR(err, "attach fails"))
		860	err = 0;
		861	else
		862	err = -EINVAL;
		863
		864	out:
		865	priv_freplace_prog__destroy(fr_skel);
		866	priv_prog__destroy(skel);
		867	return err;
		868	}
		869
791	/* this test is called with BPF FS that doesn't delegate BPF_BTF_LOAD command,	870	/* this test is called with BPF FS that doesn't delegate BPF_BTF_LOAD command,
792	* which should cause struct_ops application to fail, as BTF won't be uploaded	871	* which should cause struct_ops application to fail, as BTF won't be uploaded
793	* into the kernel, even if STRUCT_OPS programs themselves are allowed	872	* into the kernel, even if STRUCT_OPS programs themselves are allowed
@@ -1004,12 +1083,28 @@ void test_token(void) · 210 unmodified lines
1004	if (test__start_subtest("obj_priv_prog")) {	1083	if (test__start_subtest("obj_priv_prog")) {
1005	struct bpffs_opts opts = {	1084	struct bpffs_opts opts = {
1006	.cmds = bit(BPF_PROG_LOAD),	1085	.cmds = bit(BPF_PROG_LOAD),
1007	.progs = bit(BPF_PROG_TYPE_KPROBE),	1086	.progs = bit(BPF_PROG_TYPE_XDP),
1008	.attachs = ~0ULL,	1087	.attachs = ~0ULL,
1009	};	1088	};
1010		1089
1011	subtest_userns(&opts, userns_obj_priv_prog);	1090	subtest_userns(&opts, userns_obj_priv_prog);
1012	}	1091	}
		1092	if (test__start_subtest("obj_priv_freplace_prog")) {
		1093	struct bpffs_opts opts = {
		1094	.cmds = bit(BPF_BTF_LOAD) \| bit(BPF_PROG_LOAD) \| bit(BPF_BTF_GET_FD_BY_ID),
		1095	.progs = bit(BPF_PROG_TYPE_EXT) \| bit(BPF_PROG_TYPE_XDP),
		1096	.attachs = ~0ULL,
		1097	};
		1098	subtest_userns(&opts, userns_obj_priv_freplace_prog);
		1099	}
		1100	if (test__start_subtest("obj_priv_freplace_prog_fail")) {
		1101	struct bpffs_opts opts = {
		1102	.cmds = bit(BPF_BTF_LOAD) \| bit(BPF_PROG_LOAD) \| bit(BPF_BTF_GET_FD_BY_ID),
		1103	.progs = bit(BPF_PROG_TYPE_EXT) \| bit(BPF_PROG_TYPE_XDP),
		1104	.attachs = ~0ULL,
		1105	};
		1106	subtest_userns(&opts, userns_obj_priv_freplace_prog_fail);
		1107	}
1013	if (test__start_subtest("obj_priv_btf_fail")) {	1108	if (test__start_subtest("obj_priv_btf_fail")) {
1014	struct bpffs_opts opts = {	1109	struct bpffs_opts opts = {
1015	/* disallow BTF loading */	1110	/* disallow BTF loading */

tools/testing/selftests/bpf/prog_tests/usdt.c+10−1

 	LIBBPF_OPTS(bpf_usdt_opts, opts);
 	struct test_usdt *skel;
 	struct test_usdt__bss *bss;
-	int err;
+	int err, i;
 	skel = test_usdt__open_and_load();
 	if (!ASSERT_OK_PTR(skel, "skel_open"))
 	ASSERT_EQ(bss->usdt0_cookie, 0xcafedeadbeeffeed, "usdt0_cookie");
 	ASSERT_EQ(bss->usdt0_arg_cnt, 0, "usdt0_arg_cnt");
 	ASSERT_EQ(bss->usdt0_arg_ret, -ENOENT, "usdt0_arg_ret");
+	ASSERT_EQ(bss->usdt0_arg_size, -ENOENT, "usdt0_arg_size");
 	/* auto-attached usdt3 gets default zero cookie value */
 	ASSERT_EQ(bss->usdt3_cookie, 0, "usdt3_cookie");
 	ASSERT_EQ(bss->usdt3_args[0], 1, "usdt3_arg1");
 	ASSERT_EQ(bss->usdt3_args[1], 42, "usdt3_arg2");
 	ASSERT_EQ(bss->usdt3_args[2], (uintptr_t)&bla, "usdt3_arg3");
+	ASSERT_EQ(bss->usdt3_arg_sizes[0], 4, "usdt3_arg1_size");
+	ASSERT_EQ(bss->usdt3_arg_sizes[1], 8, "usdt3_arg2_size");
+	ASSERT_EQ(bss->usdt3_arg_sizes[2], 8, "usdt3_arg3_size");
 	/* auto-attached usdt12 gets default zero cookie value */
 	ASSERT_EQ(bss->usdt12_cookie, 0, "usdt12_cookie");
 	ASSERT_EQ(bss->usdt12_args[10], nums[idx], "usdt12_arg11");
 	ASSERT_EQ(bss->usdt12_args[11], t1.y, "usdt12_arg12");
+	int usdt12_expected_arg_sizes[12] = { 4, 4, 8, 8, 4, 8, 8, 8, 4, 2, 2, 1 };
+	for (i = 0; i < 12; i++)
+		ASSERT_EQ(bss->usdt12_arg_sizes[i], usdt12_expected_arg_sizes[i], "usdt12_arg_size");
 	/* trigger_func() is marked __always_inline, so USDT invocations will be
 	 * inlined in two different places, meaning that each USDT will have
 	 * at least 2 different places to be attached to. This verifies that

tools/testing/selftests/bpf/prog_tests/verifier.c+6−2

 #include "verifier_ldsx.skel.h"
 #include "verifier_leak_ptr.skel.h"
 #include "verifier_linked_scalars.skel.h"
+#include "verifier_load_acquire.skel.h"
 #include "verifier_loops1.skel.h"
 #include "verifier_lwt.skel.h"
 #include "verifier_map_in_map.skel.h"
 #include "verifier_spill_fill.skel.h"
 #include "verifier_spin_lock.skel.h"
 #include "verifier_stack_ptr.skel.h"
+#include "verifier_store_release.skel.h"
 #include "verifier_subprog_precision.skel.h"
 #include "verifier_subreg.skel.h"
 #include "verifier_tailcall_jit.skel.h"
 	/* test_verifier tests are executed w/o CAP_SYS_ADMIN, do the same here */
 	err = cap_disable_effective(1ULL << CAP_SYS_ADMIN, &old_caps);
 	if (err) {
-		PRINT_FAIL("failed to drop CAP_SYS_ADMIN: %i, %s\n", err, strerror(err));
+		PRINT_FAIL("failed to drop CAP_SYS_ADMIN: %i, %s\n", err, strerror(-err));
 		return;
 	}
 	err = cap_enable_effective(old_caps, NULL);
 	if (err)
-		PRINT_FAIL("failed to restore CAP_SYS_ADMIN: %i, %s\n", err, strerror(err));
+		PRINT_FAIL("failed to restore CAP_SYS_ADMIN: %i, %s\n", err, strerror(-err));
 }
 #define RUN(skel) run_tests_aux(#skel, skel##__elf_bytes, NULL)
 void test_verifier_iterating_callbacks(void)  { RUN(verifier_iterating_callbacks); }
 void test_verifier_jeq_infer_not_null(void)   { RUN(verifier_jeq_infer_not_null); }
 void test_verifier_jit_convergence(void)      { RUN(verifier_jit_convergence); }
+void test_verifier_load_acquire(void)         { RUN(verifier_load_acquire); }
 void test_verifier_ld_ind(void)               { RUN(verifier_ld_ind); }
 void test_verifier_ldsx(void)                  { RUN(verifier_ldsx); }
 void test_verifier_leak_ptr(void)             { RUN(verifier_leak_ptr); }
 void test_verifier_spill_fill(void)           { RUN(verifier_spill_fill); }
 void test_verifier_spin_lock(void)            { RUN(verifier_spin_lock); }
 void test_verifier_stack_ptr(void)            { RUN(verifier_stack_ptr); }
+void test_verifier_store_release(void)        { RUN(verifier_store_release); }
 void test_verifier_subprog_precision(void)    { RUN(verifier_subprog_precision); }
 void test_verifier_subreg(void)               { RUN(verifier_subreg); }
 void test_verifier_tailcall_jit(void)         { RUN(verifier_tailcall_jit); }

tools/testing/selftests/bpf/prog_tests/xdp_vlan.c+175−0

@@ -0,0 +1,175 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	/*
	4	* Network topology:
	5	* ----------- -----------
	6	* \| NS1 \| \| NS2 \|
	7	* \| veth0 -\|--------\|- veth0 \|
	8	* ----------- -----------
	9	*
	10	*/
	11
	12	#define _GNU_SOURCE
	13	#include <net/if.h>
	14	#include <uapi/linux/if_link.h>
	15
	16	#include "network_helpers.h"
	17	#include "test_progs.h"
	18	#include "test_xdp_vlan.skel.h"
	19
	20
	21	#define VETH_NAME "veth0"
	22	#define NS_MAX_SIZE 32
	23	#define NS1_NAME "ns-xdp-vlan-1-"
	24	#define NS2_NAME "ns-xdp-vlan-2-"
	25	#define NS1_IP_ADDR "100.64.10.1"
	26	#define NS2_IP_ADDR "100.64.10.2"
	27	#define VLAN_ID 4011
	28
	29	static int setup_network(char ns1, char ns2)
	30	{
	31	if (!ASSERT_OK(append_tid(ns1, NS_MAX_SIZE), "create ns1 name"))
	32	goto fail;
	33	if (!ASSERT_OK(append_tid(ns2, NS_MAX_SIZE), "create ns2 name"))
	34	goto fail;
	35
	36	SYS(fail, "ip netns add %s", ns1);
	37	SYS(fail, "ip netns add %s", ns2);
	38	SYS(fail, "ip -n %s link add %s type veth peer name %s netns %s",
	39	ns1, VETH_NAME, VETH_NAME, ns2);
	40
	41	/* NOTICE: XDP require VLAN header inside packet payload
	42	* - Thus, disable VLAN offloading driver features
	43	*/
	44	SYS(fail, "ip netns exec %s ethtool -K %s rxvlan off txvlan off", ns1, VETH_NAME);
	45	SYS(fail, "ip netns exec %s ethtool -K %s rxvlan off txvlan off", ns2, VETH_NAME);
	46
	47	/* NS1 configuration */
	48	SYS(fail, "ip -n %s addr add %s/24 dev %s", ns1, NS1_IP_ADDR, VETH_NAME);
	49	SYS(fail, "ip -n %s link set %s up", ns1, VETH_NAME);
	50
	51	/* NS2 configuration */
	52	SYS(fail, "ip -n %s link add link %s name %s.%d type vlan id %d",
	53	ns2, VETH_NAME, VETH_NAME, VLAN_ID, VLAN_ID);
	54	SYS(fail, "ip -n %s addr add %s/24 dev %s.%d", ns2, NS2_IP_ADDR, VETH_NAME, VLAN_ID);
	55	SYS(fail, "ip -n %s link set %s up", ns2, VETH_NAME);
	56	SYS(fail, "ip -n %s link set %s.%d up", ns2, VETH_NAME, VLAN_ID);
	57
	58	/* At this point ping should fail because VLAN tags are only used by NS2 */
	59	return !SYS_NOFAIL("ip netns exec %s ping -W 1 -c1 %s", ns2, NS1_IP_ADDR);
	60
	61	fail:
	62	return -1;
	63	}
	64
	65	static void cleanup_network(const char ns1, const char ns2)
	66	{
	67	SYS_NOFAIL("ip netns del %s", ns1);
	68	SYS_NOFAIL("ip netns del %s", ns2);
	69	}
	70
	71	static void xdp_vlan(struct bpf_program xdp, struct bpf_program tc, u32 flags)
	72	{
	73	LIBBPF_OPTS(bpf_tc_hook, tc_hook, .attach_point = BPF_TC_EGRESS);
	74	LIBBPF_OPTS(bpf_tc_opts, tc_opts, .handle = 1, .priority = 1);
	75	char ns1[NS_MAX_SIZE] = NS1_NAME;
	76	char ns2[NS_MAX_SIZE] = NS2_NAME;
	77	struct nstoken *nstoken = NULL;
	78	int interface;
	79	int ret;
	80
	81	if (!ASSERT_OK(setup_network(ns1, ns2), "setup network"))
	82	goto cleanup;
	83
	84	nstoken = open_netns(ns1);
	85	if (!ASSERT_OK_PTR(nstoken, "open NS1"))
	86	goto cleanup;
	87
	88	interface = if_nametoindex(VETH_NAME);
	89	if (!ASSERT_NEQ(interface, 0, "get interface index"))
	90	goto cleanup;
	91
	92	ret = bpf_xdp_attach(interface, bpf_program__fd(xdp), flags, NULL);
	93	if (!ASSERT_OK(ret, "attach xdp_vlan_change"))
	94	goto cleanup;
	95
	96	tc_hook.ifindex = interface;
	97	ret = bpf_tc_hook_create(&tc_hook);
	98	if (!ASSERT_OK(ret, "bpf_tc_hook_create"))
	99	goto detach_xdp;
	100
	101	/* Now we'll use BPF programs to pop/push the VLAN tags */
	102	tc_opts.prog_fd = bpf_program__fd(tc);
	103	ret = bpf_tc_attach(&tc_hook, &tc_opts);
	104	if (!ASSERT_OK(ret, "bpf_tc_attach"))
	105	goto detach_xdp;
	106
	107	close_netns(nstoken);
	108	nstoken = NULL;
	109
	110	/* Now the namespaces can reach each-other, test with pings */
	111	SYS(detach_tc, "ip netns exec %s ping -i 0.2 -W 2 -c 2 %s > /dev/null", ns1, NS2_IP_ADDR);
	112	SYS(detach_tc, "ip netns exec %s ping -i 0.2 -W 2 -c 2 %s > /dev/null", ns2, NS1_IP_ADDR);
	113
	114
	115	detach_tc:
	116	bpf_tc_detach(&tc_hook, &tc_opts);
	117	detach_xdp:
	118	bpf_xdp_detach(interface, flags, NULL);
	119	cleanup:
	120	close_netns(nstoken);
	121	cleanup_network(ns1, ns2);
	122	}
	123
	124	/* First test: Remove VLAN by setting VLAN ID 0, using "xdp_vlan_change"
	125	* egress use TC to add back VLAN tag 4011
	126	*/
	127	void test_xdp_vlan_change(void)
	128	{
	129	struct test_xdp_vlan *skel;
	130
	131	skel = test_xdp_vlan__open_and_load();
	132	if (!ASSERT_OK_PTR(skel, "xdp_vlan__open_and_load"))
	133	return;
	134
	135	if (test__start_subtest("0"))
	136	xdp_vlan(skel->progs.xdp_vlan_change, skel->progs.tc_vlan_push, 0);
	137
	138	if (test__start_subtest("DRV_MODE"))
	139	xdp_vlan(skel->progs.xdp_vlan_change, skel->progs.tc_vlan_push,
	140	XDP_FLAGS_DRV_MODE);
	141
	142	if (test__start_subtest("SKB_MODE"))
	143	xdp_vlan(skel->progs.xdp_vlan_change, skel->progs.tc_vlan_push,
	144	XDP_FLAGS_SKB_MODE);
	145
	146	test_xdp_vlan__destroy(skel);
	147	}
	148
	149	/* Second test: XDP prog fully remove vlan header
	150	*
	151	* Catch kernel bug for generic-XDP, that doesn't allow us to
	152	* remove a VLAN header, because skb->protocol still contain VLAN
	153	* ETH_P_8021Q indication, and this cause overwriting of our changes.
	154	*/
	155	void test_xdp_vlan_remove(void)
	156	{
	157	struct test_xdp_vlan *skel;
	158
	159	skel = test_xdp_vlan__open_and_load();
	160	if (!ASSERT_OK_PTR(skel, "xdp_vlan__open_and_load"))
	161	return;
	162
	163	if (test__start_subtest("0"))
	164	xdp_vlan(skel->progs.xdp_vlan_remove_outer2, skel->progs.tc_vlan_push, 0);
	165
	166	if (test__start_subtest("DRV_MODE"))
	167	xdp_vlan(skel->progs.xdp_vlan_remove_outer2, skel->progs.tc_vlan_push,
	168	XDP_FLAGS_DRV_MODE);
	169
	170	if (test__start_subtest("SKB_MODE"))
	171	xdp_vlan(skel->progs.xdp_vlan_remove_outer2, skel->progs.tc_vlan_push,
	172	XDP_FLAGS_SKB_MODE);
	173
	174	test_xdp_vlan__destroy(skel);
	175	}

tools/testing/selftests/bpf/progs/arena_atomics.c+119−2

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	#include <stdbool.h>	6	#include <stdbool.h>
7	#include <stdatomic.h>	7	#include <stdatomic.h>
8	#include "bpf_arena_common.h"	8	#include "bpf_arena_common.h"
		9	#include "../../../include/linux/filter.h"
		10	#include "bpf_misc.h"
9		11
10	struct {	12	struct {
11	__uint(type, BPF_MAP_TYPE_ARENA);	13	__uint(type, BPF_MAP_TYPE_ARENA);
@@ -19,9 +21,17 @@ struct { · 7 unmodified lines
19	} arena SEC(".maps");	21	} arena SEC(".maps");
20		22
21	#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)	23	#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
22	bool skip_tests __attribute((__section__(".data"))) = false;	24	bool skip_all_tests __attribute((__section__(".data"))) = false;
23	#else	25	#else
24	bool skip_tests = true;	26	bool skip_all_tests = true;
		27	#endif
		28
		29	#if defined(ENABLE_ATOMICS_TESTS) && \
		30	defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
		31	(defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
		32	bool skip_lacq_srel_tests __attribute((__section__(".data"))) = false;
		33	#else
		34	bool skip_lacq_srel_tests = true;
25	#endif	35	#endif
26		36
27	__u32 pid = 0;	37	__u32 pid = 0;
@@ -274,4 +284,111 @@ int uaf(const void *ctx) · 246 unmodified lines
274	return 0;	284	return 0;
275	}	285	}
276		286
		287	#if __clang_major__ >= 18
		288	__u8 __arena_global load_acquire8_value = 0x12;
		289	__u16 __arena_global load_acquire16_value = 0x1234;
		290	__u32 __arena_global load_acquire32_value = 0x12345678;
		291	__u64 __arena_global load_acquire64_value = 0x1234567890abcdef;
		292
		293	__u8 __arena_global load_acquire8_result = 0;
		294	__u16 __arena_global load_acquire16_result = 0;
		295	__u32 __arena_global load_acquire32_result = 0;
		296	__u64 __arena_global load_acquire64_result = 0;
		297	#else
		298	/* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
		299	* this issue by defining the below variables as 64-bit.
		300	*/
		301	__u64 __arena_global load_acquire8_value;
		302	__u64 __arena_global load_acquire16_value;
		303	__u64 __arena_global load_acquire32_value;
		304	__u64 __arena_global load_acquire64_value;
		305
		306	__u64 __arena_global load_acquire8_result;
		307	__u64 __arena_global load_acquire16_result;
		308	__u64 __arena_global load_acquire32_result;
		309	__u64 __arena_global load_acquire64_result;
		310	#endif
		311
		312	SEC("raw_tp/sys_enter")
		313	int load_acquire(const void *ctx)
		314	{
		315	#if defined(ENABLE_ATOMICS_TESTS) && \
		316	defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
		317	(defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
		318
		319	#define LOAD_ACQUIRE_ARENA(SIZEOP, SIZE, SRC, DST) \
		320	{ asm volatile ( \
		321	"r1 = %[" #SRC "] ll;" \
		322	"r1 = addr_space_cast(r1, 0x0, 0x1);" \
		323	".8byte %[load_acquire_insn];" \
		324	"r3 = %[" #DST "] ll;" \
		325	"r3 = addr_space_cast(r3, 0x0, 0x1);" \
		326	"(" #SIZE " )(r3 + 0) = r2;" \
		327	: \
		328	: __imm_addr(SRC), \
		329	__imm_insn(load_acquire_insn, \
		330	BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_LOAD_ACQ, \
		331	BPF_REG_2, BPF_REG_1, 0)), \
		332	__imm_addr(DST) \
		333	: __clobber_all); } \
		334
		335	LOAD_ACQUIRE_ARENA(B, u8, load_acquire8_value, load_acquire8_result)
		336	LOAD_ACQUIRE_ARENA(H, u16, load_acquire16_value,
		337	load_acquire16_result)
		338	LOAD_ACQUIRE_ARENA(W, u32, load_acquire32_value,
		339	load_acquire32_result)
		340	LOAD_ACQUIRE_ARENA(DW, u64, load_acquire64_value,
		341	load_acquire64_result)
		342	#undef LOAD_ACQUIRE_ARENA
		343
		344	#endif
		345	return 0;
		346	}
		347
		348	#if __clang_major__ >= 18
		349	__u8 __arena_global store_release8_result = 0;
		350	__u16 __arena_global store_release16_result = 0;
		351	__u32 __arena_global store_release32_result = 0;
		352	__u64 __arena_global store_release64_result = 0;
		353	#else
		354	/* clang-17 crashes if the .addr_space.1 ELF section has holes. Work around
		355	* this issue by defining the below variables as 64-bit.
		356	*/
		357	__u64 __arena_global store_release8_result;
		358	__u64 __arena_global store_release16_result;
		359	__u64 __arena_global store_release32_result;
		360	__u64 __arena_global store_release64_result;
		361	#endif
		362
		363	SEC("raw_tp/sys_enter")
		364	int store_release(const void *ctx)
		365	{
		366	#if defined(ENABLE_ATOMICS_TESTS) && \
		367	defined(__BPF_FEATURE_ADDR_SPACE_CAST) && \
		368	(defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
		369
		370	#define STORE_RELEASE_ARENA(SIZEOP, DST, VAL) \
		371	{ asm volatile ( \
		372	"r1 = " VAL ";" \
		373	"r2 = %[" #DST "] ll;" \
		374	"r2 = addr_space_cast(r2, 0x0, 0x1);" \
		375	".8byte %[store_release_insn];" \
		376	: \
		377	: __imm_addr(DST), \
		378	__imm_insn(store_release_insn, \
		379	BPF_ATOMIC_OP(BPF_##SIZEOP, BPF_STORE_REL, \
		380	BPF_REG_2, BPF_REG_1, 0)) \
		381	: __clobber_all); } \
		382
		383	STORE_RELEASE_ARENA(B, store_release8_result, "0x12")
		384	STORE_RELEASE_ARENA(H, store_release16_result, "0x1234")
		385	STORE_RELEASE_ARENA(W, store_release32_result, "0x12345678")
		386	STORE_RELEASE_ARENA(DW, store_release64_result,
		387	"0x1234567890abcdef ll")
		388	#undef STORE_RELEASE_ARENA
		389
		390	#endif
		391	return 0;
		392	}
		393
277	char _license[] SEC("license") = "GPL";	394	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/arena_spin_lock.c+51−0

@@ -0,0 +1,51 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#include <vmlinux.h>
	4	#include <bpf/bpf_tracing.h>
	5	#include <bpf/bpf_helpers.h>
	6	#include "bpf_misc.h"
	7	#include "bpf_arena_spin_lock.h"
	8
	9	struct {
	10	__uint(type, BPF_MAP_TYPE_ARENA);
	11	__uint(map_flags, BPF_F_MMAPABLE);
	12	__uint(max_entries, 100); /* number of pages */
	13	#ifdef __TARGET_ARCH_arm64
	14	__ulong(map_extra, 0x1ull << 32); /* start of mmap() region */
	15	#else
	16	__ulong(map_extra, 0x1ull << 44); /* start of mmap() region */
	17	#endif
	18	} arena SEC(".maps");
	19
	20	int cs_count;
	21
	22	#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
	23	arena_spinlock_t __arena lock;
	24	int test_skip = 1;
	25	#else
	26	int test_skip = 2;
	27	#endif
	28
	29	int counter;
	30	int limit;
	31
	32	SEC("tc")
	33	int prog(void *ctx)
	34	{
	35	int ret = -2;
	36
	37	#if defined(ENABLE_ATOMICS_TESTS) && defined(__BPF_FEATURE_ADDR_SPACE_CAST)
	38	unsigned long flags;
	39
	40	if ((ret = arena_spin_lock_irqsave(&lock, flags)))
	41	return ret;
	42	if (counter != limit)
	43	counter++;
	44	bpf_repeat(cs_count);
	45	ret = 0;
	46	arena_spin_unlock_irqrestore(&lock, flags);
	47	#endif
	48	return ret;
	49	}
	50
	51	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/bpf_iter_tasks.c+110−0

@@ -9,6 +9,13 @@ char _license[] SEC("license") = "GPL"; · 8 unmodified lines
9	uint32_t tid = 0;	9	uint32_t tid = 0;
10	int num_unknown_tid = 0;	10	int num_unknown_tid = 0;
11	int num_known_tid = 0;	11	int num_known_tid = 0;
		12	void *user_ptr = 0;
		13	void *user_ptr_long = 0;
		14	uint32_t pid = 0;
		15
		16	static char big_str1[5000];
		17	static char big_str2[5005];
		18	static char big_str3[4996];
12		19
13	SEC("iter/task")	20	SEC("iter/task")
14	int dump_task(struct bpf_iter__task *ctx)	21	int dump_task(struct bpf_iter__task *ctx)
@@ -35,7 +42,9 @@ int dump_task(struct bpf_iter__task *ctx) · 20 unmodified lines
35	}	42	}
36		43
37	int num_expected_failure_copy_from_user_task = 0;	44	int num_expected_failure_copy_from_user_task = 0;
		45	int num_expected_failure_copy_from_user_task_str = 0;
38	int num_success_copy_from_user_task = 0;	46	int num_success_copy_from_user_task = 0;
		47	int num_success_copy_from_user_task_str = 0;
39		48
40	SEC("iter.s/task")	49	SEC("iter.s/task")
41	int dump_task_sleepable(struct bpf_iter__task *ctx)	50	int dump_task_sleepable(struct bpf_iter__task *ctx)
@@ -44,6 +53,9 @@ int dump_task_sleepable(struct bpf_iter__task *ctx) · 2 unmodified lines
44	struct task_struct *task = ctx->task;	53	struct task_struct *task = ctx->task;
45	static const char info[] = " === END ===";	54	static const char info[] = " === END ===";
46	struct pt_regs *regs;	55	struct pt_regs *regs;
		56	char task_str1[10] = "aaaaaaaaaa";
		57	char task_str2[10], task_str3[10];
		58	char task_str4[20] = "aaaaaaaaaaaaaaaaaaaa";
47	void *ptr;	59	void *ptr;
48	uint32_t user_data = 0;	60	uint32_t user_data = 0;
49	int ret;	61	int ret;
@@ -78,8 +90,106 @@ int dump_task_sleepable(struct bpf_iter__task *ctx) · 28 unmodified lines
78	BPF_SEQ_PRINTF(seq, "%s\n", info);	90	BPF_SEQ_PRINTF(seq, "%s\n", info);
79	return 0;	91	return 0;
80	}	92	}
		93
81	++num_success_copy_from_user_task;	94	++num_success_copy_from_user_task;
82		95
		96	/* Read an invalid pointer and ensure we get an error */
		97	ptr = NULL;
		98	ret = bpf_copy_from_user_task_str((char *)task_str1, sizeof(task_str1), ptr, task, 0);
		99	if (ret >= 0 \|\| task_str1[9] != 'a' \|\| task_str1[0] != '\0') {
		100	BPF_SEQ_PRINTF(seq, "%s\n", info);
		101	return 0;
		102	}
		103
		104	/* Read an invalid pointer and ensure we get error with pad zeros flag */
		105	ptr = NULL;
		106	ret = bpf_copy_from_user_task_str((char *)task_str1, sizeof(task_str1),
		107	ptr, task, BPF_F_PAD_ZEROS);
		108	if (ret >= 0 \|\| task_str1[9] != '\0' \|\| task_str1[0] != '\0') {
		109	BPF_SEQ_PRINTF(seq, "%s\n", info);
		110	return 0;
		111	}
		112
		113	++num_expected_failure_copy_from_user_task_str;
		114
		115	/* Same length as the string */
		116	ret = bpf_copy_from_user_task_str((char *)task_str2, 10, user_ptr, task, 0);
		117	/* only need to do the task pid check once */
		118	if (bpf_strncmp(task_str2, 10, "test_data\0") != 0 \|\| ret != 10 \|\| task->tgid != pid) {
		119	BPF_SEQ_PRINTF(seq, "%s\n", info);
		120	return 0;
		121	}
		122
		123	/* Shorter length than the string */
		124	ret = bpf_copy_from_user_task_str((char *)task_str3, 2, user_ptr, task, 0);
		125	if (bpf_strncmp(task_str3, 2, "t\0") != 0 \|\| ret != 2) {
		126	BPF_SEQ_PRINTF(seq, "%s\n", info);
		127	return 0;
		128	}
		129
		130	/* Longer length than the string */
		131	ret = bpf_copy_from_user_task_str((char *)task_str4, 20, user_ptr, task, 0);
		132	if (bpf_strncmp(task_str4, 10, "test_data\0") != 0 \|\| ret != 10
		133	\|\| task_str4[sizeof(task_str4) - 1] != 'a') {
		134	BPF_SEQ_PRINTF(seq, "%s\n", info);
		135	return 0;
		136	}
		137
		138	/* Longer length than the string with pad zeros flag */
		139	ret = bpf_copy_from_user_task_str((char *)task_str4, 20, user_ptr, task, BPF_F_PAD_ZEROS);
		140	if (bpf_strncmp(task_str4, 10, "test_data\0") != 0 \|\| ret != 10
		141	\|\| task_str4[sizeof(task_str4) - 1] != '\0') {
		142	BPF_SEQ_PRINTF(seq, "%s\n", info);
		143	return 0;
		144	}
		145
		146	/* Longer length than the string past a page boundary */
		147	ret = bpf_copy_from_user_task_str(big_str1, 5000, user_ptr, task, 0);
		148	if (bpf_strncmp(big_str1, 10, "test_data\0") != 0 \|\| ret != 10) {
		149	BPF_SEQ_PRINTF(seq, "%s\n", info);
		150	return 0;
		151	}
		152
		153	/* String that crosses a page boundary */
		154	ret = bpf_copy_from_user_task_str(big_str1, 5000, user_ptr_long, task, BPF_F_PAD_ZEROS);
		155	if (bpf_strncmp(big_str1, 4, "baba") != 0 \|\| ret != 5000
		156	\|\| bpf_strncmp(big_str1 + 4996, 4, "bab\0") != 0) {
		157	BPF_SEQ_PRINTF(seq, "%s\n", info);
		158	return 0;
		159	}
		160
		161	for (int i = 0; i < 4999; ++i) {
		162	if (i % 2 == 0) {
		163	if (big_str1[i] != 'b') {
		164	BPF_SEQ_PRINTF(seq, "%s\n", info);
		165	return 0;
		166	}
		167	} else {
		168	if (big_str1[i] != 'a') {
		169	BPF_SEQ_PRINTF(seq, "%s\n", info);
		170	return 0;
		171	}
		172	}
		173	}
		174
		175	/* Longer length than the string that crosses a page boundary */
		176	ret = bpf_copy_from_user_task_str(big_str2, 5005, user_ptr_long, task, BPF_F_PAD_ZEROS);
		177	if (bpf_strncmp(big_str2, 4, "baba") != 0 \|\| ret != 5000
		178	\|\| bpf_strncmp(big_str2 + 4996, 5, "bab\0\0") != 0) {
		179	BPF_SEQ_PRINTF(seq, "%s\n", info);
		180	return 0;
		181	}
		182
		183	/* Shorter length than the string that crosses a page boundary */
		184	ret = bpf_copy_from_user_task_str(big_str3, 4996, user_ptr_long, task, 0);
		185	if (bpf_strncmp(big_str3, 4, "baba") != 0 \|\| ret != 4996
		186	\|\| bpf_strncmp(big_str3 + 4992, 4, "bab\0") != 0) {
		187	BPF_SEQ_PRINTF(seq, "%s\n", info);
		188	return 0;
		189	}
		190
		191	++num_success_copy_from_user_task_str;
		192
83	if (ctx->meta->seq_num == 0)	193	if (ctx->meta->seq_num == 0)
84	BPF_SEQ_PRINTF(seq, " tgid gid data\n");	194	BPF_SEQ_PRINTF(seq, " tgid gid data\n");
85		195

tools/testing/selftests/bpf/progs/bpf_misc.h+22−0

@@ -135,6 +135,8 @@ · 134 unmodified lines
135	#define __arch_arm64 __arch("ARM64")	135	#define __arch_arm64 __arch("ARM64")
136	#define __arch_riscv64 __arch("RISCV64")	136	#define __arch_riscv64 __arch("RISCV64")
137	#define __caps_unpriv(caps) __attribute__((btf_decl_tag("comment:test_caps_unpriv=" EXPAND_QUOTE(caps))))	137	#define __caps_unpriv(caps) __attribute__((btf_decl_tag("comment:test_caps_unpriv=" EXPAND_QUOTE(caps))))
		138	#define __load_if_JITed() __attribute__((btf_decl_tag("comment:load_mode=jited")))
		139	#define __load_if_no_JITed() __attribute__((btf_decl_tag("comment:load_mode=no_jited")))
138		140
139	/* Define common capabilities tested using __caps_unpriv */	141	/* Define common capabilities tested using __caps_unpriv */
140	#define CAP_NET_ADMIN 12	142	#define CAP_NET_ADMIN 12
@@ -172,6 +174,9 @@ · 31 unmodified lines
172	#elif defined(__TARGET_ARCH_riscv)	174	#elif defined(__TARGET_ARCH_riscv)
173	#define SYSCALL_WRAPPER 1	175	#define SYSCALL_WRAPPER 1
174	#define SYS_PREFIX "__riscv_"	176	#define SYS_PREFIX "__riscv_"
		177	#elif defined(__TARGET_ARCH_powerpc)
		178	#define SYSCALL_WRAPPER 1
		179	#define SYS_PREFIX ""
175	#else	180	#else
176	#define SYSCALL_WRAPPER 0	181	#define SYSCALL_WRAPPER 0
177	#define SYS_PREFIX "__se_"	182	#define SYS_PREFIX "__se_"
@@ -208,4 +213,21 @@ · 30 unmodified lines
208	#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))	213	#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
209	#endif	214	#endif
210		215
		216	#if (defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86) \|\| \
		217	(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) \|\| \
		218	defined(__TARGET_ARCH_arm) \|\| defined(__TARGET_ARCH_s390) \|\| \
		219	defined(__TARGET_ARCH_loongarch)) && \
		220	__clang_major__ >= 18
		221	#define CAN_USE_GOTOL
		222	#endif
		223
		224	#if _clang_major__ >= 18
		225	#define CAN_USE_BPF_ST
		226	#endif
		227
		228	#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
		229	(defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
		230	#define CAN_USE_LOAD_ACQ_STORE_REL
		231	#endif
		232
211	#endif	233	#endif

tools/testing/selftests/bpf/progs/bpf_tracing_net.h+4−0

 #define SO_KEEPALIVE		9
 #define SO_PRIORITY		12
 #define SO_REUSEPORT		15
+#if defined(__TARGET_ARCH_powerpc)
+#define SO_RCVLOWAT		16
+#else
 #define SO_RCVLOWAT		18
+#endif
 #define SO_BINDTODEVICE		25
 #define SO_MARK			36
 #define SO_MAX_PACING_RATE	47

tools/testing/selftests/bpf/progs/btf__core_reloc_arrays___err_bad_signed_arr_elem_sz.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_arrays___err_bad_signed_arr_elem_sz x) {}

tools/testing/selftests/bpf/progs/cgroup_preorder.c+41−0

@@ -0,0 +1,41 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#include <vmlinux.h>
	4	#include <bpf/bpf_helpers.h>
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	unsigned int idx;
	9	__u8 result[4];
	10
	11	SEC("cgroup/getsockopt")
	12	int child(struct bpf_sockopt *ctx)
	13	{
	14	if (idx < 4)
	15	result[idx++] = 1;
	16	return 1;
	17	}
	18
	19	SEC("cgroup/getsockopt")
	20	int child_2(struct bpf_sockopt *ctx)
	21	{
	22	if (idx < 4)
	23	result[idx++] = 2;
	24	return 1;
	25	}
	26
	27	SEC("cgroup/getsockopt")
	28	int parent(struct bpf_sockopt *ctx)
	29	{
	30	if (idx < 4)
	31	result[idx++] = 3;
	32	return 1;
	33	}
	34
	35	SEC("cgroup/getsockopt")
	36	int parent_2(struct bpf_sockopt *ctx)
	37	{
	38	if (idx < 4)
	39	result[idx++] = 4;
	40	return 1;
	41	}

tools/testing/selftests/bpf/progs/changes_pkt_data.c+0−39

@@ -1,39 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2
3	#include <linux/bpf.h>
4	#include <bpf/bpf_helpers.h>
5
6	__noinline
7	long changes_pkt_data(struct __sk_buff *sk)
8	{
9	return bpf_skb_pull_data(sk, 0);
10	}
11
12	__noinline __weak
13	long does_not_change_pkt_data(struct __sk_buff *sk)
14	{
15	return 0;
16	}
17
18	SEC("?tc")
19	int main_with_subprogs(struct __sk_buff *sk)
20	{
21	changes_pkt_data(sk);
22	does_not_change_pkt_data(sk);
23	return 0;
24	}
25
26	SEC("?tc")
27	int main_changes(struct __sk_buff *sk)
28	{
29	bpf_skb_pull_data(sk, 0);
30	return 0;
31	}
32
33	SEC("?tc")
34	int main_does_not_change(struct __sk_buff *sk)
35	{
36	return 0;
37	}
38
39	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/compute_live_registers.c+424−0

@@ -0,0 +1,424 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include "../../../include/linux/filter.h"
	6	#include "bpf_arena_common.h"
	7	#include "bpf_misc.h"
	8
	9	struct {
	10	__uint(type, BPF_MAP_TYPE_ARRAY);
	11	__uint(max_entries, 1);
	12	__type(key, __u32);
	13	__type(value, __u64);
	14	} test_map SEC(".maps");
	15
	16	struct {
	17	__uint(type, BPF_MAP_TYPE_ARENA);
	18	__uint(map_flags, BPF_F_MMAPABLE);
	19	__uint(max_entries, 1);
	20	} arena SEC(".maps");
	21
	22	SEC("socket")
	23	__log_level(2)
	24	__msg(" 0: .......... (b7) r0 = 42")
	25	__msg(" 1: 0......... (bf) r1 = r0")
	26	__msg(" 2: .1........ (bf) r2 = r1")
	27	__msg(" 3: ..2....... (bf) r3 = r2")
	28	__msg(" 4: ...3...... (bf) r4 = r3")
	29	__msg(" 5: ....4..... (bf) r5 = r4")
	30	__msg(" 6: .....5.... (bf) r6 = r5")
	31	__msg(" 7: ......6... (bf) r7 = r6")
	32	__msg(" 8: .......7.. (bf) r8 = r7")
	33	__msg(" 9: ........8. (bf) r9 = r8")
	34	__msg("10: .........9 (bf) r0 = r9")
	35	__msg("11: 0......... (95) exit")
	36	__naked void assign_chain(void)
	37	{
	38	asm volatile (
	39	"r0 = 42;"
	40	"r1 = r0;"
	41	"r2 = r1;"
	42	"r3 = r2;"
	43	"r4 = r3;"
	44	"r5 = r4;"
	45	"r6 = r5;"
	46	"r7 = r6;"
	47	"r8 = r7;"
	48	"r9 = r8;"
	49	"r0 = r9;"
	50	"exit;"
	51	::: __clobber_all);
	52	}
	53
	54	SEC("socket")
	55	__log_level(2)
	56	__msg("0: .......... (b7) r1 = 7")
	57	__msg("1: .1........ (07) r1 += 7")
	58	__msg("2: .......... (b7) r2 = 7")
	59	__msg("3: ..2....... (b7) r3 = 42")
	60	__msg("4: ..23...... (0f) r2 += r3")
	61	__msg("5: .......... (b7) r0 = 0")
	62	__msg("6: 0......... (95) exit")
	63	__naked void arithmetics(void)
	64	{
	65	asm volatile (
	66	"r1 = 7;"
	67	"r1 += 7;"
	68	"r2 = 7;"
	69	"r3 = 42;"
	70	"r2 += r3;"
	71	"r0 = 0;"
	72	"exit;"
	73	::: __clobber_all);
	74	}
	75
	76	#ifdef CAN_USE_BPF_ST
	77	SEC("socket")
	78	__log_level(2)
	79	__msg(" 1: .1........ (07) r1 += -8")
	80	__msg(" 2: .1........ (7a) (u64 )(r1 +0) = 7")
	81	__msg(" 3: .1........ (b7) r2 = 42")
	82	__msg(" 4: .12....... (7b) (u64 )(r1 +0) = r2")
	83	__msg(" 5: .12....... (7b) (u64 )(r1 +0) = r2")
	84	__msg(" 6: .......... (b7) r0 = 0")
	85	__naked void store(void)
	86	{
	87	asm volatile (
	88	"r1 = r10;"
	89	"r1 += -8;"
	90	"(u64 )(r1 +0) = 7;"
	91	"r2 = 42;"
	92	"(u64 )(r1 +0) = r2;"
	93	"(u64 )(r1 +0) = r2;"
	94	"r0 = 0;"
	95	"exit;"
	96	::: __clobber_all);
	97	}
	98	#endif
	99
	100	SEC("socket")
	101	__log_level(2)
	102	__msg("1: ....4..... (07) r4 += -8")
	103	__msg("2: ....4..... (79) r5 = (u64 )(r4 +0)")
	104	__msg("3: ....45.... (07) r4 += -8")
	105	__naked void load(void)
	106	{
	107	asm volatile (
	108	"r4 = r10;"
	109	"r4 += -8;"
	110	"r5 = (u64 )(r4 +0);"
	111	"r4 += -8;"
	112	"r0 = r5;"
	113	"exit;"
	114	::: __clobber_all);
	115	}
	116
	117	SEC("socket")
	118	__log_level(2)
	119	__msg("0: .1........ (61) r2 = (u32 )(r1 +0)")
	120	__msg("1: ..2....... (d4) r2 = le64 r2")
	121	__msg("2: ..2....... (bf) r0 = r2")
	122	__naked void endian(void)
	123	{
	124	asm volatile (
	125	"r2 = (u32 )(r1 +0);"
	126	"r2 = le64 r2;"
	127	"r0 = r2;"
	128	"exit;"
	129	::: __clobber_all);
	130	}
	131
	132	SEC("socket")
	133	__log_level(2)
	134	__msg(" 8: 0......... (b7) r1 = 1")
	135	__msg(" 9: 01........ (db) r1 = atomic64_fetch_add((u64 *)(r0 +0), r1)")
	136	__msg("10: 01........ (c3) lock (u32 )(r0 +0) += r1")
	137	__msg("11: 01........ (db) r1 = atomic64_xchg((u64 *)(r0 +0), r1)")
	138	__msg("12: 01........ (bf) r2 = r0")
	139	__msg("13: .12....... (bf) r0 = r1")
	140	__msg("14: 012....... (db) r0 = atomic64_cmpxchg((u64 *)(r2 +0), r0, r1)")
	141	__naked void atomic(void)
	142	{
	143	asm volatile (
	144	"r2 = r10;"
	145	"r2 += -8;"
	146	"r1 = 0;"
	147	"(u64 )(r2 +0) = r1;"
	148	"r1 = %[test_map] ll;"
	149	"call %[bpf_map_lookup_elem];"
	150	"if r0 == 0 goto 1f;"
	151	"r1 = 1;"
	152	"r1 = atomic_fetch_add((u64 *)(r0 +0), r1);"
	153	".8byte %[add_nofetch];" /* same as "lock (u32 )(r0 +0) += r1;" */
	154	"r1 = xchg_64(r0 + 0, r1);"
	155	"r2 = r0;"
	156	"r0 = r1;"
	157	"r0 = cmpxchg_64(r2 + 0, r0, r1);"
	158	"1: exit;"
	159	:
	160	: __imm(bpf_map_lookup_elem),
	161	__imm_addr(test_map),
	162	__imm_insn(add_nofetch, BPF_ATOMIC_OP(BPF_W, BPF_ADD, BPF_REG_0, BPF_REG_1, 0))
	163	: __clobber_all);
	164	}
	165
	166	#ifdef CAN_USE_LOAD_ACQ_STORE_REL
	167
	168	SEC("socket")
	169	__log_level(2)
	170	__msg("2: .12....... (db) store_release((u64 *)(r2 -8), r1)")
	171	__msg("3: .......... (bf) r3 = r10")
	172	__msg("4: ...3...... (db) r4 = load_acquire((u64 *)(r3 -8))")
	173	__naked void atomic_load_acq_store_rel(void)
	174	{
	175	asm volatile (
	176	"r1 = 42;"
	177	"r2 = r10;"
	178	".8byte %[store_release_insn];" /* store_release((u64 )(r2 - 8), r1); /
	179	"r3 = r10;"
	180	".8byte %[load_acquire_insn];" /* r4 = load_acquire((u64 )(r3 + 0)); /
	181	"r0 = r4;"
	182	"exit;"
	183	:
	184	: __imm_insn(store_release_insn,
	185	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_2, BPF_REG_1, -8)),
	186	__imm_insn(load_acquire_insn,
	187	BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_4, BPF_REG_3, -8))
	188	: __clobber_all);
	189	}
	190
	191	#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
	192
	193	SEC("socket")
	194	__log_level(2)
	195	__msg("4: .12....7.. (85) call bpf_trace_printk#6")
	196	__msg("5: 0......7.. (0f) r0 += r7")
	197	__naked void regular_call(void)
	198	{
	199	asm volatile (
	200	"r7 = 1;"
	201	"r1 = r10;"
	202	"r1 += -8;"
	203	"r2 = 1;"
	204	"call %[bpf_trace_printk];"
	205	"r0 += r7;"
	206	"exit;"
	207	:
	208	: __imm(bpf_trace_printk)
	209	: __clobber_all);
	210	}
	211
	212	SEC("socket")
	213	__log_level(2)
	214	__msg("2: 012....... (25) if r1 > 0x7 goto pc+1")
	215	__msg("3: ..2....... (bf) r0 = r2")
	216	__naked void if1(void)
	217	{
	218	asm volatile (
	219	"r0 = 1;"
	220	"r2 = 2;"
	221	"if r1 > 0x7 goto +1;"
	222	"r0 = r2;"
	223	"exit;"
	224	::: __clobber_all);
	225	}
	226
	227	SEC("socket")
	228	__log_level(2)
	229	__msg("3: 0123...... (2d) if r1 > r3 goto pc+1")
	230	__msg("4: ..2....... (bf) r0 = r2")
	231	__naked void if2(void)
	232	{
	233	asm volatile (
	234	"r0 = 1;"
	235	"r2 = 2;"
	236	"r3 = 7;"
	237	"if r1 > r3 goto +1;"
	238	"r0 = r2;"
	239	"exit;"
	240	::: __clobber_all);
	241	}
	242
	243	SEC("socket")
	244	__log_level(2)
	245	__msg("0: .......... (b7) r1 = 0")
	246	__msg("1: .1........ (b7) r2 = 7")
	247	__msg("2: .12....... (25) if r1 > 0x7 goto pc+4")
	248	__msg("3: .12....... (07) r1 += 1")
	249	__msg("4: .12....... (27) r2 *= 2")
	250	__msg("5: .12....... (05) goto pc+0")
	251	__msg("6: .12....... (05) goto pc-5")
	252	__msg("7: .......... (b7) r0 = 0")
	253	__msg("8: 0......... (95) exit")
	254	__naked void loop(void)
	255	{
	256	asm volatile (
	257	"r1 = 0;"
	258	"r2 = 7;"
	259	"if r1 > 0x7 goto +4;"
	260	"r1 += 1;"
	261	"r2 *= 2;"
	262	"goto +0;"
	263	"goto -5;"
	264	"r0 = 0;"
	265	"exit;"
	266	:
	267	: __imm(bpf_trace_printk)
	268	: __clobber_all);
	269	}
	270
	271	#ifdef CAN_USE_GOTOL
	272	SEC("socket")
	273	__log_level(2)
	274	__msg("2: .123...... (25) if r1 > 0x7 goto pc+2")
	275	__msg("3: ..2....... (bf) r0 = r2")
	276	__msg("4: 0......... (06) gotol pc+1")
	277	__msg("5: ...3...... (bf) r0 = r3")
	278	__msg("6: 0......... (95) exit")
	279	__naked void gotol(void)
	280	{
	281	asm volatile (
	282	"r2 = 42;"
	283	"r3 = 24;"
	284	"if r1 > 0x7 goto +2;"
	285	"r0 = r2;"
	286	"gotol +1;"
	287	"r0 = r3;"
	288	"exit;"
	289	:
	290	: __imm(bpf_trace_printk)
	291	: __clobber_all);
	292	}
	293	#endif
	294
	295	SEC("socket")
	296	__log_level(2)
	297	__msg("0: .......... (b7) r1 = 1")
	298	__msg("1: .1........ (e5) may_goto pc+1")
	299	__msg("2: .......... (05) goto pc-3")
	300	__msg("3: .1........ (bf) r0 = r1")
	301	__msg("4: 0......... (95) exit")
	302	__naked void may_goto(void)
	303	{
	304	asm volatile (
	305	"1: r1 = 1;"
	306	".8byte %[may_goto];"
	307	"goto 1b;"
	308	"r0 = r1;"
	309	"exit;"
	310	:
	311	: __imm(bpf_get_smp_processor_id),
	312	__imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, +1 /* offset */, 0))
	313	: __clobber_all);
	314	}
	315
	316	SEC("socket")
	317	__log_level(2)
	318	__msg("1: 0......... (18) r2 = 0x7")
	319	__msg("3: 0.2....... (0f) r0 += r2")
	320	__naked void ldimm64(void)
	321	{
	322	asm volatile (
	323	"r0 = 0;"
	324	"r2 = 0x7 ll;"
	325	"r0 += r2;"
	326	"exit;"
	327	:
	328	:: __clobber_all);
	329	}
	330
	331	/* No rules specific for LD_ABS/LD_IND, default behaviour kicks in */
	332	SEC("socket")
	333	__log_level(2)
	334	__msg("2: 0123456789 (30) r0 = (u8 )skb[42]")
	335	__msg("3: 012.456789 (0f) r7 += r0")
	336	__msg("4: 012.456789 (b7) r3 = 42")
	337	__msg("5: 0123456789 (50) r0 = (u8 )skb[r3 + 0]")
	338	__msg("6: 0......7.. (0f) r7 += r0")
	339	__naked void ldabs(void)
	340	{
	341	asm volatile (
	342	"r6 = r1;"
	343	"r7 = 0;"
	344	"r0 = (u8 )skb[42];"
	345	"r7 += r0;"
	346	"r3 = 42;"
	347	".8byte %[ld_ind];" /* same as "r0 = (u8 )skb[r3];" */
	348	"r7 += r0;"
	349	"r0 = r7;"
	350	"exit;"
	351	:
	352	: __imm_insn(ld_ind, BPF_LD_IND(BPF_B, BPF_REG_3, 0))
	353	: __clobber_all);
	354	}
	355
	356
	357	#ifdef __BPF_FEATURE_ADDR_SPACE_CAST
	358	SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
	359	__log_level(2)
	360	__msg(" 6: .12345.... (85) call bpf_arena_alloc_pages")
	361	__msg(" 7: 0......... (bf) r1 = addr_space_cast(r0, 0, 1)")
	362	__msg(" 8: .1........ (b7) r2 = 42")
	363	__naked void addr_space_cast(void)
	364	{
	365	asm volatile (
	366	"r1 = %[arena] ll;"
	367	"r2 = 0;"
	368	"r3 = 1;"
	369	"r4 = 0;"
	370	"r5 = 0;"
	371	"call %[bpf_arena_alloc_pages];"
	372	"r1 = addr_space_cast(r0, 0, 1);"
	373	"r2 = 42;"
	374	"(u64 )(r1 +0) = r2;"
	375	"r0 = 0;"
	376	"exit;"
	377	:
	378	: __imm(bpf_arena_alloc_pages),
	379	__imm_addr(arena)
	380	: __clobber_all);
	381	}
	382	#endif
	383
	384	static __used __naked int aux1(void)
	385	{
	386	asm volatile (
	387	"r0 = r1;"
	388	"r0 += r2;"
	389	"exit;"
	390	::: __clobber_all);
	391	}
	392
	393	SEC("socket")
	394	__log_level(2)
	395	__msg("0: ....45.... (b7) r1 = 1")
	396	__msg("1: .1..45.... (b7) r2 = 2")
	397	__msg("2: .12.45.... (b7) r3 = 3")
	398	/* Conservative liveness for subprog parameters. */
	399	__msg("3: .12345.... (85) call pc+2")
	400	__msg("4: .......... (b7) r0 = 0")
	401	__msg("5: 0......... (95) exit")
	402	__msg("6: .12....... (bf) r0 = r1")
	403	__msg("7: 0.2....... (0f) r0 += r2")
	404	/* Conservative liveness for subprog return value. */
	405	__msg("8: 0......... (95) exit")
	406	__naked void subprog1(void)
	407	{
	408	asm volatile (
	409	"r1 = 1;"
	410	"r2 = 2;"
	411	"r3 = 3;"
	412	"call aux1;"
	413	"r0 = 0;"
	414	"exit;"
	415	::: __clobber_all);
	416	}
	417
	418	/* to retain debug info for BTF generation */
	419	void kfunc_root(void)
	420	{
	421	bpf_arena_alloc_pages(0, 0, 0, 0, 0);
	422	}
	423
	424	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/connect4_dropper.c+3−1

 #define VERDICT_REJECT	0
 #define VERDICT_PROCEED	1
+int port;
 SEC("cgroup/connect4")
 int connect_v4_dropper(struct bpf_sock_addr *ctx)
 {
 	if (ctx->type != SOCK_STREAM)
 		return VERDICT_PROCEED;
-	if (ctx->user_port == bpf_htons(60120))
+	if (ctx->user_port == bpf_htons(port))
 		return VERDICT_REJECT;
 	return VERDICT_PROCEED;
 }

tools/testing/selftests/bpf/progs/core_reloc_types.h+10−0

  */
 struct core_reloc_arrays_output {
 	int a2;
+	int a3;
 	char b123;
 	int c1c;
 	int d00d;
 	struct core_reloc_arrays_substruct d[1][2];
 };
+struct core_reloc_arrays___err_bad_signed_arr_elem_sz {
+	/* int -> short (signed!): not supported case */
+	short a[5];
+	char b[2][3][4];
+	struct core_reloc_arrays_substruct c[3];
+	struct core_reloc_arrays_substruct d[1][2];
+	struct core_reloc_arrays_substruct f[][2];
+};
 /*
  * PRIMITIVES
  */

tools/testing/selftests/bpf/progs/cpumask_common.h+1−0

@@ -61,6 +61,7 @@ u32 bpf_cpumask_any_distribute(const struct cpumask *src) __ksym __weak; · 60 unmodified lines
61	u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,	61	u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
62	const struct cpumask *src2) __ksym __weak;	62	const struct cpumask *src2) __ksym __weak;
63	u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak;	63	u32 bpf_cpumask_weight(const struct cpumask *cpumask) __ksym __weak;
		64	int bpf_cpumask_populate(struct cpumask cpumask, void src, size_t src__sz) __ksym __weak;
64		65
65	void bpf_rcu_read_lock(void) __ksym __weak;	66	void bpf_rcu_read_lock(void) __ksym __weak;
66	void bpf_rcu_read_unlock(void) __ksym __weak;	67	void bpf_rcu_read_unlock(void) __ksym __weak;

tools/testing/selftests/bpf/progs/cpumask_failure.c+38−0

@@ -222,3 +222,41 @@ int BPF_PROG(test_invalid_nested_array, struct task_struct *task, u64 clone_flag · 221 unmodified lines
222		222
223	return 0;	223	return 0;
224	}	224	}
		225
		226	SEC("tp_btf/task_newtask")
		227	__failure __msg("type=scalar expected=fp")
		228	int BPF_PROG(test_populate_invalid_destination, struct task_struct *task, u64 clone_flags)
		229	{
		230	struct bpf_cpumask invalid = (struct bpf_cpumask )0x123456;
		231	u64 bits;
		232	int ret;
		233
		234	ret = bpf_cpumask_populate((struct cpumask *)invalid, &bits, sizeof(bits));
		235	if (!ret)
		236	err = 2;
		237
		238	return 0;
		239	}
		240
		241	SEC("tp_btf/task_newtask")
		242	__failure __msg("leads to invalid memory access")
		243	int BPF_PROG(test_populate_invalid_source, struct task_struct *task, u64 clone_flags)
		244	{
		245	void garbage = (void )0x123456;
		246	struct bpf_cpumask *local;
		247	int ret;
		248
		249	local = create_cpumask();
		250	if (!local) {
		251	err = 1;
		252	return 0;
		253	}
		254
		255	ret = bpf_cpumask_populate((struct cpumask *)local, garbage, 8);
		256	if (!ret)
		257	err = 2;
		258
		259	bpf_cpumask_release(local);
		260
		261	return 0;
		262	}

tools/testing/selftests/bpf/progs/cpumask_success.c+119−1

@@ -749,7 +749,6 @@ out: · 748 unmodified lines
749	}	749	}
750		750
751	SEC("tp_btf/task_newtask")	751	SEC("tp_btf/task_newtask")
752	__success
753	int BPF_PROG(test_refcount_null_tracking, struct task_struct *task, u64 clone_flags)	752	int BPF_PROG(test_refcount_null_tracking, struct task_struct *task, u64 clone_flags)
754	{	753	{
755	struct bpf_cpumask mask1, mask2;	754	struct bpf_cpumask mask1, mask2;
@@ -770,3 +769,122 @@ free_masks_return: · 14 unmodified lines
770	bpf_cpumask_release(mask2);	769	bpf_cpumask_release(mask2);
771	return 0;	770	return 0;
772	}	771	}
		772
		773	SEC("tp_btf/task_newtask")
		774	int BPF_PROG(test_populate_reject_small_mask, struct task_struct *task, u64 clone_flags)
		775	{
		776	struct bpf_cpumask *local;
		777	u8 toofewbits;
		778	int ret;
		779
		780	if (!is_test_task())
		781	return 0;
		782
		783	local = create_cpumask();
		784	if (!local)
		785	return 0;
		786
		787	/* The kfunc should prevent this operation */
		788	ret = bpf_cpumask_populate((struct cpumask *)local, &toofewbits, sizeof(toofewbits));
		789	if (ret != -EACCES)
		790	err = 2;
		791
		792	bpf_cpumask_release(local);
		793
		794	return 0;
		795	}
		796
		797	/* Mask is guaranteed to be large enough for bpf_cpumask_t. */
		798	#define CPUMASK_TEST_MASKLEN (sizeof(cpumask_t))
		799
		800	/* Add an extra word for the test_populate_reject_unaligned test. */
		801	u64 bits[CPUMASK_TEST_MASKLEN / 8 + 1];
		802	extern bool CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS __kconfig __weak;
		803
		804	SEC("tp_btf/task_newtask")
		805	int BPF_PROG(test_populate_reject_unaligned, struct task_struct *task, u64 clone_flags)
		806	{
		807	struct bpf_cpumask *mask;
		808	char *src;
		809	int ret;
		810
		811	if (!is_test_task())
		812	return 0;
		813
		814	/* Skip if unaligned accesses are fine for this arch. */
		815	if (CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)
		816	return 0;
		817
		818	mask = bpf_cpumask_create();
		819	if (!mask) {
		820	err = 1;
		821	return 0;
		822	}
		823
		824	/* Misalign the source array by a byte. */
		825	src = &((char *)bits)[1];
		826
		827	ret = bpf_cpumask_populate((struct cpumask *)mask, src, CPUMASK_TEST_MASKLEN);
		828	if (ret != -EINVAL)
		829	err = 2;
		830
		831	bpf_cpumask_release(mask);
		832
		833	return 0;
		834	}
		835
		836
		837	SEC("tp_btf/task_newtask")
		838	int BPF_PROG(test_populate, struct task_struct *task, u64 clone_flags)
		839	{
		840	struct bpf_cpumask *mask;
		841	bool bit;
		842	int ret;
		843	int i;
		844
		845	if (!is_test_task())
		846	return 0;
		847
		848	/* Set only odd bits. */
		849	__builtin_memset(bits, 0xaa, CPUMASK_TEST_MASKLEN);
		850
		851	mask = bpf_cpumask_create();
		852	if (!mask) {
		853	err = 1;
		854	return 0;
		855	}
		856
		857	/* Pass the entire bits array, the kfunc will only copy the valid bits. */
		858	ret = bpf_cpumask_populate((struct cpumask *)mask, bits, CPUMASK_TEST_MASKLEN);
		859	if (ret) {
		860	err = 2;
		861	goto out;
		862	}
		863
		864	/*
		865	* Test is there to appease the verifier. We cannot directly
		866	* access NR_CPUS, the upper bound for nr_cpus, so we infer
		867	* it from the size of cpumask_t.
		868	*/
		869	if (nr_cpus < 0 \|\| nr_cpus >= CPUMASK_TEST_MASKLEN * 8) {
		870	err = 3;
		871	goto out;
		872	}
		873
		874	bpf_for(i, 0, nr_cpus) {
		875	/* Odd-numbered bits should be set, even ones unset. */
		876	bit = bpf_cpumask_test_cpu(i, (const struct cpumask *)mask);
		877	if (bit == (i % 2 != 0))
		878	continue;
		879
		880	err = 4;
		881	break;
		882	}
		883
		884	out:
		885	bpf_cpumask_release(mask);
		886
		887	return 0;
		888	}
		889
		890	#undef CPUMASK_TEST_MASKLEN

tools/testing/selftests/bpf/progs/dynptr_success.c+118−5

@@ -1,20 +1,19 @@
1	// SPDX-License-Identifier: GPL-2.0	1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (c) 2022 Facebook */	2	/* Copyright (c) 2022 Facebook */
3		3
		4	#include <vmlinux.h>
4	#include <string.h>	5	#include <string.h>
5	#include <stdbool.h>	6	#include <stdbool.h>
6	#include <linux/bpf.h>
7	#include <bpf/bpf_helpers.h>	7	#include <bpf/bpf_helpers.h>
8	#include <bpf/bpf_tracing.h>	8	#include <bpf/bpf_tracing.h>
9	#include "bpf_misc.h"	9	#include "bpf_misc.h"
10	#include "bpf_kfuncs.h"
11	#include "errno.h"	10	#include "errno.h"
12		11
13	char _license[] SEC("license") = "GPL";	12	char _license[] SEC("license") = "GPL";
14		13
15	int pid, err, val;	14	int pid, err, val;
16		15
17	struct sample {	16	struct ringbuf_sample {
18	int pid;	17	int pid;
19	int seq;	18	int seq;
20	long value;	19	long value;
@@ -121,7 +120,7 @@ int test_dynptr_data(void *ctx) · 100 unmodified lines
121		120
122	static int ringbuf_callback(__u32 index, void *data)	121	static int ringbuf_callback(__u32 index, void *data)
123	{	122	{
124	struct sample *sample;	123	struct ringbuf_sample *sample;
125		124
126	struct bpf_dynptr ptr = (struct bpf_dynptr )data;	125	struct bpf_dynptr ptr = (struct bpf_dynptr )data;
127		126
@@ -138,7 +137,7 @@ SEC("?tp/syscalls/sys_enter_nanosleep") · 10 unmodified lines
138	int test_ringbuf(void *ctx)	137	int test_ringbuf(void *ctx)
139	{	138	{
140	struct bpf_dynptr ptr;	139	struct bpf_dynptr ptr;
141	struct sample *sample;	140	struct ringbuf_sample *sample;
142		141
143	if (bpf_get_current_pid_tgid() >> 32 != pid)	142	if (bpf_get_current_pid_tgid() >> 32 != pid)
144	return 0;	143	return 0;
@@ -567,3 +566,117 @@ int BPF_PROG(test_dynptr_skb_tp_btf, void skb, void location) · 422 unmodified lines
567		566
568	return 1;	567	return 1;
569	}	568	}
		569
		570	static inline int bpf_memcmp(const char a, const char b, u32 size)
		571	{
		572	int i;
		573
		574	bpf_for(i, 0, size) {
		575	if (a[i] != b[i])
		576	return a[i] < b[i] ? -1 : 1;
		577	}
		578	return 0;
		579	}
		580
		581	SEC("?tp/syscalls/sys_enter_nanosleep")
		582	int test_dynptr_copy(void *ctx)
		583	{
		584	char data[] = "hello there, world!!";
		585	char buf[32] = {'\0'};
		586	__u32 sz = sizeof(data);
		587	struct bpf_dynptr src, dst;
		588
		589	bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &src);
		590	bpf_ringbuf_reserve_dynptr(&ringbuf, sz, 0, &dst);
		591
		592	/* Test basic case of copying contiguous memory backed dynptrs */
		593	err = bpf_dynptr_write(&src, 0, data, sz, 0);
		594	err = err ?: bpf_dynptr_copy(&dst, 0, &src, 0, sz);
		595	err = err ?: bpf_dynptr_read(buf, sz, &dst, 0, 0);
		596	err = err ?: bpf_memcmp(data, buf, sz);
		597
		598	/* Test that offsets are handled correctly */
		599	err = err ?: bpf_dynptr_copy(&dst, 3, &src, 5, sz - 5);
		600	err = err ?: bpf_dynptr_read(buf, sz - 5, &dst, 3, 0);
		601	err = err ?: bpf_memcmp(data + 5, buf, sz - 5);
		602
		603	bpf_ringbuf_discard_dynptr(&src, 0);
		604	bpf_ringbuf_discard_dynptr(&dst, 0);
		605	return 0;
		606	}
		607
		608	SEC("xdp")
		609	int test_dynptr_copy_xdp(struct xdp_md *xdp)
		610	{
		611	struct bpf_dynptr ptr_buf, ptr_xdp;
		612	char data[] = "qwertyuiopasdfghjkl";
		613	char buf[32] = {'\0'};
		614	__u32 len = sizeof(data);
		615	int i, chunks = 200;
		616
		617	/* ptr_xdp is backed by non-contiguous memory */
		618	bpf_dynptr_from_xdp(xdp, 0, &ptr_xdp);
		619	bpf_ringbuf_reserve_dynptr(&ringbuf, len * chunks, 0, &ptr_buf);
		620
		621	/* Destination dynptr is backed by non-contiguous memory */
		622	bpf_for(i, 0, chunks) {
		623	err = bpf_dynptr_write(&ptr_buf, i * len, data, len, 0);
		624	if (err)
		625	goto out;
		626	}
		627
		628	err = bpf_dynptr_copy(&ptr_xdp, 0, &ptr_buf, 0, len * chunks);
		629	if (err)
		630	goto out;
		631
		632	bpf_for(i, 0, chunks) {
		633	__builtin_memset(buf, 0, sizeof(buf));
		634	err = bpf_dynptr_read(&buf, len, &ptr_xdp, i * len, 0);
		635	if (err)
		636	goto out;
		637	if (bpf_memcmp(data, buf, len) != 0)
		638	goto out;
		639	}
		640
		641	/* Source dynptr is backed by non-contiguous memory */
		642	__builtin_memset(buf, 0, sizeof(buf));
		643	bpf_for(i, 0, chunks) {
		644	err = bpf_dynptr_write(&ptr_buf, i * len, buf, len, 0);
		645	if (err)
		646	goto out;
		647	}
		648
		649	err = bpf_dynptr_copy(&ptr_buf, 0, &ptr_xdp, 0, len * chunks);
		650	if (err)
		651	goto out;
		652
		653	bpf_for(i, 0, chunks) {
		654	__builtin_memset(buf, 0, sizeof(buf));
		655	err = bpf_dynptr_read(&buf, len, &ptr_buf, i * len, 0);
		656	if (err)
		657	goto out;
		658	if (bpf_memcmp(data, buf, len) != 0)
		659	goto out;
		660	}
		661
		662	/* Both source and destination dynptrs are backed by non-contiguous memory */
		663	err = bpf_dynptr_copy(&ptr_xdp, 2, &ptr_xdp, len, len * (chunks - 1));
		664	if (err)
		665	goto out;
		666
		667	bpf_for(i, 0, chunks - 1) {
		668	__builtin_memset(buf, 0, sizeof(buf));
		669	err = bpf_dynptr_read(&buf, len, &ptr_xdp, 2 + i * len, 0);
		670	if (err)
		671	goto out;
		672	if (bpf_memcmp(data, buf, len) != 0)
		673	goto out;
		674	}
		675
		676	if (bpf_dynptr_copy(&ptr_xdp, 2000, &ptr_xdp, 0, len * chunks) != -E2BIG)
		677	err = 1;
		678
		679	out:
		680	bpf_ringbuf_discard_dynptr(&ptr_buf, 0);
		681	return XDP_DROP;
		682	}

tools/testing/selftests/bpf/progs/fexit_noreturns.c+15−0

@@ -0,0 +1,15 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include <bpf/bpf_tracing.h>
	6	#include "bpf_misc.h"
	7
	8	char _license[] SEC("license") = "GPL";
	9
	10	SEC("fexit/do_exit")
	11	__failure __msg("Attaching fexit/fmod_ret to __noreturn functions is rejected.")
	12	int BPF_PROG(noreturns)
	13	{
	14	return 0;
	15	}

tools/testing/selftests/bpf/progs/irq.c+70−1

@@ -222,7 +222,7 @@ int __noinline global_local_irq_balance(void) · 221 unmodified lines
222	}	222	}
223		223
224	SEC("?tc")	224	SEC("?tc")
225	__failure __msg("global function calls are not allowed with IRQs disabled")	225	__success
226	int irq_global_subprog(struct __sk_buff *ctx)	226	int irq_global_subprog(struct __sk_buff *ctx)
227	{	227	{
228	unsigned long flags;	228	unsigned long flags;
@@ -441,4 +441,73 @@ int irq_ooo_refs_array(struct __sk_buff *ctx) · 212 unmodified lines
441	return 0;	441	return 0;
442	}	442	}
443		443
		444	int __noinline
		445	global_subprog(int i)
		446	{
		447	if (i)
		448	bpf_printk("%p", &i);
		449	return i;
		450	}
		451
		452	int __noinline
		453	global_sleepable_helper_subprog(int i)
		454	{
		455	if (i)
		456	bpf_copy_from_user(&i, sizeof(i), NULL);
		457	return i;
		458	}
		459
		460	int __noinline
		461	global_sleepable_kfunc_subprog(int i)
		462	{
		463	if (i)
		464	bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
		465	global_subprog(i);
		466	return i;
		467	}
		468
		469	int __noinline
		470	global_subprog_calling_sleepable_global(int i)
		471	{
		472	if (!i)
		473	global_sleepable_kfunc_subprog(i);
		474	return i;
		475	}
		476
		477	SEC("?syscall")
		478	__success
		479	int irq_non_sleepable_global_subprog(void *ctx)
		480	{
		481	unsigned long flags;
		482
		483	bpf_local_irq_save(&flags);
		484	global_subprog(0);
		485	bpf_local_irq_restore(&flags);
		486	return 0;
		487	}
		488
		489	SEC("?syscall")
		490	__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
		491	int irq_sleepable_helper_global_subprog(void *ctx)
		492	{
		493	unsigned long flags;
		494
		495	bpf_local_irq_save(&flags);
		496	global_sleepable_helper_subprog(0);
		497	bpf_local_irq_restore(&flags);
		498	return 0;
		499	}
		500
		501	SEC("?syscall")
		502	__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
		503	int irq_sleepable_global_subprog_indirect(void *ctx)
		504	{
		505	unsigned long flags;
		506
		507	bpf_local_irq_save(&flags);
		508	global_subprog_calling_sleepable_global(0);
		509	bpf_local_irq_restore(&flags);
		510	return 0;
		511	}
		512
444	char _license[] SEC("license") = "GPL";	513	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/iters.c+139−0

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	#include "bpf_misc.h"	7	#include "bpf_misc.h"
8	#include "bpf_compiler.h"	8	#include "bpf_compiler.h"
9		9
		10	#define unlikely(x) __builtin_expect(!!(x), 0)
		11
10	static volatile int zero = 0;	12	static volatile int zero = 0;
11		13
12	int my_pid;	14	int my_pid;
@@ -1174,6 +1176,122 @@ __naked int loop_state_deps2(void) · 1161 unmodified lines
1174	);	1176	);
1175	}	1177	}
1176		1178
		1179	SEC("?raw_tp")
		1180	__failure
		1181	__msg("math between fp pointer and register with unbounded")
		1182	__flag(BPF_F_TEST_STATE_FREQ)
		1183	__naked int loop_state_deps3(void)
		1184	{
		1185	/* This is equivalent to a C program below.
		1186	*
		1187	* if (random() != 24) { // assume false branch is placed first
		1188	* i = iter_new(); // fp[-8]
		1189	* while (iter_next(i));
		1190	* iter_destroy(i);
		1191	* return;
		1192	* }
		1193	*
		1194	* for (i = 10; i > 0; i--); // increase dfs_depth for child states
		1195	*
		1196	* i = iter_new(); // fp[-8]
		1197	* b = -24; // r8
		1198	* for (;;) { // checkpoint (L)
		1199	* if (iter_next(i)) // checkpoint (N)
		1200	* break;
		1201	* if (random() == 77) { // assume false branch is placed first
		1202	* (u64 )(r10 + b) = 7; // this is not safe when b == -25
		1203	* iter_destroy(i);
		1204	* return;
		1205	* }
		1206	* if (random() == 42) { // assume false branch is placed first
		1207	* b = -25;
		1208	* }
		1209	* }
		1210	* iter_destroy(i);
		1211	*
		1212	* In case of a buggy verifier first loop might poison
		1213	* env->cur_state->loop_entry with a state having 0 branches
		1214	* and small dfs_depth. This would trigger NOT_EXACT states
		1215	* comparison for some states within second loop.
		1216	* Specifically, checkpoint (L) might be problematic if:
		1217	* - branch with '(u64 )(r10 + b) = 7' is not explored yet;
		1218	* - checkpoint (L) is first reached in state {b=-24};
		1219	* - traversal is pruned at checkpoint (N) setting checkpoint's (L)
		1220	* branch count to 0, thus making it eligible for use in pruning;
		1221	* - checkpoint (L) is next reached in state {b=-25},
		1222	* this would cause NOT_EXACT comparison with a state {b=-24}
		1223	* while 'b' is not marked precise yet.
		1224	*/
		1225	asm volatile (
		1226	"call %[bpf_get_prandom_u32];"
		1227	"if r0 == 24 goto 2f;"
		1228	"r1 = r10;"
		1229	"r1 += -8;"
		1230	"r2 = 0;"
		1231	"r3 = 5;"
		1232	"call %[bpf_iter_num_new];"
		1233	"1:"
		1234	"r1 = r10;"
		1235	"r1 += -8;"
		1236	"call %[bpf_iter_num_next];"
		1237	"if r0 != 0 goto 1b;"
		1238	"r1 = r10;"
		1239	"r1 += -8;"
		1240	"call %[bpf_iter_num_destroy];"
		1241	"r0 = 0;"
		1242	"exit;"
		1243	"2:"
		1244	/* loop to increase dfs_depth */
		1245	"r0 = 10;"
		1246	"3:"
		1247	"r0 -= 1;"
		1248	"if r0 != 0 goto 3b;"
		1249	/* end of loop */
		1250	"r1 = r10;"
		1251	"r1 += -8;"
		1252	"r2 = 0;"
		1253	"r3 = 10;"
		1254	"call %[bpf_iter_num_new];"
		1255	"r8 = -24;"
		1256	"main_loop_%=:"
		1257	"r1 = r10;"
		1258	"r1 += -8;"
		1259	"call %[bpf_iter_num_next];"
		1260	"if r0 == 0 goto main_loop_end_%=;"
		1261	/* first if */
		1262	"call %[bpf_get_prandom_u32];"
		1263	"if r0 == 77 goto unsafe_write_%=;"
		1264	/* second if */
		1265	"call %[bpf_get_prandom_u32];"
		1266	"if r0 == 42 goto poison_r8_%=;"
		1267	/* iterate */
		1268	"goto main_loop_%=;"
		1269	"main_loop_end_%=:"
		1270	"r1 = r10;"
		1271	"r1 += -8;"
		1272	"call %[bpf_iter_num_destroy];"
		1273	"r0 = 0;"
		1274	"exit;"
		1275
		1276	"unsafe_write_%=:"
		1277	"r0 = r10;"
		1278	"r0 += r8;"
		1279	"r1 = 7;"
		1280	"(u64 )(r0 + 0) = r1;"
		1281	"goto main_loop_end_%=;"
		1282
		1283	"poison_r8_%=:"
		1284	"r8 = -25;"
		1285	"goto main_loop_%=;"
		1286	:
		1287	: __imm(bpf_get_prandom_u32),
		1288	__imm(bpf_iter_num_new),
		1289	__imm(bpf_iter_num_next),
		1290	__imm(bpf_iter_num_destroy)
		1291	: __clobber_all
		1292	);
		1293	}
		1294
1177	SEC("?raw_tp")	1295	SEC("?raw_tp")
1178	__success	1296	__success
1179	__naked int triple_continue(void)	1297	__naked int triple_continue(void)
@@ -1512,4 +1630,25 @@ int iter_destroy_bad_arg(const void *ctx) · 332 unmodified lines
1512	return 0;	1630	return 0;
1513	}	1631	}
1514		1632
		1633	SEC("raw_tp")
		1634	__success
		1635	int clean_live_states(const void *ctx)
		1636	{
		1637	char buf[1];
		1638	int i, j, k, l, m, n, o;
		1639
		1640	bpf_for(i, 0, 10)
		1641	bpf_for(j, 0, 10)
		1642	bpf_for(k, 0, 10)
		1643	bpf_for(l, 0, 10)
		1644	bpf_for(m, 0, 10)
		1645	bpf_for(n, 0, 10)
		1646	bpf_for(o, 0, 10) {
		1647	if (unlikely(bpf_get_prandom_u32()))
		1648	buf[0] = 42;
		1649	bpf_printk("%s", buf);
		1650	}
		1651	return 0;
		1652	}
		1653
1515	char _license[] SEC("license") = "GPL";	1654	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/netns_cookie_prog.c+9−0

 } sock_map SEC(".maps");
 int tcx_init_netns_cookie, tcx_netns_cookie;
+int cgroup_skb_init_netns_cookie, cgroup_skb_netns_cookie;
 SEC("sockops")
 int get_netns_cookie_sockops(struct bpf_sock_ops *ctx)
 	return TCX_PASS;
 }
+SEC("cgroup_skb/ingress")
+int get_netns_cookie_cgroup_skb(struct __sk_buff *skb)
+{
+	cgroup_skb_init_netns_cookie = bpf_get_netns_cookie(NULL);
+	cgroup_skb_netns_cookie = bpf_get_netns_cookie(skb);
+	return SK_PASS;
+}
 char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/preempt_lock.c+67−1

@@ -134,7 +134,7 @@ int __noinline preempt_global_subprog(void) · 133 unmodified lines
134	}	134	}
135		135
136	SEC("?tc")	136	SEC("?tc")
137	__failure __msg("global function calls are not allowed with preemption disabled")	137	__success
138	int preempt_global_subprog_test(struct __sk_buff *ctx)	138	int preempt_global_subprog_test(struct __sk_buff *ctx)
139	{	139	{
140	preempt_disable();	140	preempt_disable();
@@ -143,4 +143,70 @@ int preempt_global_subprog_test(struct __sk_buff *ctx) · 2 unmodified lines
143	return 0;	143	return 0;
144	}	144	}
145		145
		146	int __noinline
		147	global_subprog(int i)
		148	{
		149	if (i)
		150	bpf_printk("%p", &i);
		151	return i;
		152	}
		153
		154	int __noinline
		155	global_sleepable_helper_subprog(int i)
		156	{
		157	if (i)
		158	bpf_copy_from_user(&i, sizeof(i), NULL);
		159	return i;
		160	}
		161
		162	int __noinline
		163	global_sleepable_kfunc_subprog(int i)
		164	{
		165	if (i)
		166	bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
		167	global_subprog(i);
		168	return i;
		169	}
		170
		171	int __noinline
		172	global_subprog_calling_sleepable_global(int i)
		173	{
		174	if (!i)
		175	global_sleepable_kfunc_subprog(i);
		176	return i;
		177	}
		178
		179	SEC("?syscall")
		180	__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
		181	int preempt_global_sleepable_helper_subprog(struct __sk_buff *ctx)
		182	{
		183	preempt_disable();
		184	if (ctx->mark)
		185	global_sleepable_helper_subprog(ctx->mark);
		186	preempt_enable();
		187	return 0;
		188	}
		189
		190	SEC("?syscall")
		191	__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
		192	int preempt_global_sleepable_kfunc_subprog(struct __sk_buff *ctx)
		193	{
		194	preempt_disable();
		195	if (ctx->mark)
		196	global_sleepable_kfunc_subprog(ctx->mark);
		197	preempt_enable();
		198	return 0;
		199	}
		200
		201	SEC("?syscall")
		202	__failure __msg("global functions that may sleep are not allowed in non-sleepable context")
		203	int preempt_global_sleepable_subprog_indirect(struct __sk_buff *ctx)
		204	{
		205	preempt_disable();
		206	if (ctx->mark)
		207	global_subprog_calling_sleepable_global(ctx->mark);
		208	preempt_enable();
		209	return 0;
		210	}
		211
146	char _license[] SEC("license") = "GPL";	212	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/prepare.c+28−0

@@ -0,0 +1,28 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta */
	3	#include <vmlinux.h>
	4	#include <bpf/bpf_helpers.h>
	5	//#include <bpf/bpf_tracing.h>
	6
	7	char _license[] SEC("license") = "GPL";
	8
	9	int err;
	10
	11	struct {
	12	__uint(type, BPF_MAP_TYPE_RINGBUF);
	13	__uint(max_entries, 4096);
	14	} ringbuf SEC(".maps");
	15
	16	struct {
	17	__uint(type, BPF_MAP_TYPE_ARRAY);
	18	__uint(max_entries, 1);
	19	__type(key, __u32);
	20	__type(value, __u32);
	21	} array_map SEC(".maps");
	22
	23	SEC("cgroup_skb/egress")
	24	int program(struct __sk_buff *skb)
	25	{
	26	err = 0;
	27	return 0;
	28	}

tools/testing/selftests/bpf/progs/priv_freplace_prog.c+13−0

@@ -0,0 +1,13 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3
	4	#include "vmlinux.h"
	5	#include <bpf/bpf_helpers.h>
	6
	7	char _license[] SEC("license") = "GPL";
	8
	9	SEC("freplace/xdp_prog1")
	10	int new_xdp_prog2(struct xdp_md *xd)
	11	{
	12	return XDP_DROP;
	13	}

tools/testing/selftests/bpf/progs/priv_prog.c+3−3

@@ -6,8 +6,8 @@ · 5 unmodified lines
6		6
7	char _license[] SEC("license") = "GPL";	7	char _license[] SEC("license") = "GPL";
8		8
9	SEC("kprobe")	9	SEC("xdp")
10	int kprobe_prog(void *ctx)	10	int xdp_prog1(struct xdp_md *xdp)
11	{	11	{
12	return 1;	12	return XDP_DROP;
13	}	13	}

tools/testing/selftests/bpf/progs/pro_epilogue_with_kfunc.c+88−0

@@ -0,0 +1,88 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3
	4	#include <vmlinux.h>
	5	#include <bpf/bpf_tracing.h>
	6	#include "bpf_misc.h"
	7	#include "../test_kmods/bpf_testmod.h"
	8	#include "../test_kmods/bpf_testmod_kfunc.h"
	9
	10	char _license[] SEC("license") = "GPL";
	11
	12	void __kfunc_btf_root(void)
	13	{
	14	bpf_kfunc_st_ops_inc10(NULL);
	15	}
	16
	17	static __noinline __used int subprog(struct st_ops_args *args)
	18	{
	19	args->a += 1;
	20	return args->a;
	21	}
	22
	23	__success
	24	/* prologue */
	25	__xlated("0: r8 = r1")
	26	__xlated("1: r1 = 0")
	27	__xlated("2: call kernel-function")
	28	__xlated("3: if r0 != 0x0 goto pc+5")
	29	__xlated("4: r6 = (u64 )(r8 +0)")
	30	__xlated("5: r7 = (u64 )(r6 +0)")
	31	__xlated("6: r7 += 1000")
	32	__xlated("7: (u64 )(r6 +0) = r7")
	33	__xlated("8: goto pc+2")
	34	__xlated("9: r1 = r0")
	35	__xlated("10: call kernel-function")
	36	__xlated("11: r1 = r8")
	37	/* save __u64 ctx to stack /
	38	__xlated("12: (u64 )(r10 -8) = r1")
	39	/* main prog */
	40	__xlated("13: r1 = (u64 )(r1 +0)")
	41	__xlated("14: r6 = r1")
	42	__xlated("15: call kernel-function")
	43	__xlated("16: r1 = r6")
	44	__xlated("17: call pc+")
	45	/* epilogue */
	46	__xlated("18: r1 = 0")
	47	__xlated("19: r6 = 0")
	48	__xlated("20: call kernel-function")
	49	__xlated("21: if r0 != 0x0 goto pc+6")
	50	__xlated("22: r1 = (u64 )(r10 -8)")
	51	__xlated("23: r1 = (u64 )(r1 +0)")
	52	__xlated("24: r6 = (u64 )(r1 +0)")
	53	__xlated("25: r6 += 10000")
	54	__xlated("26: (u64 )(r1 +0) = r6")
	55	__xlated("27: goto pc+2")
	56	__xlated("28: r1 = r0")
	57	__xlated("29: call kernel-function")
	58	__xlated("30: r0 = r6")
	59	__xlated("31: r0 *= 2")
	60	__xlated("32: exit")
	61	SEC("struct_ops/test_pro_epilogue")
	62	__naked int test_kfunc_pro_epilogue(void)
	63	{
	64	asm volatile (
	65	"r1 = (u64 )(r1 +0);"
	66	"r6 = r1;"
	67	"call %[bpf_kfunc_st_ops_inc10];"
	68	"r1 = r6;"
	69	"call subprog;"
	70	"exit;"
	71	:
	72	: __imm(bpf_kfunc_st_ops_inc10)
	73	: __clobber_all);
	74	}
	75
	76	SEC("syscall")
	77	__retval(22022) /* (PROLOGUE_A [1000] + KFUNC_INC10 + SUBPROG_A [1] + EPILOGUE_A [10000]) * 2 */
	78	int syscall_pro_epilogue(void *ctx)
	79	{
	80	struct st_ops_args args = {};
	81
	82	return bpf_kfunc_st_ops_test_pro_epilogue(&args);
	83	}
	84
	85	SEC(".struct_ops.link")
	86	struct bpf_testmod_st_ops pro_epilogue_with_kfunc = {
	87	.test_pro_epilogue = (void *)test_kfunc_pro_epilogue,
	88	};

tools/testing/selftests/bpf/progs/rcu_read_lock.c+60−1

@@ -242,7 +242,8 @@ out: · 241 unmodified lines
242	}	242	}
243		243
244	SEC("?lsm.s/bpf")	244	SEC("?lsm.s/bpf")
245	int BPF_PROG(inproper_sleepable_kfunc, int cmd, union bpf_attr *attr, unsigned int size)	245	int BPF_PROG(inproper_sleepable_kfunc, int cmd, union bpf_attr *attr, unsigned int size,
		246	bool kernel)
246	{	247	{
247	struct bpf_key *bkey;	248	struct bpf_key *bkey;
248		249
@@ -439,3 +440,61 @@ int rcu_read_lock_global_subprog_unlock(void *ctx) · 190 unmodified lines
439	ret += global_subprog_unlock(ret);	440	ret += global_subprog_unlock(ret);
440	return 0;	441	return 0;
441	}	442	}
		443
		444	int __noinline
		445	global_sleepable_helper_subprog(int i)
		446	{
		447	if (i)
		448	bpf_copy_from_user(&i, sizeof(i), NULL);
		449	return i;
		450	}
		451
		452	int __noinline
		453	global_sleepable_kfunc_subprog(int i)
		454	{
		455	if (i)
		456	bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
		457	global_subprog(i);
		458	return i;
		459	}
		460
		461	int __noinline
		462	global_subprog_calling_sleepable_global(int i)
		463	{
		464	if (!i)
		465	global_sleepable_kfunc_subprog(i);
		466	return i;
		467	}
		468
		469	SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
		470	int rcu_read_lock_sleepable_helper_global_subprog(void *ctx)
		471	{
		472	volatile int ret = 0;
		473
		474	bpf_rcu_read_lock();
		475	ret += global_sleepable_helper_subprog(ret);
		476	bpf_rcu_read_unlock();
		477	return 0;
		478	}
		479
		480	SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
		481	int rcu_read_lock_sleepable_kfunc_global_subprog(void *ctx)
		482	{
		483	volatile int ret = 0;
		484
		485	bpf_rcu_read_lock();
		486	ret += global_sleepable_kfunc_subprog(ret);
		487	bpf_rcu_read_unlock();
		488	return 0;
		489	}
		490
		491	SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
		492	int rcu_read_lock_sleepable_global_subprog_indirect(void *ctx)
		493	{
		494	volatile int ret = 0;
		495
		496	bpf_rcu_read_lock();
		497	ret += global_subprog_calling_sleepable_global(ret);
		498	bpf_rcu_read_unlock();
		499	return 0;
		500	}

tools/testing/selftests/bpf/progs/read_vsyscall.c+9−2

 int target_pid = 0;
 void *user_ptr = 0;
-int read_ret[9];
+int read_ret[10];
 char _license[] SEC("license") = "GPL";
 /*
- * This is the only kfunc, the others are helpers
+ * These are the kfuncs, the others are helpers
  */
 int bpf_copy_from_user_str(void *dst, u32, const void *, u64) __weak __ksym;
+int bpf_copy_from_user_task_str(void *dst, u32, const void *,
+				struct task_struct *, u64) __weak __ksym;
 SEC("fentry/" SYS_PREFIX "sys_nanosleep")
 int do_probe_read(void *ctx)
 	read_ret[7] = bpf_copy_from_user_task(buf, sizeof(buf), user_ptr,
 					      bpf_get_current_task_btf(), 0);
 	read_ret[8] = bpf_copy_from_user_str((char *)buf, sizeof(buf), user_ptr, 0);
+	read_ret[9] = bpf_copy_from_user_task_str((char *)buf,
+						  sizeof(buf),
+						  user_ptr,
+						  bpf_get_current_task_btf(),
+						  0);
 	return 0;
 }

tools/testing/selftests/bpf/progs/set_global_vars.c+47−0

@@ -0,0 +1,47 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#include "bpf_experimental.h"
	4	#include <bpf/bpf_helpers.h>
	5	#include "bpf_misc.h"
	6	#include <stdbool.h>
	7
	8	char _license[] SEC("license") = "GPL";
	9
	10	enum Enum { EA1 = 0, EA2 = 11 };
	11	enum Enumu64 {EB1 = 0llu, EB2 = 12llu };
	12	enum Enums64 { EC1 = 0ll, EC2 = 13ll };
	13
	14	const volatile __s64 var_s64 = -1;
	15	const volatile __u64 var_u64 = 0;
	16	const volatile __s32 var_s32 = -1;
	17	const volatile __u32 var_u32 = 0;
	18	const volatile __s16 var_s16 = -1;
	19	const volatile __u16 var_u16 = 0;
	20	const volatile __s8 var_s8 = -1;
	21	const volatile __u8 var_u8 = 0;
	22	const volatile enum Enum var_ea = EA1;
	23	const volatile enum Enumu64 var_eb = EB1;
	24	const volatile enum Enums64 var_ec = EC1;
	25	const volatile bool var_b = false;
	26
	27	char arr[4] = {0};
	28
	29	SEC("socket")
	30	int test_set_globals(void *ctx)
	31	{
	32	volatile __s8 a;
	33
	34	a = var_s64;
	35	a = var_u64;
	36	a = var_s32;
	37	a = var_u32;
	38	a = var_s16;
	39	a = var_u16;
	40	a = var_s8;
	41	a = var_u8;
	42	a = var_ea;
	43	a = var_eb;
	44	a = var_ec;
	45	a = var_b;
	46	return a;
	47	}

tools/testing/selftests/bpf/progs/strncmp_bench.c+4−1

 SEC("tp/syscalls/sys_enter_getpgid")
 int strncmp_no_helper(void *ctx)
 {
-	if (local_strncmp(str, cmp_str_len + 1, target) < 0)
+	const char *target_str = target;
+	barrier_var(target_str);
+	if (local_strncmp(str, cmp_str_len + 1, target_str) < 0)
 		__sync_add_and_fetch(&hits, 1);
 	return 0;
 }

tools/testing/selftests/bpf/progs/struct_ops_kptr_return.c+30−0

@@ -0,0 +1,30 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_misc.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	void bpf_task_release(struct task_struct *p) __ksym;
	9
	10	/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	11	* allow a referenced kptr or a NULL pointer to be returned. A referenced kptr to task
	12	* here is acquried automatically as the task argument is tagged with "__ref".
	13	*/
	14	SEC("struct_ops/test_return_ref_kptr")
	15	struct task_struct *BPF_PROG(kptr_return, int dummy,
	16	struct task_struct task, struct cgroup cgrp)
	17	{
	18	if (dummy % 2) {
	19	bpf_task_release(task);
	20	return NULL;
	21	}
	22	return task;
	23	}
	24
	25	SEC(".struct_ops.link")
	26	struct bpf_testmod_ops testmod_kptr_return = {
	27	.test_return_ref_kptr = (void *)kptr_return,
	28	};
	29
	30

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__invalid_scalar.c+26−0

@@ -0,0 +1,26 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_misc.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	9	void bpf_task_release(struct task_struct *p) __ksym;
	10
	11	/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	12	* reject programs returning a non-zero scalar value.
	13	*/
	14	SEC("struct_ops/test_return_ref_kptr")
	15	__failure __msg("At program exit the register R0 has smin=1 smax=1 should have been in [0, 0]")
	16	struct task_struct *BPF_PROG(kptr_return_fail__invalid_scalar, int dummy,
	17	struct task_struct task, struct cgroup cgrp)
	18	{
	19	bpf_task_release(task);
	20	return (struct task_struct *)1;
	21	}
	22
	23	SEC(".struct_ops.link")
	24	struct bpf_testmod_ops testmod_kptr_return = {
	25	.test_return_ref_kptr = (void *)kptr_return_fail__invalid_scalar,
	26	};

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__local_kptr.c+34−0

@@ -0,0 +1,34 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_experimental.h"
	5	#include "bpf_misc.h"
	6
	7	char _license[] SEC("license") = "GPL";
	8
	9	struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	10	void bpf_task_release(struct task_struct *p) __ksym;
	11
	12	/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	13	* reject programs returning a local kptr.
	14	*/
	15	SEC("struct_ops/test_return_ref_kptr")
	16	__failure __msg("At program exit the register R0 is not a known value (ptr_or_null_)")
	17	struct task_struct *BPF_PROG(kptr_return_fail__local_kptr, int dummy,
	18	struct task_struct task, struct cgroup cgrp)
	19	{
	20	struct task_struct *t;
	21
	22	bpf_task_release(task);
	23
	24	t = bpf_obj_new(typeof(*task));
	25	if (!t)
	26	return NULL;
	27
	28	return t;
	29	}
	30
	31	SEC(".struct_ops.link")
	32	struct bpf_testmod_ops testmod_kptr_return = {
	33	.test_return_ref_kptr = (void *)kptr_return_fail__local_kptr,
	34	};

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__nonzero_offset.c+25−0

@@ -0,0 +1,25 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_misc.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	9	void bpf_task_release(struct task_struct *p) __ksym;
	10
	11	/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	12	* reject programs returning a modified referenced kptr.
	13	*/
	14	SEC("struct_ops/test_return_ref_kptr")
	15	__failure __msg("dereference of modified trusted_ptr_ ptr R0 off={{[0-9]+}} disallowed")
	16	struct task_struct *BPF_PROG(kptr_return_fail__nonzero_offset, int dummy,
	17	struct task_struct task, struct cgroup cgrp)
	18	{
	19	return (struct task_struct *)&task->jobctl;
	20	}
	21
	22	SEC(".struct_ops.link")
	23	struct bpf_testmod_ops testmod_kptr_return = {
	24	.test_return_ref_kptr = (void *)kptr_return_fail__nonzero_offset,
	25	};

tools/testing/selftests/bpf/progs/struct_ops_kptr_return_fail__wrong_type.c+30−0

@@ -0,0 +1,30 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_misc.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	struct cgroup bpf_cgroup_acquire(struct cgroup p) __ksym;
	9	void bpf_task_release(struct task_struct *p) __ksym;
	10
	11	/* This test struct_ops BPF programs returning referenced kptr. The verifier should
	12	* reject programs returning a referenced kptr of the wrong type.
	13	*/
	14	SEC("struct_ops/test_return_ref_kptr")
	15	__failure __msg("At program exit the register R0 is not a known value (ptr_or_null_)")
	16	struct task_struct *BPF_PROG(kptr_return_fail__wrong_type, int dummy,
	17	struct task_struct task, struct cgroup cgrp)
	18	{
	19	struct task_struct *ret;
	20
	21	ret = (struct task_struct *)bpf_cgroup_acquire(cgrp);
	22	bpf_task_release(task);
	23
	24	return ret;
	25	}
	26
	27	SEC(".struct_ops.link")
	28	struct bpf_testmod_ops testmod_kptr_return = {
	29	.test_return_ref_kptr = (void *)kptr_return_fail__wrong_type,
	30	};

tools/testing/selftests/bpf/progs/struct_ops_refcounted.c+31−0

@@ -0,0 +1,31 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_misc.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	__attribute__((nomerge)) extern void bpf_task_release(struct task_struct *p) __ksym;
	9
	10	/* This is a test BPF program that uses struct_ops to access a referenced
	11	* kptr argument. This is a test for the verifier to ensure that it
	12	* 1) recongnizes the task as a referenced object (i.e., ref_obj_id > 0), and
	13	* 2) the same reference can be acquired from multiple paths as long as it
	14	* has not been released.
	15	*/
	16	SEC("struct_ops/test_refcounted")
	17	int BPF_PROG(refcounted, int dummy, struct task_struct *task)
	18	{
	19	if (dummy == 1)
	20	bpf_task_release(task);
	21	else
	22	bpf_task_release(task);
	23	return 0;
	24	}
	25
	26	SEC(".struct_ops.link")
	27	struct bpf_testmod_ops testmod_refcounted = {
	28	.test_refcounted = (void *)refcounted,
	29	};
	30
	31

tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__global_subprog.c+39−0

@@ -0,0 +1,39 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_misc.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	extern void bpf_task_release(struct task_struct *p) __ksym;
	9
	10	__noinline int subprog_release(__u64 *ctx __arg_ctx)
	11	{
	12	struct task_struct task = (struct task_struct )ctx[1];
	13	int dummy = (int)ctx[0];
	14
	15	bpf_task_release(task);
	16
	17	return dummy + 1;
	18	}
	19
	20	/* Test that the verifier rejects a program that contains a global
	21	* subprogram with referenced kptr arguments
	22	*/
	23	SEC("struct_ops/test_refcounted")
	24	__failure __log_level(2)
	25	__msg("Validating subprog_release() func#1...")
	26	__msg("invalid bpf_context access off=8. Reference may already be released")
	27	int refcounted_fail__global_subprog(unsigned long long *ctx)
	28	{
	29	struct task_struct task = (struct task_struct )ctx[1];
	30
	31	bpf_task_release(task);
	32
	33	return subprog_release(ctx);
	34	}
	35
	36	SEC(".struct_ops.link")
	37	struct bpf_testmod_ops testmod_ref_acquire = {
	38	.test_refcounted = (void *)refcounted_fail__global_subprog,
	39	};

tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__ref_leak.c+22−0

@@ -0,0 +1,22 @@
	1	#include <vmlinux.h>
	2	#include <bpf/bpf_tracing.h>
	3	#include "../test_kmods/bpf_testmod.h"
	4	#include "bpf_misc.h"
	5
	6	char _license[] SEC("license") = "GPL";
	7
	8	/* Test that the verifier rejects a program that acquires a referenced
	9	* kptr through context without releasing the reference
	10	*/
	11	SEC("struct_ops/test_refcounted")
	12	__failure __msg("Unreleased reference id=1 alloc_insn=0")
	13	int BPF_PROG(refcounted_fail__ref_leak, int dummy,
	14	struct task_struct *task)
	15	{
	16	return 0;
	17	}
	18
	19	SEC(".struct_ops.link")
	20	struct bpf_testmod_ops testmod_ref_acquire = {
	21	.test_refcounted = (void *)refcounted_fail__ref_leak,
	22	};

tools/testing/selftests/bpf/progs/struct_ops_refcounted_fail__tail_call.c+36−0

@@ -0,0 +1,36 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Meta Platforms, Inc. and affiliates. */
	3	#include <vmlinux.h>
	4	#include <bpf/bpf_tracing.h>
	5	#include "../test_kmods/bpf_testmod.h"
	6	#include "bpf_misc.h"
	7
	8	char _license[] SEC("license") = "GPL";
	9
	10	struct {
	11	__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
	12	__uint(max_entries, 1);
	13	__uint(key_size, sizeof(__u32));
	14	__uint(value_size, sizeof(__u32));
	15	} prog_array SEC(".maps");
	16
	17	/* Test that the verifier rejects a program with referenced kptr arguments
	18	* that tail call
	19	*/
	20	SEC("struct_ops/test_refcounted")
	21	__failure __msg("program with __ref argument cannot tail call")
	22	int refcounted_fail__tail_call(unsigned long long *ctx)
	23	{
	24	struct task_struct task = (struct task_struct )ctx[1];
	25
	26	bpf_task_release(task);
	27	bpf_tail_call(ctx, &prog_array, 0);
	28
	29	return 0;
	30	}
	31
	32	SEC(".struct_ops.link")
	33	struct bpf_testmod_ops testmod_ref_acquire = {
	34	.test_refcounted = (void *)refcounted_fail__tail_call,
	35	};
	36

tools/testing/selftests/bpf/progs/summarization.c+78−0

@@ -0,0 +1,78 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include <vmlinux.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include "bpf_misc.h"
	6
	7	__noinline
	8	long changes_pkt_data(struct __sk_buff *sk)
	9	{
	10	return bpf_skb_pull_data(sk, 0);
	11	}
	12
	13	__noinline __weak
	14	long does_not_change_pkt_data(struct __sk_buff *sk)
	15	{
	16	return 0;
	17	}
	18
	19	SEC("?tc")
	20	int main_changes_with_subprogs(struct __sk_buff *sk)
	21	{
	22	changes_pkt_data(sk);
	23	does_not_change_pkt_data(sk);
	24	return 0;
	25	}
	26
	27	SEC("?tc")
	28	int main_changes(struct __sk_buff *sk)
	29	{
	30	bpf_skb_pull_data(sk, 0);
	31	return 0;
	32	}
	33
	34	SEC("?tc")
	35	int main_does_not_change(struct __sk_buff *sk)
	36	{
	37	return 0;
	38	}
	39
	40	__noinline
	41	long might_sleep(struct pt_regs *ctx __arg_ctx)
	42	{
	43	int i;
	44
	45	bpf_copy_from_user(&i, sizeof(i), NULL);
	46	return i;
	47	}
	48
	49	__noinline __weak
	50	long does_not_sleep(struct pt_regs *ctx __arg_ctx)
	51	{
	52	return 0;
	53	}
	54
	55	SEC("?uprobe.s")
	56	int main_might_sleep_with_subprogs(struct pt_regs *ctx)
	57	{
	58	might_sleep(ctx);
	59	does_not_sleep(ctx);
	60	return 0;
	61	}
	62
	63	SEC("?uprobe.s")
	64	int main_might_sleep(struct pt_regs *ctx)
	65	{
	66	int i;
	67
	68	bpf_copy_from_user(&i, sizeof(i), NULL);
	69	return i;
	70	}
	71
	72	SEC("?uprobe.s")
	73	int main_does_not_sleep(struct pt_regs *ctx)
	74	{
	75	return 0;
	76	}
	77
	78	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/summarization_freplace.crenamed from tools/testing/selftests/bpf/progs/changes_pkt_data_freplace.c+16−1

@@ -1,6 +1,6 @@
1	// SPDX-License-Identifier: GPL-2.0	1	// SPDX-License-Identifier: GPL-2.0
2		2
3	#include <linux/bpf.h>	3	#include <vmlinux.h>
4	#include <bpf/bpf_helpers.h>	4	#include <bpf/bpf_helpers.h>
5		5
6	SEC("?freplace")	6	SEC("?freplace")
@@ -15,4 +15,19 @@ long does_not_change_pkt_data(struct __sk_buff *sk) · 8 unmodified lines
15	return 0;	15	return 0;
16	}	16	}
17		17
		18	SEC("?freplace")
		19	long might_sleep(struct pt_regs *ctx)
		20	{
		21	int i;
		22
		23	bpf_copy_from_user(&i, sizeof(i), NULL);
		24	return i;
		25	}
		26
		27	SEC("?freplace")
		28	long does_not_sleep(struct pt_regs *ctx)
		29	{
		30	return 0;
		31	}
		32
18	char _license[] SEC("license") = "GPL";	33	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_cgroup1_hierarchy.c+2−2

@@ -51,13 +51,13 @@ out: · 50 unmodified lines
51	}	51	}
52		52
53	SEC("lsm/bpf")	53	SEC("lsm/bpf")
54	int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size)	54	int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
55	{	55	{
56	return bpf_link_create_verify(cmd);	56	return bpf_link_create_verify(cmd);
57	}	57	}
58		58
59	SEC("lsm.s/bpf")	59	SEC("lsm.s/bpf")
60	int BPF_PROG(lsm_s_run, int cmd, union bpf_attr *attr, unsigned int size)	60	int BPF_PROG(lsm_s_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
61	{	61	{
62	return bpf_link_create_verify(cmd);	62	return bpf_link_create_verify(cmd);
63	}	63	}

tools/testing/selftests/bpf/progs/test_core_reloc_arrays.c+5−0

 struct core_reloc_arrays_output {
 	int a2;
+	int a3;
 	char b123;
 	int c1c;
 	int d00d;
 {
 	struct core_reloc_arrays *in = (void *)&data.in;
 	struct core_reloc_arrays_output *out = (void *)&data.out;
+	int *a;
 	if (CORE_READ(&out->a2, &in->a[2]))
 		return 1;
 	if (CORE_READ(&out->f01c, &in->f[0][1].c))
 		return 1;
+	a = __builtin_preserve_access_index(({ in->a; }));
+	out->a3 = a[0] + a[1] + a[2] + a[3];
 	return 0;
 }

tools/testing/selftests/bpf/progs/test_get_xattr.c+24−4

 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 #include "bpf_kfuncs.h"
+#include "bpf_misc.h"
 char _license[] SEC("license") = "GPL";
 char value1[32];
 char value2[32];
+/* Matches caller of test_get_xattr() in prog_tests/fs_kfuncs.c */
+static const char xattr_names[][64] = {
+	/* The following work. */
+	"user.kfuncs",
+	"security.bpf.xxx",
+	/* The following do not work. */
+	"security.bpf",
+	"security.selinux"
+};
 SEC("lsm.s/file_open")
 int BPF_PROG(test_file_open, struct file *f)
 {
 	struct bpf_dynptr value_ptr;
 	__u32 pid;
-	int ret;
+	int ret, i;
 	pid = bpf_get_current_pid_tgid() >> 32;
 	if (pid != monitored_pid)
 	bpf_dynptr_from_mem(value1, sizeof(value1), 0, &value_ptr);
-	ret = bpf_get_file_xattr(f, "user.kfuncs", &value_ptr);
+	for (i = 0; i < ARRAY_SIZE(xattr_names); i++) {
+		ret = bpf_get_file_xattr(f, xattr_names[i], &value_ptr);
+		if (ret == sizeof(expected_value))
+			break;
+	}
 	if (ret != sizeof(expected_value))
 		return 0;
 	if (bpf_strncmp(value1, ret, expected_value))
 {
 	struct bpf_dynptr value_ptr;
 	__u32 pid;
-	int ret;
+	int ret, i;
 	pid = bpf_get_current_pid_tgid() >> 32;
 	if (pid != monitored_pid)
 	bpf_dynptr_from_mem(value2, sizeof(value2), 0, &value_ptr);
-	ret = bpf_get_dentry_xattr(dentry, "user.kfuncs", &value_ptr);
+	for (i = 0; i < ARRAY_SIZE(xattr_names); i++) {
+		ret = bpf_get_dentry_xattr(dentry, xattr_names[i], &value_ptr);
+		if (ret == sizeof(expected_value))
+			break;
+	}
 	if (ret != sizeof(expected_value))
 		return 0;
 	if (bpf_strncmp(value2, ret, expected_value))

tools/testing/selftests/bpf/progs/test_kernel_flag.c+28−0

@@ -0,0 +1,28 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	/*
	4	* Copyright (C) 2025 Microsoft Corporation
	5	*
	6	* Author: Blaise Boscaccy <bboscaccy@linux.microsoft.com>
	7	*/
	8
	9	#include "vmlinux.h"
	10	#include <errno.h>
	11	#include <bpf/bpf_helpers.h>
	12	#include <bpf/bpf_tracing.h>
	13
	14	char _license[] SEC("license") = "GPL";
	15
	16	__u32 monitored_tid;
	17
	18	SEC("lsm.s/bpf")
	19	int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
	20	{
	21	__u32 tid;
	22
	23	tid = bpf_get_current_pid_tgid() & 0xFFFFFFFF;
	24	if (!kernel \|\| tid != monitored_tid)
	25	return 0;
	26	else
	27	return -EINVAL;
	28	}

tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c+3−3

@@ -36,7 +36,7 @@ char _license[] SEC("license") = "GPL"; · 35 unmodified lines
36		36
37	SEC("?lsm.s/bpf")	37	SEC("?lsm.s/bpf")
38	__failure __msg("cannot pass in dynptr at an offset=-8")	38	__failure __msg("cannot pass in dynptr at an offset=-8")
39	int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size)	39	int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
40	{	40	{
41	unsigned long val;	41	unsigned long val;
42		42
@@ -46,7 +46,7 @@ int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size) · 3 unmodified lines
46		46
47	SEC("?lsm.s/bpf")	47	SEC("?lsm.s/bpf")
48	__failure __msg("arg#0 expected pointer to stack or const struct bpf_dynptr")	48	__failure __msg("arg#0 expected pointer to stack or const struct bpf_dynptr")
49	int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size)	49	int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
50	{	50	{
51	unsigned long val = 0;	51	unsigned long val = 0;
52		52
@@ -55,7 +55,7 @@ int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size) · 2 unmodified lines
55	}	55	}
56		56
57	SEC("lsm.s/bpf")	57	SEC("lsm.s/bpf")
58	int BPF_PROG(dynptr_data_null, int cmd, union bpf_attr *attr, unsigned int size)	58	int BPF_PROG(dynptr_data_null, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
59	{	59	{
60	struct bpf_key *trusted_keyring;	60	struct bpf_key *trusted_keyring;
61	struct bpf_dynptr ptr;	61	struct bpf_dynptr ptr;

tools/testing/selftests/bpf/progs/test_lookup_key.c+1−1

@@ -23,7 +23,7 @@ extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym; · 22 unmodified lines
23	extern void bpf_key_put(struct bpf_key *key) __ksym;	23	extern void bpf_key_put(struct bpf_key *key) __ksym;
24		24
25	SEC("lsm.s/bpf")	25	SEC("lsm.s/bpf")
26	int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size)	26	int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
27	{	27	{
28	struct bpf_key *bkey;	28	struct bpf_key *bkey;
29	__u32 pid;	29	__u32 pid;

tools/testing/selftests/bpf/progs/test_ptr_untrusted.c+1−1

@@ -7,7 +7,7 @@ · 6 unmodified lines
7	char tp_name[128];	7	char tp_name[128];
8		8
9	SEC("lsm.s/bpf")	9	SEC("lsm.s/bpf")
10	int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size)	10	int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
11	{	11	{
12	switch (cmd) {	12	switch (cmd) {
13	case BPF_RAW_TRACEPOINT_OPEN:	13	case BPF_RAW_TRACEPOINT_OPEN:

tools/testing/selftests/bpf/progs/test_select_reuseport_kern.c+0−1

@@ -1,7 +1,6 @@
1	// SPDX-License-Identifier: GPL-2.0	1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (c) 2018 Facebook */	2	/* Copyright (c) 2018 Facebook */
3		3
4	#include <stdlib.h>
5	#include <linux/in.h>	4	#include <linux/in.h>
6	#include <linux/ip.h>	5	#include <linux/ip.h>
7	#include <linux/ipv6.h>	6	#include <linux/ipv6.h>

tools/testing/selftests/bpf/progs/test_set_remove_xattr.c+133−0

@@ -0,0 +1,133 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
	3
	4	#include "vmlinux.h"
	5	#include <errno.h>
	6	#include <bpf/bpf_tracing.h>
	7	#include "bpf_kfuncs.h"
	8	#include "bpf_misc.h"
	9
	10	char _license[] SEC("license") = "GPL";
	11
	12	__u32 monitored_pid;
	13
	14	const char xattr_foo[] = "security.bpf.foo";
	15	const char xattr_bar[] = "security.bpf.bar";
	16	static const char xattr_selinux[] = "security.selinux";
	17	char value_bar[] = "world";
	18	char read_value[32];
	19
	20	bool set_security_bpf_bar_success;
	21	bool remove_security_bpf_bar_success;
	22	bool set_security_selinux_fail;
	23	bool remove_security_selinux_fail;
	24
	25	char name_buf[32];
	26
	27	static inline bool name_match_foo(const char *name)
	28	{
	29	bpf_probe_read_kernel(name_buf, sizeof(name_buf), name);
	30
	31	return !bpf_strncmp(name_buf, sizeof(xattr_foo), xattr_foo);
	32	}
	33
	34	/* Test bpf_set_dentry_xattr and bpf_remove_dentry_xattr */
	35	SEC("lsm.s/inode_getxattr")
	36	int BPF_PROG(test_inode_getxattr, struct dentry dentry, char name)
	37	{
	38	struct bpf_dynptr value_ptr;
	39	__u32 pid;
	40	int ret;
	41
	42	pid = bpf_get_current_pid_tgid() >> 32;
	43	if (pid != monitored_pid)
	44	return 0;
	45
	46	/* Only do the following for security.bpf.foo */
	47	if (!name_match_foo(name))
	48	return 0;
	49
	50	bpf_dynptr_from_mem(read_value, sizeof(read_value), 0, &value_ptr);
	51
	52	/* read security.bpf.bar */
	53	ret = bpf_get_dentry_xattr(dentry, xattr_bar, &value_ptr);
	54
	55	if (ret < 0) {
	56	/* If security.bpf.bar doesn't exist, set it */
	57	bpf_dynptr_from_mem(value_bar, sizeof(value_bar), 0, &value_ptr);
	58
	59	ret = bpf_set_dentry_xattr(dentry, xattr_bar, &value_ptr, 0);
	60	if (!ret)
	61	set_security_bpf_bar_success = true;
	62	ret = bpf_set_dentry_xattr(dentry, xattr_selinux, &value_ptr, 0);
	63	if (ret)
	64	set_security_selinux_fail = true;
	65	} else {
	66	/* If security.bpf.bar exists, remove it */
	67	ret = bpf_remove_dentry_xattr(dentry, xattr_bar);
	68	if (!ret)
	69	remove_security_bpf_bar_success = true;
	70
	71	ret = bpf_remove_dentry_xattr(dentry, xattr_selinux);
	72	if (ret)
	73	remove_security_selinux_fail = true;
	74	}
	75
	76	return 0;
	77	}
	78
	79	bool locked_set_security_bpf_bar_success;
	80	bool locked_remove_security_bpf_bar_success;
	81	bool locked_set_security_selinux_fail;
	82	bool locked_remove_security_selinux_fail;
	83
	84	/* Test bpf_set_dentry_xattr_locked and bpf_remove_dentry_xattr_locked.
	85	* It not necessary to differentiate the _locked version and the
	86	* not-_locked version in the BPF program. The verifier will fix them up
	87	* properly.
	88	*/
	89	SEC("lsm.s/inode_setxattr")
	90	int BPF_PROG(test_inode_setxattr, struct mnt_idmap *idmap,
	91	struct dentry dentry, const char name,
	92	const void *value, size_t size, int flags)
	93	{
	94	struct bpf_dynptr value_ptr;
	95	__u32 pid;
	96	int ret;
	97
	98	pid = bpf_get_current_pid_tgid() >> 32;
	99	if (pid != monitored_pid)
	100	return 0;
	101
	102	/* Only do the following for security.bpf.foo */
	103	if (!name_match_foo(name))
	104	return 0;
	105
	106	bpf_dynptr_from_mem(read_value, sizeof(read_value), 0, &value_ptr);
	107
	108	/* read security.bpf.bar */
	109	ret = bpf_get_dentry_xattr(dentry, xattr_bar, &value_ptr);
	110
	111	if (ret < 0) {
	112	/* If security.bpf.bar doesn't exist, set it */
	113	bpf_dynptr_from_mem(value_bar, sizeof(value_bar), 0, &value_ptr);
	114
	115	ret = bpf_set_dentry_xattr(dentry, xattr_bar, &value_ptr, 0);
	116	if (!ret)
	117	locked_set_security_bpf_bar_success = true;
	118	ret = bpf_set_dentry_xattr(dentry, xattr_selinux, &value_ptr, 0);
	119	if (ret)
	120	locked_set_security_selinux_fail = true;
	121	} else {
	122	/* If security.bpf.bar exists, remove it */
	123	ret = bpf_remove_dentry_xattr(dentry, xattr_bar);
	124	if (!ret)
	125	locked_remove_security_bpf_bar_success = true;
	126
	127	ret = bpf_remove_dentry_xattr(dentry, xattr_selinux);
	128	if (ret)
	129	locked_remove_security_selinux_fail = true;
	130	}
	131
	132	return 0;
	133	}

tools/testing/selftests/bpf/progs/test_spin_lock_fail.c+69−0

@@ -245,4 +245,73 @@ int lock_global_subprog_call2(struct __sk_buff *ctx) · 244 unmodified lines
245	return ret;	245	return ret;
246	}	246	}
247		247
		248	int __noinline
		249	global_subprog_int(int i)
		250	{
		251	if (i)
		252	bpf_printk("%p", &i);
		253	return i;
		254	}
		255
		256	int __noinline
		257	global_sleepable_helper_subprog(int i)
		258	{
		259	if (i)
		260	bpf_copy_from_user(&i, sizeof(i), NULL);
		261	return i;
		262	}
		263
		264	int __noinline
		265	global_sleepable_kfunc_subprog(int i)
		266	{
		267	if (i)
		268	bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
		269	global_subprog_int(i);
		270	return i;
		271	}
		272
		273	int __noinline
		274	global_subprog_calling_sleepable_global(int i)
		275	{
		276	if (!i)
		277	global_sleepable_kfunc_subprog(i);
		278	return i;
		279	}
		280
		281	SEC("?syscall")
		282	int lock_global_sleepable_helper_subprog(struct __sk_buff *ctx)
		283	{
		284	int ret = 0;
		285
		286	bpf_spin_lock(&lockA);
		287	if (ctx->mark == 42)
		288	ret = global_sleepable_helper_subprog(ctx->mark);
		289	bpf_spin_unlock(&lockA);
		290	return ret;
		291	}
		292
		293	SEC("?syscall")
		294	int lock_global_sleepable_kfunc_subprog(struct __sk_buff *ctx)
		295	{
		296	int ret = 0;
		297
		298	bpf_spin_lock(&lockA);
		299	if (ctx->mark == 42)
		300	ret = global_sleepable_kfunc_subprog(ctx->mark);
		301	bpf_spin_unlock(&lockA);
		302	return ret;
		303	}
		304
		305	SEC("?syscall")
		306	int lock_global_sleepable_subprog_indirect(struct __sk_buff *ctx)
		307	{
		308	int ret = 0;
		309
		310	bpf_spin_lock(&lockA);
		311	if (ctx->mark == 42)
		312	ret = global_subprog_calling_sleepable_global(ctx->mark);
		313	bpf_spin_unlock(&lockA);
		314	return ret;
		315	}
		316
248	char _license[] SEC("license") = "GPL";	317	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_task_under_cgroup.c+1−1

@@ -49,7 +49,7 @@ out: · 48 unmodified lines
49	}	49	}
50		50
51	SEC("lsm.s/bpf")	51	SEC("lsm.s/bpf")
52	int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size)	52	int BPF_PROG(lsm_run, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
53	{	53	{
54	struct cgroup *cgrp = NULL;	54	struct cgroup *cgrp = NULL;
55	struct task_struct *task;	55	struct task_struct *task;

tools/testing/selftests/bpf/progs/test_usdt.c+14−0

 u64 usdt0_cookie;
 int usdt0_arg_cnt;
 int usdt0_arg_ret;
+int usdt0_arg_size;
 SEC("usdt")
 int usdt0(struct pt_regs *ctx)
 	usdt0_arg_cnt = bpf_usdt_arg_cnt(ctx);
 	/* should return -ENOENT for any arg_num */
 	usdt0_arg_ret = bpf_usdt_arg(ctx, bpf_get_prandom_u32(), &tmp);
+	usdt0_arg_size = bpf_usdt_arg_size(ctx, bpf_get_prandom_u32());
 	return 0;
 }
 int usdt3_arg_cnt;
 int usdt3_arg_rets[3];
 u64 usdt3_args[3];
+int usdt3_arg_sizes[3];
 SEC("usdt//proc/self/exe:test:usdt3")
 int usdt3(struct pt_regs *ctx)
 	usdt3_arg_rets[0] = bpf_usdt_arg(ctx, 0, &tmp);
 	usdt3_args[0] = (int)tmp;
+	usdt3_arg_sizes[0] = bpf_usdt_arg_size(ctx, 0);
 	usdt3_arg_rets[1] = bpf_usdt_arg(ctx, 1, &tmp);
 	usdt3_args[1] = (long)tmp;
+	usdt3_arg_sizes[1] = bpf_usdt_arg_size(ctx, 1);
 	usdt3_arg_rets[2] = bpf_usdt_arg(ctx, 2, &tmp);
 	usdt3_args[2] = (uintptr_t)tmp;
+	usdt3_arg_sizes[2] = bpf_usdt_arg_size(ctx, 2);
 	return 0;
 }
 u64 usdt12_cookie;
 int usdt12_arg_cnt;
 u64 usdt12_args[12];
+int usdt12_arg_sizes[12];
 SEC("usdt//proc/self/exe:test:usdt12")
 int BPF_USDT(usdt12, int a1, int a2, long a3, long a4, unsigned a5,
 		     long a6, __u64 a7, uintptr_t a8, int a9, short a10,
 		     short a11, signed char a12)
 {
+	int i;
 	if (my_pid != (bpf_get_current_pid_tgid() >> 32))
 		return 0;
 	usdt12_args[9] = a10;
 	usdt12_args[10] = a11;
 	usdt12_args[11] = a12;
+	bpf_for(i, 0, 12) {
+		usdt12_arg_sizes[i] = bpf_usdt_arg_size(ctx, i);
+	}
 	return 0;
 }

tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c+1−1

@@ -37,7 +37,7 @@ struct { · 36 unmodified lines
37	char _license[] SEC("license") = "GPL";	37	char _license[] SEC("license") = "GPL";
38		38
39	SEC("lsm.s/bpf")	39	SEC("lsm.s/bpf")
40	int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size)	40	int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
41	{	41	{
42	struct bpf_dynptr data_ptr, sig_ptr;	42	struct bpf_dynptr data_ptr, sig_ptr;
43	struct data *data_val;	43	struct data *data_val;

tools/testing/selftests/bpf/progs/test_xdp_vlan.c+10−10

@@ -102,8 +102,8 @@ bool parse_eth_frame(struct ethhdr eth, void data_end, struct parse_pkt *pkt) · 101 unmodified lines
102	#define TESTVLAN 4011 /* 0xFAB */	102	#define TESTVLAN 4011 /* 0xFAB */
103	// #define TO_VLAN 4000 /* 0xFA0 (hint 0xOA0 = 160) */	103	// #define TO_VLAN 4000 /* 0xFA0 (hint 0xOA0 = 160) */
104		104
105	SEC("xdp_drop_vlan_4011")	105	SEC("xdp")
106	int xdp_prognum0(struct xdp_md *ctx)	106	int xdp_drop_vlan_4011(struct xdp_md *ctx)
107	{	107	{
108	void data_end = (void )(long)ctx->data_end;	108	void data_end = (void )(long)ctx->data_end;
109	void data = (void )(long)ctx->data;	109	void data = (void )(long)ctx->data;
@@ -144,8 +144,8 @@ Load prog with ip tool: · 34 unmodified lines
144	/* Changing VLAN to zero, have same practical effect as removing the VLAN. */	144	/* Changing VLAN to zero, have same practical effect as removing the VLAN. */
145	#define TO_VLAN 0	145	#define TO_VLAN 0
146		146
147	SEC("xdp_vlan_change")	147	SEC("xdp")
148	int xdp_prognum1(struct xdp_md *ctx)	148	int xdp_vlan_change(struct xdp_md *ctx)
149	{	149	{
150	void data_end = (void )(long)ctx->data_end;	150	void data_end = (void )(long)ctx->data_end;
151	void data = (void )(long)ctx->data;	151	void data = (void )(long)ctx->data;
@@ -178,8 +178,8 @@ int xdp_prognum1(struct xdp_md *ctx) · 26 unmodified lines
178	#endif	178	#endif
179	#define VLAN_HDR_SZ 4 /* bytes */	179	#define VLAN_HDR_SZ 4 /* bytes */
180		180
181	SEC("xdp_vlan_remove_outer")	181	SEC("xdp")
182	int xdp_prognum2(struct xdp_md *ctx)	182	int xdp_vlan_remove_outer(struct xdp_md *ctx)
183	{	183	{
184	void data_end = (void )(long)ctx->data_end;	184	void data_end = (void )(long)ctx->data_end;
185	void data = (void )(long)ctx->data;	185	void data = (void )(long)ctx->data;
@@ -224,8 +224,8 @@ void shift_mac_4bytes_32bit(void *data) · 38 unmodified lines
224	p[1] = p[0];	224	p[1] = p[0];
225	}	225	}
226		226
227	SEC("xdp_vlan_remove_outer2")	227	SEC("xdp")
228	int xdp_prognum3(struct xdp_md *ctx)	228	int xdp_vlan_remove_outer2(struct xdp_md *ctx)
229	{	229	{
230	void data_end = (void )(long)ctx->data_end;	230	void data_end = (void )(long)ctx->data_end;
231	void data = (void )(long)ctx->data;	231	void data = (void )(long)ctx->data;
@@ -254,8 +254,8 @@ int xdp_prognum3(struct xdp_md *ctx) · 22 unmodified lines
254	* The TC-clsact eBPF programs (currently) need to be attach via TC commands	254	* The TC-clsact eBPF programs (currently) need to be attach via TC commands
255	*/	255	*/
256		256
257	SEC("tc_vlan_push")	257	SEC("tc")
258	int _tc_progA(struct __sk_buff *ctx)	258	int tc_vlan_push(struct __sk_buff *ctx)
259	{	259	{
260	bpf_skb_vlan_push(ctx, bpf_htons(ETH_P_8021Q), TESTVLAN);	260	bpf_skb_vlan_push(ctx, bpf_htons(ETH_P_8021Q), TESTVLAN);
261		261

tools/testing/selftests/bpf/progs/verifier_bpf_fastcall.c+48−10

@@ -620,23 +620,61 @@ __naked void helper_call_does_not_prevent_bpf_fastcall(void) · 619 unmodified lines
620		620
621	SEC("raw_tp")	621	SEC("raw_tp")
622	__arch_x86_64	622	__arch_x86_64
		623	__log_level(4) __msg("stack depth 24")
		624	/* may_goto counter at -24 */
		625	__xlated("0: (u64 )(r10 -24) =")
		626	/* may_goto timestamp at -16 */
		627	__xlated("1: (u64 )(r10 -16) =")
		628	__xlated("2: r1 = 1")
		629	__xlated("...")
		630	__xlated("4: r0 = &(void __percpu *)(r0)")
		631	__xlated("...")
		632	/* may_goto expansion starts */
		633	__xlated("6: r11 = (u64 )(r10 -24)")
		634	__xlated("7: if r11 == 0x0 goto pc+6")
		635	__xlated("8: r11 -= 1")
		636	__xlated("9: if r11 != 0x0 goto pc+2")
		637	__xlated("10: r11 = -24")
		638	__xlated("11: call unknown")
		639	__xlated("12: (u64 )(r10 -24) = r11")
		640	/* may_goto expansion ends */
		641	__xlated("13: (u64 )(r10 -8) = r1")
		642	__xlated("14: exit")
		643	__success
		644	__naked void may_goto_interaction_x86_64(void)
		645	{
		646	asm volatile (
		647	"r1 = 1;"
		648	"(u64 )(r10 - 16) = r1;"
		649	"call %[bpf_get_smp_processor_id];"
		650	"r1 = (u64 )(r10 - 16);"
		651	".8byte %[may_goto];"
		652	/* just touch some stack at -8 */
		653	"(u64 )(r10 - 8) = r1;"
		654	"exit;"
		655	:
		656	: __imm(bpf_get_smp_processor_id),
		657	__imm_insn(may_goto, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, +1 /* offset */, 0))
		658	: __clobber_all);
		659	}
		660
		661	SEC("raw_tp")
		662	__arch_arm64
623	__log_level(4) __msg("stack depth 16")	663	__log_level(4) __msg("stack depth 16")
624	/* may_goto counter at -16 */	664	/* may_goto counter at -16 */
625	__xlated("0: (u64 )(r10 -16) =")	665	__xlated("0: (u64 )(r10 -16) =")
626	__xlated("1: r1 = 1")	666	__xlated("1: r1 = 1")
627	__xlated("...")	667	__xlated("2: call bpf_get_smp_processor_id")
628	__xlated("3: r0 = &(void __percpu *)(r0)")
629	__xlated("...")
630	/* may_goto expansion starts */	668	/* may_goto expansion starts */
631	__xlated("5: r11 = (u64 )(r10 -16)")	669	__xlated("3: r11 = (u64 )(r10 -16)")
632	__xlated("6: if r11 == 0x0 goto pc+3")	670	__xlated("4: if r11 == 0x0 goto pc+3")
633	__xlated("7: r11 -= 1")	671	__xlated("5: r11 -= 1")
634	__xlated("8: (u64 )(r10 -16) = r11")	672	__xlated("6: (u64 )(r10 -16) = r11")
635	/* may_goto expansion ends */	673	/* may_goto expansion ends */
636	__xlated("9: (u64 )(r10 -8) = r1")	674	__xlated("7: (u64 )(r10 -8) = r1")
637	__xlated("10: exit")	675	__xlated("8: exit")
638	__success	676	__success
639	__naked void may_goto_interaction(void)	677	__naked void may_goto_interaction_arm64(void)
640	{	678	{
641	asm volatile (	679	asm volatile (
642	"r1 = 1;"	680	"r1 = 1;"

tools/testing/selftests/bpf/progs/verifier_gotol.c+1−5

 #include <bpf/bpf_helpers.h>
 #include "bpf_misc.h"
-#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
+#ifdef CAN_USE_GOTOL
-	(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
-	defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \
-	defined(__TARGET_ARCH_loongarch)) && \
-	__clang_major__ >= 18
 SEC("socket")
 __description("gotol, small_imm")

tools/testing/selftests/bpf/progs/verifier_iterating_callbacks.c+1−5

 	: __clobber_all);
 }
-#if (defined(__TARGET_ARCH_arm64) || defined(__TARGET_ARCH_x86) || \
+#ifdef CAN_USE_GOTOL
-	(defined(__TARGET_ARCH_riscv) && __riscv_xlen == 64) || \
-	defined(__TARGET_ARCH_arm) || defined(__TARGET_ARCH_s390) || \
-	defined(__TARGET_ARCH_loongarch)) && \
-	__clang_major__ >= 18
 SEC("socket")
 __success __retval(0)
 __naked void gotol_and_may_goto(void)

tools/testing/selftests/bpf/progs/verifier_load_acquire.c+218−0

@@ -0,0 +1,218 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Google LLC. */
	3
	4	#include <linux/bpf.h>
	5	#include <bpf/bpf_helpers.h>
	6	#include "../../../include/linux/filter.h"
	7	#include "bpf_misc.h"
	8
	9	#ifdef CAN_USE_LOAD_ACQ_STORE_REL
	10
	11	SEC("socket")
	12	__description("load-acquire, 8-bit")
	13	__success __success_unpriv __retval(0x12)
	14	__naked void load_acquire_8(void)
	15	{
	16	asm volatile (
	17	"w1 = 0x12;"
	18	"(u8 )(r10 - 1) = w1;"
	19	".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r10 - 1));
	20	"exit;"
	21	:
	22	: __imm_insn(load_acquire_insn,
	23	BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -1))
	24	: __clobber_all);
	25	}
	26
	27	SEC("socket")
	28	__description("load-acquire, 16-bit")
	29	__success __success_unpriv __retval(0x1234)
	30	__naked void load_acquire_16(void)
	31	{
	32	asm volatile (
	33	"w1 = 0x1234;"
	34	"(u16 )(r10 - 2) = w1;"
	35	".8byte %[load_acquire_insn];" // w0 = load_acquire((u16 *)(r10 - 2));
	36	"exit;"
	37	:
	38	: __imm_insn(load_acquire_insn,
	39	BPF_ATOMIC_OP(BPF_H, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -2))
	40	: __clobber_all);
	41	}
	42
	43	SEC("socket")
	44	__description("load-acquire, 32-bit")
	45	__success __success_unpriv __retval(0x12345678)
	46	__naked void load_acquire_32(void)
	47	{
	48	asm volatile (
	49	"w1 = 0x12345678;"
	50	"(u32 )(r10 - 4) = w1;"
	51	".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 4));
	52	"exit;"
	53	:
	54	: __imm_insn(load_acquire_insn,
	55	BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -4))
	56	: __clobber_all);
	57	}
	58
	59	SEC("socket")
	60	__description("load-acquire, 64-bit")
	61	__success __success_unpriv __retval(0x1234567890abcdef)
	62	__naked void load_acquire_64(void)
	63	{
	64	asm volatile (
	65	"r1 = 0x1234567890abcdef ll;"
	66	"(u64 )(r10 - 8) = r1;"
	67	".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r10 - 8));
	68	"exit;"
	69	:
	70	: __imm_insn(load_acquire_insn,
	71	BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -8))
	72	: __clobber_all);
	73	}
	74
	75	SEC("socket")
	76	__description("load-acquire with uninitialized src_reg")
	77	__failure __failure_unpriv __msg("R2 !read_ok")
	78	__naked void load_acquire_with_uninitialized_src_reg(void)
	79	{
	80	asm volatile (
	81	".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r2 + 0));
	82	"exit;"
	83	:
	84	: __imm_insn(load_acquire_insn,
	85	BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
	86	: __clobber_all);
	87	}
	88
	89	SEC("socket")
	90	__description("load-acquire with non-pointer src_reg")
	91	__failure __failure_unpriv __msg("R1 invalid mem access 'scalar'")
	92	__naked void load_acquire_with_non_pointer_src_reg(void)
	93	{
	94	asm volatile (
	95	"r1 = 0;"
	96	".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r1 + 0));
	97	"exit;"
	98	:
	99	: __imm_insn(load_acquire_insn,
	100	BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
	101	: __clobber_all);
	102	}
	103
	104	SEC("socket")
	105	__description("misaligned load-acquire")
	106	__failure __failure_unpriv __msg("misaligned stack access off")
	107	__flag(BPF_F_ANY_ALIGNMENT)
	108	__naked void load_acquire_misaligned(void)
	109	{
	110	asm volatile (
	111	"r1 = 0;"
	112	"(u64 )(r10 - 8) = r1;"
	113	".8byte %[load_acquire_insn];" // w0 = load_acquire((u32 *)(r10 - 5));
	114	"exit;"
	115	:
	116	: __imm_insn(load_acquire_insn,
	117	BPF_ATOMIC_OP(BPF_W, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_10, -5))
	118	: __clobber_all);
	119	}
	120
	121	SEC("socket")
	122	__description("load-acquire from ctx pointer")
	123	__failure __failure_unpriv __msg("BPF_ATOMIC loads from R1 ctx is not allowed")
	124	__naked void load_acquire_from_ctx_pointer(void)
	125	{
	126	asm volatile (
	127	".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r1 + 0));
	128	"exit;"
	129	:
	130	: __imm_insn(load_acquire_insn,
	131	BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_1, 0))
	132	: __clobber_all);
	133	}
	134
	135	SEC("xdp")
	136	__description("load-acquire from pkt pointer")
	137	__failure __msg("BPF_ATOMIC loads from R2 pkt is not allowed")
	138	__naked void load_acquire_from_pkt_pointer(void)
	139	{
	140	asm volatile (
	141	"r2 = (u32 )(r1 + %[xdp_md_data]);"
	142	"r3 = (u32 )(r1 + %[xdp_md_data_end]);"
	143	"r1 = r2;"
	144	"r1 += 8;"
	145	"if r1 >= r3 goto l0_%=;"
	146	".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
	147	"l0_%=: r0 = 0;"
	148	"exit;"
	149	:
	150	: __imm_const(xdp_md_data, offsetof(struct xdp_md, data)),
	151	__imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)),
	152	__imm_insn(load_acquire_insn,
	153	BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
	154	: __clobber_all);
	155	}
	156
	157	SEC("flow_dissector")
	158	__description("load-acquire from flow_keys pointer")
	159	__failure __msg("BPF_ATOMIC loads from R2 flow_keys is not allowed")
	160	__naked void load_acquire_from_flow_keys_pointer(void)
	161	{
	162	asm volatile (
	163	"r2 = (u64 )(r1 + %[__sk_buff_flow_keys]);"
	164	".8byte %[load_acquire_insn];" // w0 = load_acquire((u8 *)(r2 + 0));
	165	"exit;"
	166	:
	167	: __imm_const(__sk_buff_flow_keys,
	168	offsetof(struct __sk_buff, flow_keys)),
	169	__imm_insn(load_acquire_insn,
	170	BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2, 0))
	171	: __clobber_all);
	172	}
	173
	174	SEC("sk_reuseport")
	175	__description("load-acquire from sock pointer")
	176	__failure __msg("BPF_ATOMIC loads from R2 sock is not allowed")
	177	__naked void load_acquire_from_sock_pointer(void)
	178	{
	179	asm volatile (
	180	"r2 = (u64 )(r1 + %[sk_reuseport_md_sk]);"
	181	// w0 = load_acquire((u8 *)(r2 + offsetof(struct bpf_sock, family)));
	182	".8byte %[load_acquire_insn];"
	183	"exit;"
	184	:
	185	: __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)),
	186	__imm_insn(load_acquire_insn,
	187	BPF_ATOMIC_OP(BPF_B, BPF_LOAD_ACQ, BPF_REG_0, BPF_REG_2,
	188	offsetof(struct bpf_sock, family)))
	189	: __clobber_all);
	190	}
	191
	192	SEC("socket")
	193	__description("load-acquire with invalid register R15")
	194	__failure __failure_unpriv __msg("R15 is invalid")
	195	__naked void load_acquire_with_invalid_reg(void)
	196	{
	197	asm volatile (
	198	".8byte %[load_acquire_insn];" // r0 = load_acquire((u64 *)(r15 + 0));
	199	"exit;"
	200	:
	201	: __imm_insn(load_acquire_insn,
	202	BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_0, 15 /* invalid reg */, 0))
	203	: __clobber_all);
	204	}
	205
	206	#else /* CAN_USE_LOAD_ACQ_STORE_REL */
	207
	208	SEC("socket")
	209	__description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support load-acquire, use a dummy test")
	210	__success
	211	int dummy_test(void)
	212	{
	213	return 0;
	214	}
	215
	216	#endif /* CAN_USE_LOAD_ACQ_STORE_REL */
	217
	218	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/verifier_may_goto_1.c+32−2

@@ -69,8 +69,38 @@ __naked void may_goto_batch_1(void) · 68 unmodified lines
69	}	69	}
70		70
71	SEC("raw_tp")	71	SEC("raw_tp")
72	__description("may_goto batch with offsets 2/0")	72	__description("may_goto batch with offsets 2/0 - x86_64")
73	__arch_x86_64	73	__arch_x86_64
		74	__xlated("0: (u64 )(r10 -16) = 65535")
		75	__xlated("1: (u64 )(r10 -8) = 0")
		76	__xlated("2: r11 = (u64 )(r10 -16)")
		77	__xlated("3: if r11 == 0x0 goto pc+6")
		78	__xlated("4: r11 -= 1")
		79	__xlated("5: if r11 != 0x0 goto pc+2")
		80	__xlated("6: r11 = -16")
		81	__xlated("7: call unknown")
		82	__xlated("8: (u64 )(r10 -16) = r11")
		83	__xlated("9: r0 = 1")
		84	__xlated("10: r0 = 2")
		85	__xlated("11: exit")
		86	__success
		87	__naked void may_goto_batch_2_x86_64(void)
		88	{
		89	asm volatile (
		90	".8byte %[may_goto1];"
		91	".8byte %[may_goto3];"
		92	"r0 = 1;"
		93	"r0 = 2;"
		94	"exit;"
		95	:
		96	: __imm_insn(may_goto1, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, 2 /* offset */, 0)),
		97	__imm_insn(may_goto3, BPF_RAW_INSN(BPF_JMP \| BPF_JCOND, 0, 0, 0 /* offset */, 0))
		98	: __clobber_all);
		99	}
		100
		101	SEC("raw_tp")
		102	__description("may_goto batch with offsets 2/0 - arm64")
		103	__arch_arm64
74	__xlated("0: (u64 )(r10 -8) = 8388608")	104	__xlated("0: (u64 )(r10 -8) = 8388608")
75	__xlated("1: r11 = (u64 )(r10 -8)")	105	__xlated("1: r11 = (u64 )(r10 -8)")
76	__xlated("2: if r11 == 0x0 goto pc+3")	106	__xlated("2: if r11 == 0x0 goto pc+3")
@@ -80,7 +110,7 @@ __xlated("5: r0 = 1") · 3 unmodified lines
80	__xlated("6: r0 = 2")	110	__xlated("6: r0 = 2")
81	__xlated("7: exit")	111	__xlated("7: exit")
82	__success	112	__success
83	__naked void may_goto_batch_2(void)	113	__naked void may_goto_batch_2_arm64(void)
84	{	114	{
85	asm volatile (	115	asm volatile (
86	".8byte %[may_goto1];"	116	".8byte %[may_goto1];"

tools/testing/selftests/bpf/progs/verifier_precision.c+49−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	/* Copyright (C) 2023 SUSE LLC */	2	/* Copyright (C) 2023 SUSE LLC */
3	#include <linux/bpf.h>	3	#include <linux/bpf.h>
4	#include <bpf/bpf_helpers.h>	4	#include <bpf/bpf_helpers.h>
		5	#include "../../../include/linux/filter.h"
5	#include "bpf_misc.h"	6	#include "bpf_misc.h"
6		7
7	SEC("?raw_tp")	8	SEC("?raw_tp")
@@ -90,6 +91,54 @@ __naked int bpf_end_bswap(void) · 82 unmodified lines
90	::: __clobber_all);	91	::: __clobber_all);
91	}	92	}
92		93
		94	#if defined(ENABLE_ATOMICS_TESTS) && \
		95	(defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
		96
		97	SEC("?raw_tp")
		98	__success __log_level(2)
		99	__msg("mark_precise: frame0: regs=r2 stack= before 3: (bf) r3 = r10")
		100	__msg("mark_precise: frame0: regs=r2 stack= before 2: (db) r2 = load_acquire((u64 *)(r10 -8))")
		101	__msg("mark_precise: frame0: regs= stack=-8 before 1: (7b) (u64 )(r10 -8) = r1")
		102	__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
		103	__naked int bpf_load_acquire(void)
		104	{
		105	asm volatile (
		106	"r1 = 8;"
		107	"(u64 )(r10 - 8) = r1;"
		108	".8byte %[load_acquire_insn];" /* r2 = load_acquire((u64 )(r10 - 8)); /
		109	"r3 = r10;"
		110	"r3 += r2;" /* mark_precise */
		111	"r0 = 0;"
		112	"exit;"
		113	:
		114	: __imm_insn(load_acquire_insn,
		115	BPF_ATOMIC_OP(BPF_DW, BPF_LOAD_ACQ, BPF_REG_2, BPF_REG_10, -8))
		116	: __clobber_all);
		117	}
		118
		119	SEC("?raw_tp")
		120	__success __log_level(2)
		121	__msg("mark_precise: frame0: regs=r1 stack= before 3: (bf) r2 = r10")
		122	__msg("mark_precise: frame0: regs=r1 stack= before 2: (79) r1 = (u64 )(r10 -8)")
		123	__msg("mark_precise: frame0: regs= stack=-8 before 1: (db) store_release((u64 *)(r10 -8), r1)")
		124	__msg("mark_precise: frame0: regs=r1 stack= before 0: (b7) r1 = 8")
		125	__naked int bpf_store_release(void)
		126	{
		127	asm volatile (
		128	"r1 = 8;"
		129	".8byte %[store_release_insn];" /* store_release((u64 )(r10 - 8), r1); /
		130	"r1 = (u64 )(r10 - 8);"
		131	"r2 = r10;"
		132	"r2 += r1;" /* mark_precise */
		133	"r0 = 0;"
		134	"exit;"
		135	:
		136	: __imm_insn(store_release_insn,
		137	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8))
		138	: __clobber_all);
		139	}
		140
		141	#endif /* load-acquire, store-release */
93	#endif /* v4 instruction */	142	#endif /* v4 instruction */
94		143
95	SEC("?raw_tp")	144	SEC("?raw_tp")

tools/testing/selftests/bpf/progs/verifier_stack_ptr.c+52−0

@@ -481,4 +481,56 @@ l1_%=: r0 = 42; \ · 480 unmodified lines
481	: __clobber_all);	481	: __clobber_all);
482	}	482	}
483		483
		484	SEC("socket")
		485	__description("PTR_TO_STACK stack size > 512")
		486	__failure __msg("invalid write to stack R1 off=-520 size=8")
		487	__naked void stack_check_size_gt_512(void)
		488	{
		489	asm volatile (" \
		490	r1 = r10; \
		491	r1 += -520; \
		492	r0 = 42; \
		493	(u64)(r1 + 0) = r0; \
		494	exit; \
		495	" ::: __clobber_all);
		496	}
		497
		498	#ifdef __BPF_FEATURE_MAY_GOTO
		499	SEC("socket")
		500	__description("PTR_TO_STACK stack size 512 with may_goto with jit")
		501	__load_if_JITed()
		502	__success __retval(42)
		503	__naked void stack_check_size_512_with_may_goto_jit(void)
		504	{
		505	asm volatile (" \
		506	r1 = r10; \
		507	r1 += -512; \
		508	r0 = 42; \
		509	(u32)(r1 + 0) = r0; \
		510	may_goto l0_%=; \
		511	r2 = 100; \
		512	l0_%=: \
		513	exit; \
		514	" ::: __clobber_all);
		515	}
		516
		517	SEC("socket")
		518	__description("PTR_TO_STACK stack size 512 with may_goto without jit")
		519	__load_if_no_JITed()
		520	__failure __msg("stack size 520(extra 8) is too large")
		521	__naked void stack_check_size_512_with_may_goto(void)
		522	{
		523	asm volatile (" \
		524	r1 = r10; \
		525	r1 += -512; \
		526	r0 = 42; \
		527	(u32)(r1 + 0) = r0; \
		528	may_goto l0_%=; \
		529	r2 = 100; \
		530	l0_%=: \
		531	exit; \
		532	" ::: __clobber_all);
		533	}
		534	#endif
		535
484	char _license[] SEC("license") = "GPL";	536	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/verifier_store_release.c+286−0

@@ -0,0 +1,286 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2025 Google LLC. */
	3
	4	#include <linux/bpf.h>
	5	#include <bpf/bpf_helpers.h>
	6	#include "../../../include/linux/filter.h"
	7	#include "bpf_misc.h"
	8
	9	#if __clang_major__ >= 18 && defined(ENABLE_ATOMICS_TESTS) && \
	10	(defined(__TARGET_ARCH_arm64) \|\| defined(__TARGET_ARCH_x86))
	11
	12	SEC("socket")
	13	__description("store-release, 8-bit")
	14	__success __success_unpriv __retval(0x12)
	15	__naked void store_release_8(void)
	16	{
	17	asm volatile (
	18	"w1 = 0x12;"
	19	".8byte %[store_release_insn];" // store_release((u8 *)(r10 - 1), w1);
	20	"w0 = (u8 )(r10 - 1);"
	21	"exit;"
	22	:
	23	: __imm_insn(store_release_insn,
	24	BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -1))
	25	: __clobber_all);
	26	}
	27
	28	SEC("socket")
	29	__description("store-release, 16-bit")
	30	__success __success_unpriv __retval(0x1234)
	31	__naked void store_release_16(void)
	32	{
	33	asm volatile (
	34	"w1 = 0x1234;"
	35	".8byte %[store_release_insn];" // store_release((u16 *)(r10 - 2), w1);
	36	"w0 = (u16 )(r10 - 2);"
	37	"exit;"
	38	:
	39	: __imm_insn(store_release_insn,
	40	BPF_ATOMIC_OP(BPF_H, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -2))
	41	: __clobber_all);
	42	}
	43
	44	SEC("socket")
	45	__description("store-release, 32-bit")
	46	__success __success_unpriv __retval(0x12345678)
	47	__naked void store_release_32(void)
	48	{
	49	asm volatile (
	50	"w1 = 0x12345678;"
	51	".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 4), w1);
	52	"w0 = (u32 )(r10 - 4);"
	53	"exit;"
	54	:
	55	: __imm_insn(store_release_insn,
	56	BPF_ATOMIC_OP(BPF_W, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -4))
	57	: __clobber_all);
	58	}
	59
	60	SEC("socket")
	61	__description("store-release, 64-bit")
	62	__success __success_unpriv __retval(0x1234567890abcdef)
	63	__naked void store_release_64(void)
	64	{
	65	asm volatile (
	66	"r1 = 0x1234567890abcdef ll;"
	67	".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1);
	68	"r0 = (u64 )(r10 - 8);"
	69	"exit;"
	70	:
	71	: __imm_insn(store_release_insn,
	72	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8))
	73	: __clobber_all);
	74	}
	75
	76	SEC("socket")
	77	__description("store-release with uninitialized src_reg")
	78	__failure __failure_unpriv __msg("R2 !read_ok")
	79	__naked void store_release_with_uninitialized_src_reg(void)
	80	{
	81	asm volatile (
	82	".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r2);
	83	"exit;"
	84	:
	85	: __imm_insn(store_release_insn,
	86	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_2, -8))
	87	: __clobber_all);
	88	}
	89
	90	SEC("socket")
	91	__description("store-release with uninitialized dst_reg")
	92	__failure __failure_unpriv __msg("R2 !read_ok")
	93	__naked void store_release_with_uninitialized_dst_reg(void)
	94	{
	95	asm volatile (
	96	"r1 = 0;"
	97	".8byte %[store_release_insn];" // store_release((u64 *)(r2 - 8), r1);
	98	"exit;"
	99	:
	100	: __imm_insn(store_release_insn,
	101	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_2, BPF_REG_1, -8))
	102	: __clobber_all);
	103	}
	104
	105	SEC("socket")
	106	__description("store-release with non-pointer dst_reg")
	107	__failure __failure_unpriv __msg("R1 invalid mem access 'scalar'")
	108	__naked void store_release_with_non_pointer_dst_reg(void)
	109	{
	110	asm volatile (
	111	"r1 = 0;"
	112	".8byte %[store_release_insn];" // store_release((u64 *)(r1 + 0), r1);
	113	"exit;"
	114	:
	115	: __imm_insn(store_release_insn,
	116	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_1, BPF_REG_1, 0))
	117	: __clobber_all);
	118	}
	119
	120	SEC("socket")
	121	__description("misaligned store-release")
	122	__failure __failure_unpriv __msg("misaligned stack access off")
	123	__flag(BPF_F_ANY_ALIGNMENT)
	124	__naked void store_release_misaligned(void)
	125	{
	126	asm volatile (
	127	"w0 = 0;"
	128	".8byte %[store_release_insn];" // store_release((u32 *)(r10 - 5), w0);
	129	"exit;"
	130	:
	131	: __imm_insn(store_release_insn,
	132	BPF_ATOMIC_OP(BPF_W, BPF_STORE_REL, BPF_REG_10, BPF_REG_0, -5))
	133	: __clobber_all);
	134	}
	135
	136	SEC("socket")
	137	__description("store-release to ctx pointer")
	138	__failure __failure_unpriv __msg("BPF_ATOMIC stores into R1 ctx is not allowed")
	139	__naked void store_release_to_ctx_pointer(void)
	140	{
	141	asm volatile (
	142	"w0 = 0;"
	143	// store_release((u8 *)(r1 + offsetof(struct __sk_buff, cb[0])), w0);
	144	".8byte %[store_release_insn];"
	145	"exit;"
	146	:
	147	: __imm_insn(store_release_insn,
	148	BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_1, BPF_REG_0,
	149	offsetof(struct __sk_buff, cb[0])))
	150	: __clobber_all);
	151	}
	152
	153	SEC("xdp")
	154	__description("store-release to pkt pointer")
	155	__failure __msg("BPF_ATOMIC stores into R2 pkt is not allowed")
	156	__naked void store_release_to_pkt_pointer(void)
	157	{
	158	asm volatile (
	159	"w0 = 0;"
	160	"r2 = (u32 )(r1 + %[xdp_md_data]);"
	161	"r3 = (u32 )(r1 + %[xdp_md_data_end]);"
	162	"r1 = r2;"
	163	"r1 += 8;"
	164	"if r1 >= r3 goto l0_%=;"
	165	".8byte %[store_release_insn];" // store_release((u8 *)(r2 + 0), w0);
	166	"l0_%=: r0 = 0;"
	167	"exit;"
	168	:
	169	: __imm_const(xdp_md_data, offsetof(struct xdp_md, data)),
	170	__imm_const(xdp_md_data_end, offsetof(struct xdp_md, data_end)),
	171	__imm_insn(store_release_insn,
	172	BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_2, BPF_REG_0, 0))
	173	: __clobber_all);
	174	}
	175
	176	SEC("flow_dissector")
	177	__description("store-release to flow_keys pointer")
	178	__failure __msg("BPF_ATOMIC stores into R2 flow_keys is not allowed")
	179	__naked void store_release_to_flow_keys_pointer(void)
	180	{
	181	asm volatile (
	182	"w0 = 0;"
	183	"r2 = (u64 )(r1 + %[__sk_buff_flow_keys]);"
	184	".8byte %[store_release_insn];" // store_release((u8 *)(r2 + 0), w0);
	185	"exit;"
	186	:
	187	: __imm_const(__sk_buff_flow_keys,
	188	offsetof(struct __sk_buff, flow_keys)),
	189	__imm_insn(store_release_insn,
	190	BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_2, BPF_REG_0, 0))
	191	: __clobber_all);
	192	}
	193
	194	SEC("sk_reuseport")
	195	__description("store-release to sock pointer")
	196	__failure __msg("R2 cannot write into sock")
	197	__naked void store_release_to_sock_pointer(void)
	198	{
	199	asm volatile (
	200	"w0 = 0;"
	201	"r2 = (u64 )(r1 + %[sk_reuseport_md_sk]);"
	202	".8byte %[store_release_insn];" // store_release((u8 *)(r2 + 0), w0);
	203	"exit;"
	204	:
	205	: __imm_const(sk_reuseport_md_sk, offsetof(struct sk_reuseport_md, sk)),
	206	__imm_insn(store_release_insn,
	207	BPF_ATOMIC_OP(BPF_B, BPF_STORE_REL, BPF_REG_2, BPF_REG_0, 0))
	208	: __clobber_all);
	209	}
	210
	211	SEC("socket")
	212	__description("store-release, leak pointer to stack")
	213	__success __success_unpriv __retval(0)
	214	__naked void store_release_leak_pointer_to_stack(void)
	215	{
	216	asm volatile (
	217	".8byte %[store_release_insn];" // store_release((u64 *)(r10 - 8), r1);
	218	"r0 = 0;"
	219	"exit;"
	220	:
	221	: __imm_insn(store_release_insn,
	222	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_10, BPF_REG_1, -8))
	223	: __clobber_all);
	224	}
	225
	226	struct {
	227	__uint(type, BPF_MAP_TYPE_HASH);
	228	__uint(max_entries, 1);
	229	__type(key, long long);
	230	__type(value, long long);
	231	} map_hash_8b SEC(".maps");
	232
	233	SEC("socket")
	234	__description("store-release, leak pointer to map")
	235	__success __retval(0)
	236	__failure_unpriv __msg_unpriv("R6 leaks addr into map")
	237	__naked void store_release_leak_pointer_to_map(void)
	238	{
	239	asm volatile (
	240	"r6 = r1;"
	241	"r1 = %[map_hash_8b] ll;"
	242	"r2 = 0;"
	243	"(u64 )(r10 - 8) = r2;"
	244	"r2 = r10;"
	245	"r2 += -8;"
	246	"call %[bpf_map_lookup_elem];"
	247	"if r0 == 0 goto l0_%=;"
	248	".8byte %[store_release_insn];" // store_release((u64 *)(r0 + 0), r6);
	249	"l0_%=:"
	250	"r0 = 0;"
	251	"exit;"
	252	:
	253	: __imm_addr(map_hash_8b),
	254	__imm(bpf_map_lookup_elem),
	255	__imm_insn(store_release_insn,
	256	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, BPF_REG_0, BPF_REG_6, 0))
	257	: __clobber_all);
	258	}
	259
	260	SEC("socket")
	261	__description("store-release with invalid register R15")
	262	__failure __failure_unpriv __msg("R15 is invalid")
	263	__naked void store_release_with_invalid_reg(void)
	264	{
	265	asm volatile (
	266	".8byte %[store_release_insn];" // store_release((u64 *)(r15 + 0), r1);
	267	"exit;"
	268	:
	269	: __imm_insn(store_release_insn,
	270	BPF_ATOMIC_OP(BPF_DW, BPF_STORE_REL, 15 /* invalid reg */, BPF_REG_1, 0))
	271	: __clobber_all);
	272	}
	273
	274	#else
	275
	276	SEC("socket")
	277	__description("Clang version < 18, ENABLE_ATOMICS_TESTS not defined, and/or JIT doesn't support store-release, use a dummy test")
	278	__success
	279	int dummy_test(void)
	280	{
	281	return 0;
	282	}
	283
	284	#endif
	285
	286	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/xdp_redirect_map.c+88−0

@@ -1,7 +1,10 @@
1	// SPDX-License-Identifier: GPL-2.0	1	// SPDX-License-Identifier: GPL-2.0
2		2
		3	#include <linux/if_ether.h>
		4
3	#include <linux/bpf.h>	5	#include <linux/bpf.h>
4	#include <bpf/bpf_helpers.h>	6	#include <bpf/bpf_helpers.h>
		7	#include <bpf/bpf_endian.h>
5		8
6	struct {	9	struct {
7	__uint(type, BPF_MAP_TYPE_DEVMAP);	10	__uint(type, BPF_MAP_TYPE_DEVMAP);
@@ -28,4 +31,89 @@ int xdp_redirect_map_2(struct xdp_md *xdp) · 20 unmodified lines
28	return bpf_redirect_map(&tx_port, 2, 0);	31	return bpf_redirect_map(&tx_port, 2, 0);
29	}	32	}
30		33
		34	struct {
		35	__uint(type, BPF_MAP_TYPE_ARRAY);
		36	__uint(max_entries, 3);
		37	__type(key, __u32);
		38	__type(value, __u64);
		39	} rxcnt SEC(".maps");
		40
		41	static int xdp_count(struct xdp_md *xdp, __u32 key)
		42	{
		43	void data_end = (void )(long)xdp->data_end;
		44	void data = (void )(long)xdp->data;
		45	struct ethhdr *eth = data;
		46	__u64 *count;
		47
		48	if (data + sizeof(*eth) > data_end)
		49	return XDP_DROP;
		50
		51	if (bpf_htons(eth->h_proto) == ETH_P_IP) {
		52	/* We only count IPv4 packets */
		53	count = bpf_map_lookup_elem(&rxcnt, &key);
		54	if (count)
		55	*count += 1;
		56	}
		57
		58	return XDP_PASS;
		59	}
		60
		61	SEC("xdp")
		62	int xdp_count_0(struct xdp_md *xdp)
		63	{
		64	return xdp_count(xdp, 0);
		65	}
		66
		67	SEC("xdp")
		68	int xdp_count_1(struct xdp_md *xdp)
		69	{
		70	return xdp_count(xdp, 1);
		71	}
		72
		73	SEC("xdp")
		74	int xdp_count_2(struct xdp_md *xdp)
		75	{
		76	return xdp_count(xdp, 2);
		77	}
		78
		79	struct {
		80	__uint(type, BPF_MAP_TYPE_ARRAY);
		81	__uint(max_entries, 2);
		82	__type(key, __u32);
		83	__type(value, __be64);
		84	} rx_mac SEC(".maps");
		85
		86	static int store_mac(struct xdp_md *xdp, __u32 id)
		87	{
		88	void data_end = (void )(long)xdp->data_end;
		89	void data = (void )(long)xdp->data;
		90	struct ethhdr *eth = data;
		91	__u32 key = id;
		92	__be64 mac = 0;
		93
		94	if (data + sizeof(*eth) > data_end)
		95	return XDP_DROP;
		96
		97	/* Only store IPv4 MAC to avoid being polluted by IPv6 packets */
		98	if (eth->h_proto == bpf_htons(ETH_P_IP)) {
		99	__builtin_memcpy(&mac, eth->h_source, ETH_ALEN);
		100	bpf_map_update_elem(&rx_mac, &key, &mac, 0);
		101	bpf_printk("%s - %x", __func__, mac);
		102	}
		103
		104	return XDP_PASS;
		105	}
		106
		107	SEC("xdp")
		108	int store_mac_1(struct xdp_md *xdp)
		109	{
		110	return store_mac(xdp, 0);
		111	}
		112
		113	SEC("xdp")
		114	int store_mac_2(struct xdp_md *xdp)
		115	{
		116	return store_mac(xdp, 1);
		117	}
		118
31	char _license[] SEC("license") = "GPL";	119	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/xdp_redirect_multi_kern.c+29−12

@@ -34,6 +34,14 @@ struct { · 33 unmodified lines
34	__uint(max_entries, 128);	34	__uint(max_entries, 128);
35	} mac_map SEC(".maps");	35	} mac_map SEC(".maps");
36		36
		37	/* map to store redirect flags for each protocol*/
		38	struct {
		39	__uint(type, BPF_MAP_TYPE_HASH);
		40	__type(key, __u16);
		41	__type(value, __u64);
		42	__uint(max_entries, 16);
		43	} redirect_flags SEC(".maps");
		44
37	SEC("xdp")	45	SEC("xdp")
38	int xdp_redirect_map_multi_prog(struct xdp_md *ctx)	46	int xdp_redirect_map_multi_prog(struct xdp_md *ctx)
39	{	47	{
@@ -41,25 +49,34 @@ int xdp_redirect_map_multi_prog(struct xdp_md *ctx) · 1 unmodified line
41	void data = (void )(long)ctx->data;	49	void data = (void )(long)ctx->data;
42	int if_index = ctx->ingress_ifindex;	50	int if_index = ctx->ingress_ifindex;
43	struct ethhdr *eth = data;	51	struct ethhdr *eth = data;
		52	__u64 *flags_from_map;
44	__u16 h_proto;	53	__u16 h_proto;
45	__u64 nh_off;	54	__u64 nh_off;
		55	__u64 flags;
46		56
47	nh_off = sizeof(*eth);	57	nh_off = sizeof(*eth);
48	if (data + nh_off > data_end)	58	if (data + nh_off > data_end)
49	return XDP_DROP;	59	return XDP_DROP;
50		60
51	h_proto = eth->h_proto;	61	h_proto = bpf_htons(eth->h_proto);
52		62
53	/* Using IPv4 for (BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS) testing */	63	flags_from_map = bpf_map_lookup_elem(&redirect_flags, &h_proto);
54	if (h_proto == bpf_htons(ETH_P_IP))	64
55	return bpf_redirect_map(&map_all, 0,	65	/* Default flags for IPv4 : (BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS) */
56	BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS);	66	if (h_proto == ETH_P_IP) {
57	/* Using IPv6 for none flag testing */	67	flags = flags_from_map ? *flags_from_map : BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS;
58	else if (h_proto == bpf_htons(ETH_P_IPV6))	68	return bpf_redirect_map(&map_all, 0, flags);
59	return bpf_redirect_map(&map_all, if_index, 0);	69	}
60	/* All others for BPF_F_BROADCAST testing */	70	/* Default flags for IPv6 : 0 */
61	else	71	if (h_proto == ETH_P_IPV6) {
62	return bpf_redirect_map(&map_all, 0, BPF_F_BROADCAST);	72	flags = flags_from_map ? *flags_from_map : 0;
		73	return bpf_redirect_map(&map_all, if_index, flags);
		74	}
		75	/* Default flags for others BPF_F_BROADCAST : 0 */
		76	else {
		77	flags = flags_from_map ? *flags_from_map : BPF_F_BROADCAST;
		78	return bpf_redirect_map(&map_all, 0, flags);
		79	}
63	}	80	}
64		81
65	/* The following 2 progs are for 2nd devmap prog testing */	82	/* The following 2 progs are for 2nd devmap prog testing */

tools/testing/selftests/bpf/test_btf.h+6−0

 #define BTF_TYPE_FLOAT_ENC(name, sz) \
 	BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz)
+#define BTF_DECL_ATTR_ENC(value, type, component_idx)	\
+	BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 1, 0), type), (component_idx)
 #define BTF_DECL_TAG_ENC(value, type, component_idx)	\
 	BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx)
+#define BTF_TYPE_ATTR_ENC(value, type)	\
+	BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 1, 0), type)
 #define BTF_TYPE_TAG_ENC(value, type)	\
 	BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type)

tools/testing/selftests/bpf/test_kmods/bpf_testmod.c+108−0

@@ -1130,6 +1130,7 @@ static const struct btf_kfunc_id_set bpf_testmod_kfunc_set = { · 1129 unmodified lines
1130	};	1130	};
1131		1131
1132	static const struct bpf_verifier_ops bpf_testmod_verifier_ops = {	1132	static const struct bpf_verifier_ops bpf_testmod_verifier_ops = {
		1133	.get_func_proto = bpf_base_func_proto,
1133	.is_valid_access = bpf_testmod_ops_is_valid_access,	1134	.is_valid_access = bpf_testmod_ops_is_valid_access,
1134	};	1135	};
1135		1136
@@ -1176,10 +1177,25 @@ static int bpf_testmod_ops__test_maybe_null(int dummy, · 40 unmodified lines
1176	return 0;	1177	return 0;
1177	}	1178	}
1178		1179
		1180	static int bpf_testmod_ops__test_refcounted(int dummy,
		1181	struct task_struct *task__ref)
		1182	{
		1183	return 0;
		1184	}
		1185
		1186	static struct task_struct *
		1187	bpf_testmod_ops__test_return_ref_kptr(int dummy, struct task_struct *task__ref,
		1188	struct cgroup *cgrp)
		1189	{
		1190	return NULL;
		1191	}
		1192
1179	static struct bpf_testmod_ops __bpf_testmod_ops = {	1193	static struct bpf_testmod_ops __bpf_testmod_ops = {
1180	.test_1 = bpf_testmod_test_1,	1194	.test_1 = bpf_testmod_test_1,
1181	.test_2 = bpf_testmod_test_2,	1195	.test_2 = bpf_testmod_test_2,
1182	.test_maybe_null = bpf_testmod_ops__test_maybe_null,	1196	.test_maybe_null = bpf_testmod_ops__test_maybe_null,
		1197	.test_refcounted = bpf_testmod_ops__test_refcounted,
		1198	.test_return_ref_kptr = bpf_testmod_ops__test_return_ref_kptr,
1183	};	1199	};
1184		1200
1185	struct bpf_struct_ops bpf_bpf_testmod_ops = {	1201	struct bpf_struct_ops bpf_bpf_testmod_ops = {
@@ -1293,6 +1309,85 @@ static int bpf_test_mod_st_ops__test_pro_epilogue(struct st_ops_args *args) · 107 unmodified lines
1293	return 0;	1309	return 0;
1294	}	1310	}
1295		1311
		1312	static int bpf_cgroup_from_id_id;
		1313	static int bpf_cgroup_release_id;
		1314
		1315	static int st_ops_gen_prologue_with_kfunc(struct bpf_insn *insn_buf, bool direct_write,
		1316	const struct bpf_prog *prog)
		1317	{
		1318	struct bpf_insn *insn = insn_buf;
		1319
		1320	/* r8 = r1; // r8 will be "u64 *ctx".
		1321	* r1 = 0;
		1322	* r0 = bpf_cgroup_from_id(r1);
		1323	* if r0 != 0 goto pc+5;
		1324	* r6 = r8[0]; // r6 will be "struct st_ops *args".
		1325	* r7 = r6->a;
		1326	* r7 += 1000;
		1327	* r6->a = r7;
		1328	* goto pc+2;
		1329	* r1 = r0;
		1330	* bpf_cgroup_release(r1);
		1331	* r1 = r8;
		1332	*/
		1333	*insn++ = BPF_MOV64_REG(BPF_REG_8, BPF_REG_1);
		1334	*insn++ = BPF_MOV64_IMM(BPF_REG_1, 0);
		1335	*insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_from_id_id);
		1336	*insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 5);
		1337	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_8, 0);
		1338	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_6, offsetof(struct st_ops_args, a));
		1339	*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_7, 1000);
		1340	*insn++ = BPF_STX_MEM(BPF_DW, BPF_REG_6, BPF_REG_7, offsetof(struct st_ops_args, a));
		1341	*insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 2);
		1342	*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
		1343	*insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id),
		1344	*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_8);
		1345	*insn++ = prog->insnsi[0];
		1346
		1347	return insn - insn_buf;
		1348	}
		1349
		1350	static int st_ops_gen_epilogue_with_kfunc(struct bpf_insn insn_buf, const struct bpf_prog prog,
		1351	s16 ctx_stack_off)
		1352	{
		1353	struct bpf_insn *insn = insn_buf;
		1354
		1355	/* r1 = 0;
		1356	* r6 = 0;
		1357	* r0 = bpf_cgroup_from_id(r1);
		1358	* if r0 != 0 goto pc+6;
		1359	* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
		1360	* r1 = r1[0]; // r1 will be "struct st_ops *args"
		1361	* r6 = r1->a;
		1362	* r6 += 10000;
		1363	* r1->a = r6;
		1364	* goto pc+2
		1365	* r1 = r0;
		1366	* bpf_cgroup_release(r1);
		1367	* r0 = r6;
		1368	* r0 *= 2;
		1369	* BPF_EXIT;
		1370	*/
		1371	*insn++ = BPF_MOV64_IMM(BPF_REG_1, 0);
		1372	*insn++ = BPF_MOV64_IMM(BPF_REG_6, 0);
		1373	*insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_from_id_id);
		1374	*insn++ = BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 6);
		1375	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_FP, ctx_stack_off);
		1376	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
		1377	*insn++ = BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, offsetof(struct st_ops_args, a));
		1378	*insn++ = BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 10000);
		1379	*insn++ = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6, offsetof(struct st_ops_args, a));
		1380	*insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 2);
		1381	*insn++ = BPF_MOV64_REG(BPF_REG_1, BPF_REG_0);
		1382	*insn++ = BPF_CALL_KFUNC(0, bpf_cgroup_release_id),
		1383	*insn++ = BPF_MOV64_REG(BPF_REG_0, BPF_REG_6);
		1384	*insn++ = BPF_ALU64_IMM(BPF_MUL, BPF_REG_0, 2);
		1385	*insn++ = BPF_EXIT_INSN();
		1386
		1387	return insn - insn_buf;
		1388	}
		1389
		1390	#define KFUNC_PRO_EPI_PREFIX "test_kfunc_"
1296	static int st_ops_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,	1391	static int st_ops_gen_prologue(struct bpf_insn *insn_buf, bool direct_write,
1297	const struct bpf_prog *prog)	1392	const struct bpf_prog *prog)
1298	{	1393	{
@@ -1302,6 +1397,9 @@ static int st_ops_gen_prologue(struct bpf_insn *insn_buf, bool direct_write, · 3 unmodified lines
1302	strcmp(prog->aux->attach_func_name, "test_pro_epilogue"))	1397	strcmp(prog->aux->attach_func_name, "test_pro_epilogue"))
1303	return 0;	1398	return 0;
1304		1399
		1400	if (!strncmp(prog->aux->name, KFUNC_PRO_EPI_PREFIX, strlen(KFUNC_PRO_EPI_PREFIX)))
		1401	return st_ops_gen_prologue_with_kfunc(insn_buf, direct_write, prog);
		1402
1305	/* r6 = r1[0]; // r6 will be "struct st_ops args". r1 is "u64 ctx".	1403	/* r6 = r1[0]; // r6 will be "struct st_ops args". r1 is "u64 ctx".
1306	* r7 = r6->a;	1404	* r7 = r6->a;
1307	* r7 += 1000;	1405	* r7 += 1000;
@@ -1325,6 +1423,9 @@ static int st_ops_gen_epilogue(struct bpf_insn *insn_buf, const struct bpf_prog · 17 unmodified lines
1325	strcmp(prog->aux->attach_func_name, "test_pro_epilogue"))	1423	strcmp(prog->aux->attach_func_name, "test_pro_epilogue"))
1326	return 0;	1424	return 0;
1327		1425
		1426	if (!strncmp(prog->aux->name, KFUNC_PRO_EPI_PREFIX, strlen(KFUNC_PRO_EPI_PREFIX)))
		1427	return st_ops_gen_epilogue_with_kfunc(insn_buf, prog, ctx_stack_off);
		1428
1328	/* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"	1429	/* r1 = stack[ctx_stack_off]; // r1 will be "u64 *ctx"
1329	* r1 = r1[0]; // r1 will be "struct st_ops *args"	1430	* r1 = r1[0]; // r1 will be "struct st_ops *args"
1330	* r6 = r1->a;	1431	* r6 = r1->a;
@@ -1395,6 +1496,13 @@ static void st_ops_unreg(void kdata, struct bpf_link link) · 64 unmodified lines
1395		1496
1396	static int st_ops_init(struct btf *btf)	1497	static int st_ops_init(struct btf *btf)
1397	{	1498	{
		1499	struct btf *kfunc_btf;
		1500
		1501	bpf_cgroup_from_id_id = bpf_find_btf_id("bpf_cgroup_from_id", BTF_KIND_FUNC, &kfunc_btf);
		1502	bpf_cgroup_release_id = bpf_find_btf_id("bpf_cgroup_release", BTF_KIND_FUNC, &kfunc_btf);
		1503	if (bpf_cgroup_from_id_id < 0 \|\| bpf_cgroup_release_id < 0)
		1504	return -EINVAL;
		1505
1398	return 0;	1506	return 0;
1399	}	1507	}
1400		1508

tools/testing/selftests/bpf/test_kmods/bpf_testmod.h+6−0

 #include <linux/types.h>
 struct task_struct;
+struct cgroup;
 struct bpf_testmod_test_read_ctx {
 	char *buf;
 	/* Used to test nullable arguments. */
 	int (*test_maybe_null)(int dummy, struct task_struct *task);
 	int (*unsupported_ops)(void);
+	/* Used to test ref_acquired arguments. */
+	int (*test_refcounted)(int dummy, struct task_struct *task);
+	/* Used to test returning referenced kptr. */
+	struct task_struct *(*test_return_ref_kptr)(int dummy, struct task_struct *task,
+						    struct cgroup *cgrp);
 	/* The following fields are used to test shadow copies. */
 	char onebyte;

tools/testing/selftests/bpf/test_loader.c+29−3

 #define TEST_TAG_JITED_PFX "comment:test_jited="
 #define TEST_TAG_JITED_PFX_UNPRIV "comment:test_jited_unpriv="
 #define TEST_TAG_CAPS_UNPRIV "comment:test_caps_unpriv="
+#define TEST_TAG_LOAD_MODE_PFX "comment:load_mode="
 /* Warning: duplicated in bpf_misc.h */
 #define POINTER_VALUE	0xcafe4all
 	UNPRIV = 2
 };
+enum load_mode {
+	JITED		= 1 << 0,
+	NO_JITED	= 1 << 1,
+};
 struct expect_msg {
 	const char *substr; /* substring match */
 	regex_t regex;
 	int prog_flags;
 	int mode_mask;
 	int arch_mask;
+	int load_mask;
 	bool auxiliary;
 	bool valid;
 };
 	bool collect_jit = false;
 	int func_id, i, err = 0;
 	u32 arch_mask = 0;
+	u32 load_mask = 0;
 	struct btf *btf;
 	enum arch arch;
 			if (err)
 				goto cleanup;
 			spec->mode_mask |= UNPRIV;
+		} else if (str_has_pfx(s, TEST_TAG_LOAD_MODE_PFX)) {
+			val = s + sizeof(TEST_TAG_LOAD_MODE_PFX) - 1;
+			if (strcmp(val, "jited") == 0) {
+				load_mask = JITED;
+			} else if (strcmp(val, "no_jited") == 0) {
+				load_mask = NO_JITED;
+			} else {
+				PRINT_FAIL("bad load spec: '%s'", val);
+				err = -EINVAL;
+				goto cleanup;
+			}
 		}
 	}
 	spec->arch_mask = arch_mask ?: -1;
+	spec->load_mask = load_mask ?: (JITED | NO_JITED);
 	if (spec->mode_mask == 0)
 		spec->mode_mask = PRIV;
 	err = cap_disable_effective(caps_to_drop, &caps->old_caps);
 	if (err) {
-		PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(err));
+		PRINT_FAIL("failed to drop capabilities: %i, %s\n", err, strerror(-err));
 		return err;
 	}
 	err = cap_enable_effective(caps->old_caps, NULL);
 	if (err)
-		PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(err));
+		PRINT_FAIL("failed to restore capabilities: %i, %s\n", err, strerror(-err));
 	caps->initialized = false;
 	return err;
 }
 		 bool unpriv)
 {
 	struct test_subspec *subspec = unpriv ? &spec->unpriv : &spec->priv;
+	int current_runtime = is_jit_enabled() ? JITED : NO_JITED;
 	struct bpf_program *tprog = NULL, *tprog_iter;
 	struct bpf_link *link, *links[32] = {};
 	struct test_spec *spec_iter;
 		return;
 	}
+	if ((current_runtime & spec->load_mask) == 0) {
+		test__skip();
+		return;
+	}
 	if (unpriv) {
 		if (!can_execute_unpriv(tester, spec)) {
 			test__skip();
 		if (subspec->caps) {
 			err = cap_enable_effective(subspec->caps, NULL);
 			if (err) {
-				PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(err));
+				PRINT_FAIL("failed to set capabilities: %i, %s\n", err, strerror(-err));
 				goto subtest_cleanup;
 			}
 		}

tools/testing/selftests/bpf/test_lwt_ip_encap.sh+0−476

@@ -1,476 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3	#
4	# Setup/topology:
5	#
6	# NS1 NS2 NS3
7	# veth1 <---> veth2 veth3 <---> veth4 (the top route)
8	# veth5 <---> veth6 veth7 <---> veth8 (the bottom route)
9	#
10	# each vethN gets IPv[4\|6]_N address
11	#
12	# IPv_SRC = IPv_1
13	# IPv_DST = IPv_4
14	#
15	# all tests test pings from IPv_SRC to IPv_DST
16	#
17	# by default, routes are configured to allow packets to go
18	# IP_1 <=> IP_2 <=> IP_3 <=> IP_4 (the top route)
19	#
20	# a GRE device is installed in NS3 with IPv*_GRE, and
21	# NS1/NS2 are configured to route packets to IPv_GRE via IP_8
22	# (the bottom route)
23	#
24	# Tests:
25	#
26	# 1. routes NS2->IPv*_DST are brought down, so the only way a ping
27	# from IP_SRC to IP_DST can work is via IPv*_GRE
28	#
29	# 2a. in an egress test, a bpf LWT_XMIT program is installed on veth1
30	# that encaps the packets with an IP/GRE header to route to IPv*_GRE
31	#
32	# ping: SRC->[encap at veth1:egress]->GRE:decap->DST
33	# ping replies go DST->SRC directly
34	#
35	# 2b. in an ingress test, a bpf LWT_IN program is installed on veth2
36	# that encaps the packets with an IP/GRE header to route to IPv*_GRE
37	#
38	# ping: SRC->[encap at veth2:ingress]->GRE:decap->DST
39	# ping replies go DST->SRC directly
40
41	BPF_FILE="test_lwt_ip_encap.bpf.o"
42	if [[ $EUID -ne 0 ]]; then
43	echo "This script must be run as root"
44	echo "FAIL"
45	exit 1
46	fi
47
48	readonly NS1="ns1-$(mktemp -u XXXXXX)"
49	readonly NS2="ns2-$(mktemp -u XXXXXX)"
50	readonly NS3="ns3-$(mktemp -u XXXXXX)"
51
52	readonly IPv4_1="172.16.1.100"
53	readonly IPv4_2="172.16.2.100"
54	readonly IPv4_3="172.16.3.100"
55	readonly IPv4_4="172.16.4.100"
56	readonly IPv4_5="172.16.5.100"
57	readonly IPv4_6="172.16.6.100"
58	readonly IPv4_7="172.16.7.100"
59	readonly IPv4_8="172.16.8.100"
60	readonly IPv4_GRE="172.16.16.100"
61
62	readonly IPv4_SRC=$IPv4_1
63	readonly IPv4_DST=$IPv4_4
64
65	readonly IPv6_1="fb01::1"
66	readonly IPv6_2="fb02::1"
67	readonly IPv6_3="fb03::1"
68	readonly IPv6_4="fb04::1"
69	readonly IPv6_5="fb05::1"
70	readonly IPv6_6="fb06::1"
71	readonly IPv6_7="fb07::1"
72	readonly IPv6_8="fb08::1"
73	readonly IPv6_GRE="fb10::1"
74
75	readonly IPv6_SRC=$IPv6_1
76	readonly IPv6_DST=$IPv6_4
77
78	TEST_STATUS=0
79	TESTS_SUCCEEDED=0
80	TESTS_FAILED=0
81
82	TMPFILE=""
83
84	process_test_results()
85	{
86	if [[ "${TEST_STATUS}" -eq 0 ]] ; then
87	echo "PASS"
88	TESTS_SUCCEEDED=$((TESTS_SUCCEEDED+1))
89	else
90	echo "FAIL"
91	TESTS_FAILED=$((TESTS_FAILED+1))
92	fi
93	}
94
95	print_test_summary_and_exit()
96	{
97	echo "passed tests: ${TESTS_SUCCEEDED}"
98	echo "failed tests: ${TESTS_FAILED}"
99	if [ "${TESTS_FAILED}" -eq "0" ] ; then
100	exit 0
101	else
102	exit 1
103	fi
104	}
105
106	setup()
107	{
108	set -e # exit on error
109	TEST_STATUS=0
110
111	# create devices and namespaces
112	ip netns add "${NS1}"
113	ip netns add "${NS2}"
114	ip netns add "${NS3}"
115
116	# rp_filter gets confused by what these tests are doing, so disable it
117	ip netns exec ${NS1} sysctl -wq net.ipv4.conf.all.rp_filter=0
118	ip netns exec ${NS2} sysctl -wq net.ipv4.conf.all.rp_filter=0
119	ip netns exec ${NS3} sysctl -wq net.ipv4.conf.all.rp_filter=0
120	ip netns exec ${NS1} sysctl -wq net.ipv4.conf.default.rp_filter=0
121	ip netns exec ${NS2} sysctl -wq net.ipv4.conf.default.rp_filter=0
122	ip netns exec ${NS3} sysctl -wq net.ipv4.conf.default.rp_filter=0
123
124	# disable IPv6 DAD because it sometimes takes too long and fails tests
125	ip netns exec ${NS1} sysctl -wq net.ipv6.conf.all.accept_dad=0
126	ip netns exec ${NS2} sysctl -wq net.ipv6.conf.all.accept_dad=0
127	ip netns exec ${NS3} sysctl -wq net.ipv6.conf.all.accept_dad=0
128	ip netns exec ${NS1} sysctl -wq net.ipv6.conf.default.accept_dad=0
129	ip netns exec ${NS2} sysctl -wq net.ipv6.conf.default.accept_dad=0
130	ip netns exec ${NS3} sysctl -wq net.ipv6.conf.default.accept_dad=0
131
132	ip link add veth1 type veth peer name veth2
133	ip link add veth3 type veth peer name veth4
134	ip link add veth5 type veth peer name veth6
135	ip link add veth7 type veth peer name veth8
136
137	ip netns exec ${NS2} sysctl -wq net.ipv4.ip_forward=1
138	ip netns exec ${NS2} sysctl -wq net.ipv6.conf.all.forwarding=1
139
140	ip link set veth1 netns ${NS1}
141	ip link set veth2 netns ${NS2}
142	ip link set veth3 netns ${NS2}
143	ip link set veth4 netns ${NS3}
144	ip link set veth5 netns ${NS1}
145	ip link set veth6 netns ${NS2}
146	ip link set veth7 netns ${NS2}
147	ip link set veth8 netns ${NS3}
148
149	if [ ! -z "${VRF}" ] ; then
150	ip -netns ${NS1} link add red type vrf table 1001
151	ip -netns ${NS1} link set red up
152	ip -netns ${NS1} route add table 1001 unreachable default metric 8192
153	ip -netns ${NS1} -6 route add table 1001 unreachable default metric 8192
154	ip -netns ${NS1} link set veth1 vrf red
155	ip -netns ${NS1} link set veth5 vrf red
156
157	ip -netns ${NS2} link add red type vrf table 1001
158	ip -netns ${NS2} link set red up
159	ip -netns ${NS2} route add table 1001 unreachable default metric 8192
160	ip -netns ${NS2} -6 route add table 1001 unreachable default metric 8192
161	ip -netns ${NS2} link set veth2 vrf red
162	ip -netns ${NS2} link set veth3 vrf red
163	ip -netns ${NS2} link set veth6 vrf red
164	ip -netns ${NS2} link set veth7 vrf red
165	fi
166
167	# configure addesses: the top route (1-2-3-4)
168	ip -netns ${NS1} addr add ${IPv4_1}/24 dev veth1
169	ip -netns ${NS2} addr add ${IPv4_2}/24 dev veth2
170	ip -netns ${NS2} addr add ${IPv4_3}/24 dev veth3
171	ip -netns ${NS3} addr add ${IPv4_4}/24 dev veth4
172	ip -netns ${NS1} -6 addr add ${IPv6_1}/128 nodad dev veth1
173	ip -netns ${NS2} -6 addr add ${IPv6_2}/128 nodad dev veth2
174	ip -netns ${NS2} -6 addr add ${IPv6_3}/128 nodad dev veth3
175	ip -netns ${NS3} -6 addr add ${IPv6_4}/128 nodad dev veth4
176
177	# configure addresses: the bottom route (5-6-7-8)
178	ip -netns ${NS1} addr add ${IPv4_5}/24 dev veth5
179	ip -netns ${NS2} addr add ${IPv4_6}/24 dev veth6
180	ip -netns ${NS2} addr add ${IPv4_7}/24 dev veth7
181	ip -netns ${NS3} addr add ${IPv4_8}/24 dev veth8
182	ip -netns ${NS1} -6 addr add ${IPv6_5}/128 nodad dev veth5
183	ip -netns ${NS2} -6 addr add ${IPv6_6}/128 nodad dev veth6
184	ip -netns ${NS2} -6 addr add ${IPv6_7}/128 nodad dev veth7
185	ip -netns ${NS3} -6 addr add ${IPv6_8}/128 nodad dev veth8
186
187	ip -netns ${NS1} link set dev veth1 up
188	ip -netns ${NS2} link set dev veth2 up
189	ip -netns ${NS2} link set dev veth3 up
190	ip -netns ${NS3} link set dev veth4 up
191	ip -netns ${NS1} link set dev veth5 up
192	ip -netns ${NS2} link set dev veth6 up
193	ip -netns ${NS2} link set dev veth7 up
194	ip -netns ${NS3} link set dev veth8 up
195
196	# configure routes: IP_SRC -> veth1/IP_2 (= top route) default;
197	# the bottom route to specific bottom addresses
198
199	# NS1
200	# top route
201	ip -netns ${NS1} route add ${IPv4_2}/32 dev veth1 ${VRF}
202	ip -netns ${NS1} route add default dev veth1 via ${IPv4_2} ${VRF} # go top by default
203	ip -netns ${NS1} -6 route add ${IPv6_2}/128 dev veth1 ${VRF}
204	ip -netns ${NS1} -6 route add default dev veth1 via ${IPv6_2} ${VRF} # go top by default
205	# bottom route
206	ip -netns ${NS1} route add ${IPv4_6}/32 dev veth5 ${VRF}
207	ip -netns ${NS1} route add ${IPv4_7}/32 dev veth5 via ${IPv4_6} ${VRF}
208	ip -netns ${NS1} route add ${IPv4_8}/32 dev veth5 via ${IPv4_6} ${VRF}
209	ip -netns ${NS1} -6 route add ${IPv6_6}/128 dev veth5 ${VRF}
210	ip -netns ${NS1} -6 route add ${IPv6_7}/128 dev veth5 via ${IPv6_6} ${VRF}
211	ip -netns ${NS1} -6 route add ${IPv6_8}/128 dev veth5 via ${IPv6_6} ${VRF}
212
213	# NS2
214	# top route
215	ip -netns ${NS2} route add ${IPv4_1}/32 dev veth2 ${VRF}
216	ip -netns ${NS2} route add ${IPv4_4}/32 dev veth3 ${VRF}
217	ip -netns ${NS2} -6 route add ${IPv6_1}/128 dev veth2 ${VRF}
218	ip -netns ${NS2} -6 route add ${IPv6_4}/128 dev veth3 ${VRF}
219	# bottom route
220	ip -netns ${NS2} route add ${IPv4_5}/32 dev veth6 ${VRF}
221	ip -netns ${NS2} route add ${IPv4_8}/32 dev veth7 ${VRF}
222	ip -netns ${NS2} -6 route add ${IPv6_5}/128 dev veth6 ${VRF}
223	ip -netns ${NS2} -6 route add ${IPv6_8}/128 dev veth7 ${VRF}
224
225	# NS3
226	# top route
227	ip -netns ${NS3} route add ${IPv4_3}/32 dev veth4
228	ip -netns ${NS3} route add ${IPv4_1}/32 dev veth4 via ${IPv4_3}
229	ip -netns ${NS3} route add ${IPv4_2}/32 dev veth4 via ${IPv4_3}
230	ip -netns ${NS3} -6 route add ${IPv6_3}/128 dev veth4
231	ip -netns ${NS3} -6 route add ${IPv6_1}/128 dev veth4 via ${IPv6_3}
232	ip -netns ${NS3} -6 route add ${IPv6_2}/128 dev veth4 via ${IPv6_3}
233	# bottom route
234	ip -netns ${NS3} route add ${IPv4_7}/32 dev veth8
235	ip -netns ${NS3} route add ${IPv4_5}/32 dev veth8 via ${IPv4_7}
236	ip -netns ${NS3} route add ${IPv4_6}/32 dev veth8 via ${IPv4_7}
237	ip -netns ${NS3} -6 route add ${IPv6_7}/128 dev veth8
238	ip -netns ${NS3} -6 route add ${IPv6_5}/128 dev veth8 via ${IPv6_7}
239	ip -netns ${NS3} -6 route add ${IPv6_6}/128 dev veth8 via ${IPv6_7}
240
241	# configure IPv4 GRE device in NS3, and a route to it via the "bottom" route
242	ip -netns ${NS3} tunnel add gre_dev mode gre remote ${IPv4_1} local ${IPv4_GRE} ttl 255
243	ip -netns ${NS3} link set gre_dev up
244	ip -netns ${NS3} addr add ${IPv4_GRE} dev gre_dev
245	ip -netns ${NS1} route add ${IPv4_GRE}/32 dev veth5 via ${IPv4_6} ${VRF}
246	ip -netns ${NS2} route add ${IPv4_GRE}/32 dev veth7 via ${IPv4_8} ${VRF}
247
248
249	# configure IPv6 GRE device in NS3, and a route to it via the "bottom" route
250	ip -netns ${NS3} -6 tunnel add name gre6_dev mode ip6gre remote ${IPv6_1} local ${IPv6_GRE} ttl 255
251	ip -netns ${NS3} link set gre6_dev up
252	ip -netns ${NS3} -6 addr add ${IPv6_GRE} nodad dev gre6_dev
253	ip -netns ${NS1} -6 route add ${IPv6_GRE}/128 dev veth5 via ${IPv6_6} ${VRF}
254	ip -netns ${NS2} -6 route add ${IPv6_GRE}/128 dev veth7 via ${IPv6_8} ${VRF}
255
256	TMPFILE=$(mktemp /tmp/test_lwt_ip_encap.XXXXXX)
257
258	sleep 1 # reduce flakiness
259	set +e
260	}
261
262	cleanup()
263	{
264	if [ -f ${TMPFILE} ] ; then
265	rm ${TMPFILE}
266	fi
267
268	ip netns del ${NS1} 2> /dev/null
269	ip netns del ${NS2} 2> /dev/null
270	ip netns del ${NS3} 2> /dev/null
271	}
272
273	trap cleanup EXIT
274
275	remove_routes_to_gredev()
276	{
277	ip -netns ${NS1} route del ${IPv4_GRE} dev veth5 ${VRF}
278	ip -netns ${NS2} route del ${IPv4_GRE} dev veth7 ${VRF}
279	ip -netns ${NS1} -6 route del ${IPv6_GRE}/128 dev veth5 ${VRF}
280	ip -netns ${NS2} -6 route del ${IPv6_GRE}/128 dev veth7 ${VRF}
281	}
282
283	add_unreachable_routes_to_gredev()
284	{
285	ip -netns ${NS1} route add unreachable ${IPv4_GRE}/32 ${VRF}
286	ip -netns ${NS2} route add unreachable ${IPv4_GRE}/32 ${VRF}
287	ip -netns ${NS1} -6 route add unreachable ${IPv6_GRE}/128 ${VRF}
288	ip -netns ${NS2} -6 route add unreachable ${IPv6_GRE}/128 ${VRF}
289	}
290
291	test_ping()
292	{
293	local readonly PROTO=$1
294	local readonly EXPECTED=$2
295	local RET=0
296
297	if [ "${PROTO}" == "IPv4" ] ; then
298	ip netns exec ${NS1} ping -c 1 -W 1 -I veth1 ${IPv4_DST} 2>&1 > /dev/null
299	RET=$?
300	elif [ "${PROTO}" == "IPv6" ] ; then
301	ip netns exec ${NS1} ping6 -c 1 -W 1 -I veth1 ${IPv6_DST} 2>&1 > /dev/null
302	RET=$?
303	else
304	echo " test_ping: unknown PROTO: ${PROTO}"
305	TEST_STATUS=1
306	fi
307
308	if [ "0" != "${RET}" ]; then
309	RET=1
310	fi
311
312	if [ "${EXPECTED}" != "${RET}" ] ; then
313	echo " test_ping failed: expected: ${EXPECTED}; got ${RET}"
314	TEST_STATUS=1
315	fi
316	}
317
318	test_gso()
319	{
320	local readonly PROTO=$1
321	local readonly PKT_SZ=5000
322	local IP_DST=""
323	: > ${TMPFILE} # trim the capture file
324
325	# check that nc is present
326	command -v nc >/dev/null 2>&1 \|\| \
327	{ echo >&2 "nc is not available: skipping TSO tests"; return; }
328
329	# listen on port 9000, capture TCP into $TMPFILE
330	if [ "${PROTO}" == "IPv4" ] ; then
331	IP_DST=${IPv4_DST}
332	ip netns exec ${NS3} bash -c \
333	"nc -4 -l -p 9000 > ${TMPFILE} &"
334	elif [ "${PROTO}" == "IPv6" ] ; then
335	IP_DST=${IPv6_DST}
336	ip netns exec ${NS3} bash -c \
337	"nc -6 -l -p 9000 > ${TMPFILE} &"
338	RET=$?
339	else
340	echo " test_gso: unknown PROTO: ${PROTO}"
341	TEST_STATUS=1
342	fi
343	sleep 1 # let nc start listening
344
345	# send a packet larger than MTU
346	ip netns exec ${NS1} bash -c \
347	"dd if=/dev/zero bs=$PKT_SZ count=1 > /dev/tcp/${IP_DST}/9000 2>/dev/null"
348	sleep 2 # let the packet get delivered
349
350	# verify we received all expected bytes
351	SZ=$(stat -c %s ${TMPFILE})
352	if [ "$SZ" != "$PKT_SZ" ] ; then
353	echo " test_gso failed: ${PROTO}"
354	TEST_STATUS=1
355	fi
356	}
357
358	test_egress()
359	{
360	local readonly ENCAP=$1
361	echo "starting egress ${ENCAP} encap test ${VRF}"
362	setup
363
364	# by default, pings work
365	test_ping IPv4 0
366	test_ping IPv6 0
367
368	# remove NS2->DST routes, ping fails
369	ip -netns ${NS2} route del ${IPv4_DST}/32 dev veth3 ${VRF}
370	ip -netns ${NS2} -6 route del ${IPv6_DST}/128 dev veth3 ${VRF}
371	test_ping IPv4 1
372	test_ping IPv6 1
373
374	# install replacement routes (LWT/eBPF), pings succeed
375	if [ "${ENCAP}" == "IPv4" ] ; then
376	ip -netns ${NS1} route add ${IPv4_DST} encap bpf xmit obj \
377	${BPF_FILE} sec encap_gre dev veth1 ${VRF}
378	ip -netns ${NS1} -6 route add ${IPv6_DST} encap bpf xmit obj \
379	${BPF_FILE} sec encap_gre dev veth1 ${VRF}
380	elif [ "${ENCAP}" == "IPv6" ] ; then
381	ip -netns ${NS1} route add ${IPv4_DST} encap bpf xmit obj \
382	${BPF_FILE} sec encap_gre6 dev veth1 ${VRF}
383	ip -netns ${NS1} -6 route add ${IPv6_DST} encap bpf xmit obj \
384	${BPF_FILE} sec encap_gre6 dev veth1 ${VRF}
385	else
386	echo " unknown encap ${ENCAP}"
387	TEST_STATUS=1
388	fi
389	test_ping IPv4 0
390	test_ping IPv6 0
391
392	# skip GSO tests with VRF: VRF routing needs properly assigned
393	# source IP/device, which is easy to do with ping and hard with dd/nc.
394	if [ -z "${VRF}" ] ; then
395	test_gso IPv4
396	test_gso IPv6
397	fi
398
399	# a negative test: remove routes to GRE devices: ping fails
400	remove_routes_to_gredev
401	test_ping IPv4 1
402	test_ping IPv6 1
403
404	# another negative test
405	add_unreachable_routes_to_gredev
406	test_ping IPv4 1
407	test_ping IPv6 1
408
409	cleanup
410	process_test_results
411	}
412
413	test_ingress()
414	{
415	local readonly ENCAP=$1
416	echo "starting ingress ${ENCAP} encap test ${VRF}"
417	setup
418
419	# need to wait a bit for IPv6 to autoconf, otherwise
420	# ping6 sometimes fails with "unable to bind to address"
421
422	# by default, pings work
423	test_ping IPv4 0
424	test_ping IPv6 0
425
426	# remove NS2->DST routes, pings fail
427	ip -netns ${NS2} route del ${IPv4_DST}/32 dev veth3 ${VRF}
428	ip -netns ${NS2} -6 route del ${IPv6_DST}/128 dev veth3 ${VRF}
429	test_ping IPv4 1
430	test_ping IPv6 1
431
432	# install replacement routes (LWT/eBPF), pings succeed
433	if [ "${ENCAP}" == "IPv4" ] ; then
434	ip -netns ${NS2} route add ${IPv4_DST} encap bpf in obj \
435	${BPF_FILE} sec encap_gre dev veth2 ${VRF}
436	ip -netns ${NS2} -6 route add ${IPv6_DST} encap bpf in obj \
437	${BPF_FILE} sec encap_gre dev veth2 ${VRF}
438	elif [ "${ENCAP}" == "IPv6" ] ; then
439	ip -netns ${NS2} route add ${IPv4_DST} encap bpf in obj \
440	${BPF_FILE} sec encap_gre6 dev veth2 ${VRF}
441	ip -netns ${NS2} -6 route add ${IPv6_DST} encap bpf in obj \
442	${BPF_FILE} sec encap_gre6 dev veth2 ${VRF}
443	else
444	echo "FAIL: unknown encap ${ENCAP}"
445	TEST_STATUS=1
446	fi
447	test_ping IPv4 0
448	test_ping IPv6 0
449
450	# a negative test: remove routes to GRE devices: ping fails
451	remove_routes_to_gredev
452	test_ping IPv4 1
453	test_ping IPv6 1
454
455	# another negative test
456	add_unreachable_routes_to_gredev
457	test_ping IPv4 1
458	test_ping IPv6 1
459
460	cleanup
461	process_test_results
462	}
463
464	VRF=""
465	test_egress IPv4
466	test_egress IPv6
467	test_ingress IPv4
468	test_ingress IPv6
469
470	VRF="vrf red"
471	test_egress IPv4
472	test_egress IPv6
473	test_ingress IPv4
474	test_ingress IPv6
475
476	print_test_summary_and_exit

tools/testing/selftests/bpf/test_lwt_seg6local.sh+0−156

@@ -1,156 +0,0 @@
1	#!/bin/bash
2	# Connects 6 network namespaces through veths.
3	# Each NS may have different IPv6 global scope addresses :
4	# NS1 ---- NS2 ---- NS3 ---- NS4 ---- NS5 ---- NS6
5	# fb00::1 fd00::1 fd00::2 fd00::3 fb00::6
6	# fc42::1 fd00::4
7	#
8	# All IPv6 packets going to fb00::/16 through NS2 will be encapsulated in a
9	# IPv6 header with a Segment Routing Header, with segments :
10	# fd00::1 -> fd00::2 -> fd00::3 -> fd00::4
11	#
12	# 3 fd00::/16 IPv6 addresses are binded to seg6local End.BPF actions :
13	# - fd00::1 : add a TLV, change the flags and apply a End.X action to fc42::1
14	# - fd00::2 : remove the TLV, change the flags, add a tag
15	# - fd00::3 : apply an End.T action to fd00::4, through routing table 117
16	#
17	# fd00::4 is a simple Segment Routing node decapsulating the inner IPv6 packet.
18	# Each End.BPF action will validate the operations applied on the SRH by the
19	# previous BPF program in the chain, otherwise the packet is dropped.
20	#
21	# An UDP datagram is sent from fb00::1 to fb00::6. The test succeeds if this
22	# datagram can be read on NS6 when binding to fb00::6.
23
24	# Kselftest framework requirement - SKIP code is 4.
25	ksft_skip=4
26	BPF_FILE="test_lwt_seg6local.bpf.o"
27	readonly NS1="ns1-$(mktemp -u XXXXXX)"
28	readonly NS2="ns2-$(mktemp -u XXXXXX)"
29	readonly NS3="ns3-$(mktemp -u XXXXXX)"
30	readonly NS4="ns4-$(mktemp -u XXXXXX)"
31	readonly NS5="ns5-$(mktemp -u XXXXXX)"
32	readonly NS6="ns6-$(mktemp -u XXXXXX)"
33
34	msg="skip all tests:"
35	if [ $UID != 0 ]; then
36	echo $msg please run this as root >&2
37	exit $ksft_skip
38	fi
39
40	TMP_FILE="/tmp/selftest_lwt_seg6local.txt"
41
42	cleanup()
43	{
44	if [ "$?" = "0" ]; then
45	echo "selftests: test_lwt_seg6local [PASS]";
46	else
47	echo "selftests: test_lwt_seg6local [FAILED]";
48	fi
49
50	set +e
51	ip netns del ${NS1} 2> /dev/null
52	ip netns del ${NS2} 2> /dev/null
53	ip netns del ${NS3} 2> /dev/null
54	ip netns del ${NS4} 2> /dev/null
55	ip netns del ${NS5} 2> /dev/null
56	ip netns del ${NS6} 2> /dev/null
57	rm -f $TMP_FILE
58	}
59
60	set -e
61
62	ip netns add ${NS1}
63	ip netns add ${NS2}
64	ip netns add ${NS3}
65	ip netns add ${NS4}
66	ip netns add ${NS5}
67	ip netns add ${NS6}
68
69	trap cleanup 0 2 3 6 9
70
71	ip link add veth1 type veth peer name veth2
72	ip link add veth3 type veth peer name veth4
73	ip link add veth5 type veth peer name veth6
74	ip link add veth7 type veth peer name veth8
75	ip link add veth9 type veth peer name veth10
76
77	ip link set veth1 netns ${NS1}
78	ip link set veth2 netns ${NS2}
79	ip link set veth3 netns ${NS2}
80	ip link set veth4 netns ${NS3}
81	ip link set veth5 netns ${NS3}
82	ip link set veth6 netns ${NS4}
83	ip link set veth7 netns ${NS4}
84	ip link set veth8 netns ${NS5}
85	ip link set veth9 netns ${NS5}
86	ip link set veth10 netns ${NS6}
87
88	ip netns exec ${NS1} ip link set dev veth1 up
89	ip netns exec ${NS2} ip link set dev veth2 up
90	ip netns exec ${NS2} ip link set dev veth3 up
91	ip netns exec ${NS3} ip link set dev veth4 up
92	ip netns exec ${NS3} ip link set dev veth5 up
93	ip netns exec ${NS4} ip link set dev veth6 up
94	ip netns exec ${NS4} ip link set dev veth7 up
95	ip netns exec ${NS5} ip link set dev veth8 up
96	ip netns exec ${NS5} ip link set dev veth9 up
97	ip netns exec ${NS6} ip link set dev veth10 up
98	ip netns exec ${NS6} ip link set dev lo up
99
100	# All link scope addresses and routes required between veths
101	ip netns exec ${NS1} ip -6 addr add fb00::12/16 dev veth1 scope link
102	ip netns exec ${NS1} ip -6 route add fb00::21 dev veth1 scope link
103	ip netns exec ${NS2} ip -6 addr add fb00::21/16 dev veth2 scope link
104	ip netns exec ${NS2} ip -6 addr add fb00::34/16 dev veth3 scope link
105	ip netns exec ${NS2} ip -6 route add fb00::43 dev veth3 scope link
106	ip netns exec ${NS3} ip -6 route add fb00::65 dev veth5 scope link
107	ip netns exec ${NS3} ip -6 addr add fb00::43/16 dev veth4 scope link
108	ip netns exec ${NS3} ip -6 addr add fb00::56/16 dev veth5 scope link
109	ip netns exec ${NS4} ip -6 addr add fb00::65/16 dev veth6 scope link
110	ip netns exec ${NS4} ip -6 addr add fb00::78/16 dev veth7 scope link
111	ip netns exec ${NS4} ip -6 route add fb00::87 dev veth7 scope link
112	ip netns exec ${NS5} ip -6 addr add fb00::87/16 dev veth8 scope link
113	ip netns exec ${NS5} ip -6 addr add fb00::910/16 dev veth9 scope link
114	ip netns exec ${NS5} ip -6 route add fb00::109 dev veth9 scope link
115	ip netns exec ${NS5} ip -6 route add fb00::109 table 117 dev veth9 scope link
116	ip netns exec ${NS6} ip -6 addr add fb00::109/16 dev veth10 scope link
117
118	ip netns exec ${NS1} ip -6 addr add fb00::1/16 dev lo
119	ip netns exec ${NS1} ip -6 route add fb00::6 dev veth1 via fb00::21
120
121	ip netns exec ${NS2} ip -6 route add fb00::6 encap bpf in obj ${BPF_FILE} sec encap_srh dev veth2
122	ip netns exec ${NS2} ip -6 route add fd00::1 dev veth3 via fb00::43 scope link
123
124	ip netns exec ${NS3} ip -6 route add fc42::1 dev veth5 via fb00::65
125	ip netns exec ${NS3} ip -6 route add fd00::1 encap seg6local action End.BPF endpoint obj ${BPF_FILE} sec add_egr_x dev veth4
126
127	ip netns exec ${NS4} ip -6 route add fd00::2 encap seg6local action End.BPF endpoint obj ${BPF_FILE} sec pop_egr dev veth6
128	ip netns exec ${NS4} ip -6 addr add fc42::1 dev lo
129	ip netns exec ${NS4} ip -6 route add fd00::3 dev veth7 via fb00::87
130
131	ip netns exec ${NS5} ip -6 route add fd00::4 table 117 dev veth9 via fb00::109
132	ip netns exec ${NS5} ip -6 route add fd00::3 encap seg6local action End.BPF endpoint obj ${BPF_FILE} sec inspect_t dev veth8
133
134	ip netns exec ${NS6} ip -6 addr add fb00::6/16 dev lo
135	ip netns exec ${NS6} ip -6 addr add fd00::4/16 dev lo
136
137	ip netns exec ${NS1} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
138	ip netns exec ${NS2} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
139	ip netns exec ${NS3} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
140	ip netns exec ${NS4} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
141	ip netns exec ${NS5} sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
142
143	ip netns exec ${NS6} sysctl net.ipv6.conf.all.seg6_enabled=1 > /dev/null
144	ip netns exec ${NS6} sysctl net.ipv6.conf.lo.seg6_enabled=1 > /dev/null
145	ip netns exec ${NS6} sysctl net.ipv6.conf.veth10.seg6_enabled=1 > /dev/null
146
147	ip netns exec ${NS6} nc -l -6 -u -d 7330 > $TMP_FILE &
148	ip netns exec ${NS1} bash -c "echo 'foobar' \| nc -w0 -6 -u -p 2121 -s fb00::1 fb00::6 7330"
149	sleep 5 # wait enough time to ensure the UDP datagram arrived to the last segment
150	kill -TERM $!
151
152	if [[ $(< $TMP_FILE) != "foobar" ]]; then
153	exit 1
154	fi
155
156	exit 0

tools/testing/selftests/bpf/test_maps.c+5−4

 #define MAX_DELAY_US 50000
 #define MIN_DELAY_RANGE_US 5000
-static bool retry_for_again_or_busy(int err)
+static bool can_retry(int err)
 {
-	return (err == EAGAIN || err == EBUSY);
+	return (err == EAGAIN || err == EBUSY ||
+		(err == ENOMEM && map_opts.map_flags == BPF_F_NO_PREALLOC));
 }
 int map_update_retriable(int map_fd, const void *key, const void *value, int flags, int attempts,
 		if (do_update) {
 			err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES,
-						   retry_for_again_or_busy);
+						   can_retry);
 			if (err)
 				printf("error %d %d\n", err, errno);
 			assert(err == 0);
 			err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES,
-						   retry_for_again_or_busy);
+						   can_retry);
 			if (err)
 				printf("error %d %d\n", err, errno);
 			assert(err == 0);

tools/testing/selftests/bpf/test_progs.c+43−29

 #endif
 }
-static void stdio_restore_cleanup(void)
+static pthread_mutex_t stdout_lock = PTHREAD_MUTEX_INITIALIZER;
+static void stdio_restore(void)
 {
 #ifdef __GLIBC__
 	if (verbose() && env.worker_id == -1) {
 	fflush(stdout);
+	pthread_mutex_lock(&stdout_lock);
 	if (env.subtest_state) {
-		fclose(env.subtest_state->stdout_saved);
+		if (env.subtest_state->stdout_saved)
+			fclose(env.subtest_state->stdout_saved);
 		env.subtest_state->stdout_saved = NULL;
 		stdout = env.test_state->stdout_saved;
 		stderr = env.test_state->stdout_saved;
 	} else {
-		fclose(env.test_state->stdout_saved);
+		if (env.test_state->stdout_saved)
+			fclose(env.test_state->stdout_saved);
 		env.test_state->stdout_saved = NULL;
+		stdout = env.stdout_saved;
+		stderr = env.stderr_saved;
 	}
+	pthread_mutex_unlock(&stdout_lock);
 #endif
 }
-static void stdio_restore(void)
+static int traffic_monitor_print_fn(const char *format, va_list args)
 {
-#ifdef __GLIBC__
+	pthread_mutex_lock(&stdout_lock);
-	if (verbose() && env.worker_id == -1) {
+	vfprintf(stdout, format, args);
-		/* nothing to do, output to stdout by default */
+	pthread_mutex_unlock(&stdout_lock);
-		return;
-	}
-	if (stdout == env.stdout_saved)
-		return;
-	stdio_restore_cleanup();
-	stdout = env.stdout_saved;
+	return 0;
-	stderr = env.stderr_saved;
-#endif
 }
 /* Adapted from perf/util/string.c */
 	print_test_result(test, test_state);
 }
-static void stdio_restore(void);
 /* A bunch of tests set custom affinity per-thread and/or per-process. Reset
  * it after each test/sub-test.
  */
 	err = sched_setaffinity(0, sizeof(cpuset), &cpuset);
 	if (err < 0) {
-		stdio_restore();
 		fprintf(stderr, "Failed to reset process affinity: %d!\n", err);
 		exit(EXIT_ERR_SETUP_INFRA);
 	}
 	err = pthread_setaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
 	if (err < 0) {
-		stdio_restore();
 		fprintf(stderr, "Failed to reset thread affinity: %d!\n", err);
 		exit(EXIT_ERR_SETUP_INFRA);
 	}
 static void restore_netns(void)
 {
 	if (setns(env.saved_netns_fd, CLONE_NEWNET) == -1) {
-		stdio_restore();
 		perror("setns(CLONE_NEWNS)");
 		exit(EXIT_ERR_SETUP_INFRA);
 	}
 				   test_result(subtest_state->error_cnt,
 					       subtest_state->skipped));
-	stdio_restore_cleanup();
+	stdio_restore();
 	env.subtest_state = NULL;
 }
 	sz = backtrace(bt, ARRAY_SIZE(bt));
-	if (env.stdout_saved)
+	fflush(stdout);
-		stdio_restore();
+	stdout = env.stdout_saved;
+	stderr = env.stderr_saved;
 	if (env.test) {
 		env.test_state->error_cnt++;
 		dump_test_log(env.test, env.test_state, true, false, NULL);
 	return ret;
 }
+static bool ns_is_needed(const char *test_name)
+{
+	if (strlen(test_name) < 3)
+		return false;
+	return !strncmp(test_name, "ns_", 3);
+}
 static void run_one_test(int test_num)
 {
 	struct prog_test_def *test = &prog_test_defs[test_num];
 	struct test_state *state = &test_states[test_num];
+	struct netns_obj *ns = NULL;
 	env.test = test;
 	env.test_state = state;
 	stdio_hijack(&state->log_buf, &state->log_cnt);
 	watchdog_start();
+	if (ns_is_needed(test->test_name))
+		ns = netns_new(test->test_name, true);
 	if (test->run_test)
 		test->run_test();
 	else if (test->run_serial_test)
 		test->run_serial_test();
+	netns_free(ns);
 	watchdog_stop();
 	/* ensure last sub-test is finalized properly */
 	state->tested = true;
+	stdio_restore();
 	if (verbose() && env.worker_id == -1)
 		print_test_result(test, state);
 	if (test->need_cgroup_cleanup)
 		cleanup_cgroup_environment();
-	stdio_restore();
 	free(stop_libbpf_log_capture());
 	dump_test_log(test, state, false, false, NULL);
 	sigaction(SIGSEGV, &sigact, NULL);
+	env.stdout_saved = stdout;
+	env.stderr_saved = stderr;
 	env.secs_till_notify = 10;
 	env.secs_till_kill = 120;
 	err = argp_parse(&argp, argc, argv, 0, NULL, &env);
 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
 	libbpf_set_print(libbpf_print_fn);
+	traffic_monitor_set_print(traffic_monitor_print_fn);
 	srand(time(NULL));
 	env.jit_enabled = is_jit_enabled();
 		return -1;
 	}
-	env.stdout_saved = stdout;
-	env.stderr_saved = stderr;
 	env.has_testmod = true;
 	if (!env.list_test_names) {
 		/* ensure previous instance of the module is unloaded */

tools/testing/selftests/bpf/test_progs.h+8−0

@@ -427,6 +427,14 @@ void hexdump(const char prefix, const void buf, size_t len); · 426 unmodified lines
427	goto goto_label; \	427	goto goto_label; \
428	})	428	})
429		429
		430	#define SYS_FAIL(goto_label, fmt, ...) \
		431	({ \
		432	char cmd[1024]; \
		433	snprintf(cmd, sizeof(cmd), fmt, ##__VA_ARGS__); \
		434	if (!ASSERT_NEQ(0, system(cmd), cmd)) \
		435	goto goto_label; \
		436	})
		437
430	#define ALL_TO_DEV_NULL " >/dev/null 2>&1"	438	#define ALL_TO_DEV_NULL " >/dev/null 2>&1"
431		439
432	#define SYS_NOFAIL(fmt, ...) \	440	#define SYS_NOFAIL(fmt, ...) \

tools/testing/selftests/bpf/test_tunnel.sh+0−645

@@ -1,645 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3
4	# End-to-end eBPF tunnel test suite
5	# The script tests BPF network tunnel implementation.
6	#
7	# Topology:
8	# ---------
9	# root namespace \| at_ns0 namespace
10	# \|
11	# ----------- \| -----------
12	# \| tnl dev \| \| \| tnl dev \| (overlay network)
13	# ----------- \| -----------
14	# metadata-mode \| native-mode
15	# with bpf \|
16	# \|
17	# ---------- \| ----------
18	# \| veth1 \| --------- \| veth0 \| (underlay network)
19	# ---------- peer ----------
20	#
21	#
22	# Device Configuration
23	# --------------------
24	# Root namespace with metadata-mode tunnel + BPF
25	# Device names and addresses:
26	# veth1 IP: 172.16.1.200, IPv6: 00::22 (underlay)
27	# tunnel dev <type>11, ex: gre11, IPv4: 10.1.1.200, IPv6: 1::22 (overlay)
28	#
29	# Namespace at_ns0 with native tunnel
30	# Device names and addresses:
31	# veth0 IPv4: 172.16.1.100, IPv6: 00::11 (underlay)
32	# tunnel dev <type>00, ex: gre00, IPv4: 10.1.1.100, IPv6: 1::11 (overlay)
33	#
34	#
35	# End-to-end ping packet flow
36	# ---------------------------
37	# Most of the tests start by namespace creation, device configuration,
38	# then ping the underlay and overlay network. When doing 'ping 10.1.1.100'
39	# from root namespace, the following operations happen:
40	# 1) Route lookup shows 10.1.1.100/24 belongs to tnl dev, fwd to tnl dev.
41	# 2) Tnl device's egress BPF program is triggered and set the tunnel metadata,
42	# with remote_ip=172.16.1.100 and others.
43	# 3) Outer tunnel header is prepended and route the packet to veth1's egress
44	# 4) veth0's ingress queue receive the tunneled packet at namespace at_ns0
45	# 5) Tunnel protocol handler, ex: vxlan_rcv, decap the packet
46	# 6) Forward the packet to the overlay tnl dev
47
48	BPF_FILE="test_tunnel_kern.bpf.o"
49	BPF_PIN_TUNNEL_DIR="/sys/fs/bpf/tc/tunnel"
50	PING_ARG="-c 3 -w 10 -q"
51	ret=0
52	GREEN='\033[0;92m'
53	RED='\033[0;31m'
54	NC='\033[0m' # No Color
55
56	config_device()
57	{
58	ip netns add at_ns0
59	ip link add veth0 type veth peer name veth1
60	ip link set veth0 netns at_ns0
61	ip netns exec at_ns0 ip addr add 172.16.1.100/24 dev veth0
62	ip netns exec at_ns0 ip link set dev veth0 up
63	ip link set dev veth1 up mtu 1500
64	ip addr add dev veth1 172.16.1.200/24
65	}
66
67	add_gre_tunnel()
68	{
69	tun_key=
70	if [ -n "$1" ]; then
71	tun_key="key $1"
72	fi
73
74	# at_ns0 namespace
75	ip netns exec at_ns0 \
76	ip link add dev $DEV_NS type $TYPE seq $tun_key \
77	local 172.16.1.100 remote 172.16.1.200
78	ip netns exec at_ns0 ip link set dev $DEV_NS up
79	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
80
81	# root namespace
82	ip link add dev $DEV type $TYPE $tun_key external
83	ip link set dev $DEV up
84	ip addr add dev $DEV 10.1.1.200/24
85	}
86
87	add_ip6gretap_tunnel()
88	{
89
90	# assign ipv6 address
91	ip netns exec at_ns0 ip addr add ::11/96 dev veth0
92	ip netns exec at_ns0 ip link set dev veth0 up
93	ip addr add dev veth1 ::22/96
94	ip link set dev veth1 up
95
96	# at_ns0 namespace
97	ip netns exec at_ns0 \
98	ip link add dev $DEV_NS type $TYPE seq flowlabel 0xbcdef key 2 \
99	local ::11 remote ::22
100
101	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
102	ip netns exec at_ns0 ip addr add dev $DEV_NS fc80::100/96
103	ip netns exec at_ns0 ip link set dev $DEV_NS up
104
105	# root namespace
106	ip link add dev $DEV type $TYPE external
107	ip addr add dev $DEV 10.1.1.200/24
108	ip addr add dev $DEV fc80::200/24
109	ip link set dev $DEV up
110	}
111
112	add_erspan_tunnel()
113	{
114	# at_ns0 namespace
115	if [ "$1" == "v1" ]; then
116	ip netns exec at_ns0 \
117	ip link add dev $DEV_NS type $TYPE seq key 2 \
118	local 172.16.1.100 remote 172.16.1.200 \
119	erspan_ver 1 erspan 123
120	else
121	ip netns exec at_ns0 \
122	ip link add dev $DEV_NS type $TYPE seq key 2 \
123	local 172.16.1.100 remote 172.16.1.200 \
124	erspan_ver 2 erspan_dir egress erspan_hwid 3
125	fi
126	ip netns exec at_ns0 ip link set dev $DEV_NS up
127	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
128
129	# root namespace
130	ip link add dev $DEV type $TYPE external
131	ip link set dev $DEV up
132	ip addr add dev $DEV 10.1.1.200/24
133	}
134
135	add_ip6erspan_tunnel()
136	{
137
138	# assign ipv6 address
139	ip netns exec at_ns0 ip addr add ::11/96 dev veth0
140	ip netns exec at_ns0 ip link set dev veth0 up
141	ip addr add dev veth1 ::22/96
142	ip link set dev veth1 up
143
144	# at_ns0 namespace
145	if [ "$1" == "v1" ]; then
146	ip netns exec at_ns0 \
147	ip link add dev $DEV_NS type $TYPE seq key 2 \
148	local ::11 remote ::22 \
149	erspan_ver 1 erspan 123
150	else
151	ip netns exec at_ns0 \
152	ip link add dev $DEV_NS type $TYPE seq key 2 \
153	local ::11 remote ::22 \
154	erspan_ver 2 erspan_dir egress erspan_hwid 7
155	fi
156	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
157	ip netns exec at_ns0 ip link set dev $DEV_NS up
158
159	# root namespace
160	ip link add dev $DEV type $TYPE external
161	ip addr add dev $DEV 10.1.1.200/24
162	ip link set dev $DEV up
163	}
164
165	add_geneve_tunnel()
166	{
167	# at_ns0 namespace
168	ip netns exec at_ns0 \
169	ip link add dev $DEV_NS type $TYPE \
170	id 2 dstport 6081 remote 172.16.1.200
171	ip netns exec at_ns0 ip link set dev $DEV_NS up
172	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
173
174	# root namespace
175	ip link add dev $DEV type $TYPE dstport 6081 external
176	ip link set dev $DEV up
177	ip addr add dev $DEV 10.1.1.200/24
178	}
179
180	add_ip6geneve_tunnel()
181	{
182	ip netns exec at_ns0 ip addr add ::11/96 dev veth0
183	ip netns exec at_ns0 ip link set dev veth0 up
184	ip addr add dev veth1 ::22/96
185	ip link set dev veth1 up
186
187	# at_ns0 namespace
188	ip netns exec at_ns0 \
189	ip link add dev $DEV_NS type $TYPE id 22 \
190	remote ::22 # geneve has no local option
191	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
192	ip netns exec at_ns0 ip link set dev $DEV_NS up
193
194	# root namespace
195	ip link add dev $DEV type $TYPE external
196	ip addr add dev $DEV 10.1.1.200/24
197	ip link set dev $DEV up
198	}
199
200	add_ipip_tunnel()
201	{
202	# at_ns0 namespace
203	ip netns exec at_ns0 \
204	ip link add dev $DEV_NS type $TYPE \
205	local 172.16.1.100 remote 172.16.1.200
206	ip netns exec at_ns0 ip link set dev $DEV_NS up
207	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
208
209	# root namespace
210	ip link add dev $DEV type $TYPE external
211	ip link set dev $DEV up
212	ip addr add dev $DEV 10.1.1.200/24
213	}
214
215	add_ip6tnl_tunnel()
216	{
217	ip netns exec at_ns0 ip addr add ::11/96 dev veth0
218	ip netns exec at_ns0 ip link set dev veth0 up
219	ip addr add dev veth1 ::22/96
220	ip link set dev veth1 up
221
222	# at_ns0 namespace
223	ip netns exec at_ns0 \
224	ip link add dev $DEV_NS type $TYPE \
225	local ::11 remote ::22
226	ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
227	ip netns exec at_ns0 ip addr add dev $DEV_NS 1::11/96
228	ip netns exec at_ns0 ip link set dev $DEV_NS up
229
230	# root namespace
231	ip link add dev $DEV type $TYPE external
232	ip addr add dev $DEV 10.1.1.200/24
233	ip addr add dev $DEV 1::22/96
234	ip link set dev $DEV up
235	}
236
237	test_gre()
238	{
239	TYPE=gretap
240	DEV_NS=gretap00
241	DEV=gretap11
242	ret=0
243
244	check $TYPE
245	config_device
246	add_gre_tunnel 2
247	attach_bpf $DEV gre_set_tunnel gre_get_tunnel
248	ping $PING_ARG 10.1.1.100
249	check_err $?
250	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
251	check_err $?
252	cleanup
253
254	if [ $ret -ne 0 ]; then
255	echo -e ${RED}"FAIL: $TYPE"${NC}
256	return 1
257	fi
258	echo -e ${GREEN}"PASS: $TYPE"${NC}
259	}
260
261	test_gre_no_tunnel_key()
262	{
263	TYPE=gre
264	DEV_NS=gre00
265	DEV=gre11
266	ret=0
267
268	check $TYPE
269	config_device
270	add_gre_tunnel
271	attach_bpf $DEV gre_set_tunnel_no_key gre_get_tunnel
272	ping $PING_ARG 10.1.1.100
273	check_err $?
274	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
275	check_err $?
276	cleanup
277
278	if [ $ret -ne 0 ]; then
279	echo -e ${RED}"FAIL: $TYPE"${NC}
280	return 1
281	fi
282	echo -e ${GREEN}"PASS: $TYPE"${NC}
283	}
284
285	test_ip6gre()
286	{
287	TYPE=ip6gre
288	DEV_NS=ip6gre00
289	DEV=ip6gre11
290	ret=0
291
292	check $TYPE
293	config_device
294	# reuse the ip6gretap function
295	add_ip6gretap_tunnel
296	attach_bpf $DEV ip6gretap_set_tunnel ip6gretap_get_tunnel
297	# underlay
298	ping6 $PING_ARG ::11
299	# overlay: ipv4 over ipv6
300	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
301	ping $PING_ARG 10.1.1.100
302	check_err $?
303	# overlay: ipv6 over ipv6
304	ip netns exec at_ns0 ping6 $PING_ARG fc80::200
305	check_err $?
306	cleanup
307
308	if [ $ret -ne 0 ]; then
309	echo -e ${RED}"FAIL: $TYPE"${NC}
310	return 1
311	fi
312	echo -e ${GREEN}"PASS: $TYPE"${NC}
313	}
314
315	test_ip6gretap()
316	{
317	TYPE=ip6gretap
318	DEV_NS=ip6gretap00
319	DEV=ip6gretap11
320	ret=0
321
322	check $TYPE
323	config_device
324	add_ip6gretap_tunnel
325	attach_bpf $DEV ip6gretap_set_tunnel ip6gretap_get_tunnel
326	# underlay
327	ping6 $PING_ARG ::11
328	# overlay: ipv4 over ipv6
329	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
330	ping $PING_ARG 10.1.1.100
331	check_err $?
332	# overlay: ipv6 over ipv6
333	ip netns exec at_ns0 ping6 $PING_ARG fc80::200
334	check_err $?
335	cleanup
336
337	if [ $ret -ne 0 ]; then
338	echo -e ${RED}"FAIL: $TYPE"${NC}
339	return 1
340	fi
341	echo -e ${GREEN}"PASS: $TYPE"${NC}
342	}
343
344	test_erspan()
345	{
346	TYPE=erspan
347	DEV_NS=erspan00
348	DEV=erspan11
349	ret=0
350
351	check $TYPE
352	config_device
353	add_erspan_tunnel $1
354	attach_bpf $DEV erspan_set_tunnel erspan_get_tunnel
355	ping $PING_ARG 10.1.1.100
356	check_err $?
357	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
358	check_err $?
359	cleanup
360
361	if [ $ret -ne 0 ]; then
362	echo -e ${RED}"FAIL: $TYPE"${NC}
363	return 1
364	fi
365	echo -e ${GREEN}"PASS: $TYPE"${NC}
366	}
367
368	test_ip6erspan()
369	{
370	TYPE=ip6erspan
371	DEV_NS=ip6erspan00
372	DEV=ip6erspan11
373	ret=0
374
375	check $TYPE
376	config_device
377	add_ip6erspan_tunnel $1
378	attach_bpf $DEV ip4ip6erspan_set_tunnel ip4ip6erspan_get_tunnel
379	ping6 $PING_ARG ::11
380	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
381	check_err $?
382	cleanup
383
384	if [ $ret -ne 0 ]; then
385	echo -e ${RED}"FAIL: $TYPE"${NC}
386	return 1
387	fi
388	echo -e ${GREEN}"PASS: $TYPE"${NC}
389	}
390
391	test_geneve()
392	{
393	TYPE=geneve
394	DEV_NS=geneve00
395	DEV=geneve11
396	ret=0
397
398	check $TYPE
399	config_device
400	add_geneve_tunnel
401	attach_bpf $DEV geneve_set_tunnel geneve_get_tunnel
402	ping $PING_ARG 10.1.1.100
403	check_err $?
404	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
405	check_err $?
406	cleanup
407
408	if [ $ret -ne 0 ]; then
409	echo -e ${RED}"FAIL: $TYPE"${NC}
410	return 1
411	fi
412	echo -e ${GREEN}"PASS: $TYPE"${NC}
413	}
414
415	test_ip6geneve()
416	{
417	TYPE=geneve
418	DEV_NS=ip6geneve00
419	DEV=ip6geneve11
420	ret=0
421
422	check $TYPE
423	config_device
424	add_ip6geneve_tunnel
425	attach_bpf $DEV ip6geneve_set_tunnel ip6geneve_get_tunnel
426	ping $PING_ARG 10.1.1.100
427	check_err $?
428	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
429	check_err $?
430	cleanup
431
432	if [ $ret -ne 0 ]; then
433	echo -e ${RED}"FAIL: ip6$TYPE"${NC}
434	return 1
435	fi
436	echo -e ${GREEN}"PASS: ip6$TYPE"${NC}
437	}
438
439	test_ipip()
440	{
441	TYPE=ipip
442	DEV_NS=ipip00
443	DEV=ipip11
444	ret=0
445
446	check $TYPE
447	config_device
448	add_ipip_tunnel
449	ip link set dev veth1 mtu 1500
450	attach_bpf $DEV ipip_set_tunnel ipip_get_tunnel
451	ping $PING_ARG 10.1.1.100
452	check_err $?
453	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
454	check_err $?
455	cleanup
456
457	if [ $ret -ne 0 ]; then
458	echo -e ${RED}"FAIL: $TYPE"${NC}
459	return 1
460	fi
461	echo -e ${GREEN}"PASS: $TYPE"${NC}
462	}
463
464	test_ipip6()
465	{
466	TYPE=ip6tnl
467	DEV_NS=ipip6tnl00
468	DEV=ipip6tnl11
469	ret=0
470
471	check $TYPE
472	config_device
473	add_ip6tnl_tunnel
474	ip link set dev veth1 mtu 1500
475	attach_bpf $DEV ipip6_set_tunnel ipip6_get_tunnel
476	# underlay
477	ping6 $PING_ARG ::11
478	# ip4 over ip6
479	ping $PING_ARG 10.1.1.100
480	check_err $?
481	ip netns exec at_ns0 ping $PING_ARG 10.1.1.200
482	check_err $?
483	cleanup
484
485	if [ $ret -ne 0 ]; then
486	echo -e ${RED}"FAIL: $TYPE"${NC}
487	return 1
488	fi
489	echo -e ${GREEN}"PASS: $TYPE"${NC}
490	}
491
492	test_ip6ip6()
493	{
494	TYPE=ip6tnl
495	DEV_NS=ip6ip6tnl00
496	DEV=ip6ip6tnl11
497	ret=0
498
499	check $TYPE
500	config_device
501	add_ip6tnl_tunnel
502	ip link set dev veth1 mtu 1500
503	attach_bpf $DEV ip6ip6_set_tunnel ip6ip6_get_tunnel
504	# underlay
505	ping6 $PING_ARG ::11
506	# ip6 over ip6
507	ping6 $PING_ARG 1::11
508	check_err $?
509	ip netns exec at_ns0 ping6 $PING_ARG 1::22
510	check_err $?
511	cleanup
512
513	if [ $ret -ne 0 ]; then
514	echo -e ${RED}"FAIL: ip6$TYPE"${NC}
515	return 1
516	fi
517	echo -e ${GREEN}"PASS: ip6$TYPE"${NC}
518	}
519
520	attach_bpf()
521	{
522	DEV=$1
523	SET=$2
524	GET=$3
525	mkdir -p ${BPF_PIN_TUNNEL_DIR}
526	bpftool prog loadall ${BPF_FILE} ${BPF_PIN_TUNNEL_DIR}/
527	tc qdisc add dev $DEV clsact
528	tc filter add dev $DEV egress bpf da object-pinned ${BPF_PIN_TUNNEL_DIR}/$SET
529	tc filter add dev $DEV ingress bpf da object-pinned ${BPF_PIN_TUNNEL_DIR}/$GET
530	}
531
532	cleanup()
533	{
534	rm -rf ${BPF_PIN_TUNNEL_DIR}
535
536	ip netns delete at_ns0 2> /dev/null
537	ip link del veth1 2> /dev/null
538	ip link del ipip11 2> /dev/null
539	ip link del ipip6tnl11 2> /dev/null
540	ip link del ip6ip6tnl11 2> /dev/null
541	ip link del gretap11 2> /dev/null
542	ip link del gre11 2> /dev/null
543	ip link del ip6gre11 2> /dev/null
544	ip link del ip6gretap11 2> /dev/null
545	ip link del geneve11 2> /dev/null
546	ip link del ip6geneve11 2> /dev/null
547	ip link del erspan11 2> /dev/null
548	ip link del ip6erspan11 2> /dev/null
549	}
550
551	cleanup_exit()
552	{
553	echo "CATCH SIGKILL or SIGINT, cleanup and exit"
554	cleanup
555	exit 0
556	}
557
558	check()
559	{
560	ip link help 2>&1 \| grep -q "\s$1\s"
561	if [ $? -ne 0 ];then
562	echo "SKIP $1: iproute2 not support"
563	cleanup
564	return 1
565	fi
566	}
567
568	enable_debug()
569	{
570	echo 'file ip_gre.c +p' > /sys/kernel/debug/dynamic_debug/control
571	echo 'file ip6_gre.c +p' > /sys/kernel/debug/dynamic_debug/control
572	echo 'file geneve.c +p' > /sys/kernel/debug/dynamic_debug/control
573	echo 'file ipip.c +p' > /sys/kernel/debug/dynamic_debug/control
574	}
575
576	check_err()
577	{
578	if [ $ret -eq 0 ]; then
579	ret=$1
580	fi
581	}
582
583	bpf_tunnel_test()
584	{
585	local errors=0
586
587	echo "Testing GRE tunnel..."
588	test_gre
589	errors=$(( $errors + $? ))
590
591	echo "Testing GRE tunnel (without tunnel keys)..."
592	test_gre_no_tunnel_key
593	errors=$(( $errors + $? ))
594
595	echo "Testing IP6GRE tunnel..."
596	test_ip6gre
597	errors=$(( $errors + $? ))
598
599	echo "Testing IP6GRETAP tunnel..."
600	test_ip6gretap
601	errors=$(( $errors + $? ))
602
603	echo "Testing ERSPAN tunnel..."
604	test_erspan v2
605	errors=$(( $errors + $? ))
606
607	echo "Testing IP6ERSPAN tunnel..."
608	test_ip6erspan v2
609	errors=$(( $errors + $? ))
610
611	echo "Testing GENEVE tunnel..."
612	test_geneve
613	errors=$(( $errors + $? ))
614
615	echo "Testing IP6GENEVE tunnel..."
616	test_ip6geneve
617	errors=$(( $errors + $? ))
618
619	echo "Testing IPIP tunnel..."
620	test_ipip
621	errors=$(( $errors + $? ))
622
623	echo "Testing IPIP6 tunnel..."
624	test_ipip6
625	errors=$(( $errors + $? ))
626
627	echo "Testing IP6IP6 tunnel..."
628	test_ip6ip6
629	errors=$(( $errors + $? ))
630
631	return $errors
632	}
633
634	trap cleanup 0 3 6
635	trap cleanup_exit 2 9
636
637	cleanup
638	bpf_tunnel_test
639
640	if [ $? -ne 0 ]; then
641	echo -e "$(basename $0): ${RED}FAIL${NC}"
642	exit 1
643	fi
644	echo -e "$(basename $0): ${GREEN}PASS${NC}"
645	exit 0

tools/testing/selftests/bpf/test_xdp_redirect_multi.sh+0−214

@@ -1,214 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3	#
4	# Test topology:
5	# - - - - - - - - - - - - - - - - - - -
6	# \| veth1 veth2 veth3 \| ns0
7	# - -\| - - - - - - \| - - - - - - \| - -
8	# --------- --------- ---------
9	# \| veth0 \| \| veth0 \| \| veth0 \|
10	# --------- --------- ---------
11	# ns1 ns2 ns3
12	#
13	# Test modules:
14	# XDP modes: generic, native, native + egress_prog
15	#
16	# Test cases:
17	# ARP: Testing BPF_F_BROADCAST, the ingress interface also should receive
18	# the redirects.
19	# ns1 -> gw: ns1, ns2, ns3, should receive the arp request
20	# IPv4: Testing BPF_F_BROADCAST \| BPF_F_EXCLUDE_INGRESS, the ingress
21	# interface should not receive the redirects.
22	# ns1 -> gw: ns1 should not receive, ns2, ns3 should receive redirects.
23	# IPv6: Testing none flag, all the pkts should be redirected back
24	# ping test: ns1 -> ns2 (block), echo requests will be redirect back
25	# egress_prog:
26	# all src mac should be egress interface's mac
27
28	# netns numbers
29	NUM=3
30	IFACES=""
31	DRV_MODE="xdpgeneric xdpdrv xdpegress"
32	PASS=0
33	FAIL=0
34	LOG_DIR=$(mktemp -d)
35	declare -a NS
36	NS[0]="ns0-$(mktemp -u XXXXXX)"
37	NS[1]="ns1-$(mktemp -u XXXXXX)"
38	NS[2]="ns2-$(mktemp -u XXXXXX)"
39	NS[3]="ns3-$(mktemp -u XXXXXX)"
40
41	test_pass()
42	{
43	echo "Pass: $@"
44	PASS=$((PASS + 1))
45	}
46
47	test_fail()
48	{
49	echo "fail: $@"
50	FAIL=$((FAIL + 1))
51	}
52
53	clean_up()
54	{
55	for i in $(seq 0 $NUM); do
56	ip netns del ${NS[$i]} 2> /dev/null
57	done
58	}
59
60	# Kselftest framework requirement - SKIP code is 4.
61	check_env()
62	{
63	ip link set dev lo xdpgeneric off &>/dev/null
64	if [ $? -ne 0 ];then
65	echo "selftests: [SKIP] Could not run test without the ip xdpgeneric support"
66	exit 4
67	fi
68
69	which tcpdump &>/dev/null
70	if [ $? -ne 0 ];then
71	echo "selftests: [SKIP] Could not run test without tcpdump"
72	exit 4
73	fi
74	}
75
76	setup_ns()
77	{
78	local mode=$1
79	IFACES=""
80
81	if [ "$mode" = "xdpegress" ]; then
82	mode="xdpdrv"
83	fi
84
85	ip netns add ${NS[0]}
86	for i in $(seq $NUM); do
87	ip netns add ${NS[$i]}
88	ip -n ${NS[$i]} link add veth0 type veth peer name veth$i netns ${NS[0]}
89	ip -n ${NS[$i]} link set veth0 up
90	ip -n ${NS[0]} link set veth$i up
91
92	ip -n ${NS[$i]} addr add 192.0.2.$i/24 dev veth0
93	ip -n ${NS[$i]} addr add 2001:db8::$i/64 dev veth0
94	# Add a neigh entry for IPv4 ping test
95	ip -n ${NS[$i]} neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0
96	ip -n ${NS[$i]} link set veth0 $mode obj \
97	xdp_dummy.bpf.o sec xdp &> /dev/null \|\| \
98	{ test_fail "Unable to load dummy xdp" && exit 1; }
99	IFACES="$IFACES veth$i"
100	veth_mac[$i]=$(ip -n ${NS[0]} link show veth$i \| awk '/link\/ether/ {print $2}')
101	done
102	}
103
104	do_egress_tests()
105	{
106	local mode=$1
107
108	# mac test
109	ip netns exec ${NS[2]} tcpdump -e -i veth0 -nn -l -e &> ${LOG_DIR}/mac_ns1-2_${mode}.log &
110	ip netns exec ${NS[3]} tcpdump -e -i veth0 -nn -l -e &> ${LOG_DIR}/mac_ns1-3_${mode}.log &
111	sleep 0.5
112	ip netns exec ${NS[1]} ping 192.0.2.254 -i 0.1 -c 4 &> /dev/null
113	sleep 0.5
114	pkill tcpdump
115
116	# mac check
117	grep -q "${veth_mac[2]} > ff:ff:ff:ff:ff:ff" ${LOG_DIR}/mac_ns1-2_${mode}.log && \
118	test_pass "$mode mac ns1-2" \|\| test_fail "$mode mac ns1-2"
119	grep -q "${veth_mac[3]} > ff:ff:ff:ff:ff:ff" ${LOG_DIR}/mac_ns1-3_${mode}.log && \
120	test_pass "$mode mac ns1-3" \|\| test_fail "$mode mac ns1-3"
121	}
122
123	do_ping_tests()
124	{
125	local mode=$1
126
127	# ping6 test: echo request should be redirect back to itself, not others
128	ip netns exec ${NS[1]} ip neigh add 2001:db8::2 dev veth0 lladdr 00:00:00:00:00:02
129
130	ip netns exec ${NS[1]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-1_${mode}.log &
131	ip netns exec ${NS[2]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-2_${mode}.log &
132	ip netns exec ${NS[3]} tcpdump -i veth0 -nn -l -e &> ${LOG_DIR}/ns1-3_${mode}.log &
133	sleep 0.5
134	# ARP test
135	ip netns exec ${NS[1]} arping -q -c 2 -I veth0 192.0.2.254
136	# IPv4 test
137	ip netns exec ${NS[1]} ping 192.0.2.253 -i 0.1 -c 4 &> /dev/null
138	# IPv6 test
139	ip netns exec ${NS[1]} ping6 2001:db8::2 -i 0.1 -c 2 &> /dev/null
140	sleep 0.5
141	pkill tcpdump
142
143	# All netns should receive the redirect arp requests
144	[ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
145	test_pass "$mode arp(F_BROADCAST) ns1-1" \|\| \
146	test_fail "$mode arp(F_BROADCAST) ns1-1"
147	[ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-2_${mode}.log) -eq 2 ] && \
148	test_pass "$mode arp(F_BROADCAST) ns1-2" \|\| \
149	test_fail "$mode arp(F_BROADCAST) ns1-2"
150	[ $(grep -cF "who-has 192.0.2.254" ${LOG_DIR}/ns1-3_${mode}.log) -eq 2 ] && \
151	test_pass "$mode arp(F_BROADCAST) ns1-3" \|\| \
152	test_fail "$mode arp(F_BROADCAST) ns1-3"
153
154	# ns1 should not receive the redirect echo request, others should
155	[ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
156	test_pass "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-1" \|\| \
157	test_fail "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-1"
158	[ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-2_${mode}.log) -eq 4 ] && \
159	test_pass "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-2" \|\| \
160	test_fail "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-2"
161	[ $(grep -c "ICMP echo request" ${LOG_DIR}/ns1-3_${mode}.log) -eq 4 ] && \
162	test_pass "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-3" \|\| \
163	test_fail "$mode IPv4 (F_BROADCAST\|F_EXCLUDE_INGRESS) ns1-3"
164
165	# ns1 should receive the echo request, ns2 should not
166	[ $(grep -c "ICMP6, echo request" ${LOG_DIR}/ns1-1_${mode}.log) -eq 4 ] && \
167	test_pass "$mode IPv6 (no flags) ns1-1" \|\| \
168	test_fail "$mode IPv6 (no flags) ns1-1"
169	[ $(grep -c "ICMP6, echo request" ${LOG_DIR}/ns1-2_${mode}.log) -eq 0 ] && \
170	test_pass "$mode IPv6 (no flags) ns1-2" \|\| \
171	test_fail "$mode IPv6 (no flags) ns1-2"
172	}
173
174	do_tests()
175	{
176	local mode=$1
177	local drv_p
178
179	case ${mode} in
180	xdpdrv) drv_p="-N";;
181	xdpegress) drv_p="-X";;
182	xdpgeneric) drv_p="-S";;
183	esac
184
185	ip netns exec ${NS[0]} ./xdp_redirect_multi $drv_p $IFACES &> ${LOG_DIR}/xdp_redirect_${mode}.log &
186	xdp_pid=$!
187	sleep 1
188	if ! ps -p $xdp_pid > /dev/null; then
189	test_fail "$mode xdp_redirect_multi start failed"
190	return 1
191	fi
192
193	if [ "$mode" = "xdpegress" ]; then
194	do_egress_tests $mode
195	else
196	do_ping_tests $mode
197	fi
198
199	kill $xdp_pid
200	}
201
202	check_env
203
204	trap clean_up EXIT
205
206	for mode in ${DRV_MODE}; do
207	setup_ns $mode
208	do_tests $mode
209	clean_up
210	done
211	rm -rf ${LOG_DIR}
212
213	echo "Summary: PASS $PASS, FAIL $FAIL"
214	[ $FAIL -eq 0 ] && exit 0 \|\| exit 1

tools/testing/selftests/bpf/test_xdp_vlan.sh+0−233

@@ -1,233 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3	# Author: Jesper Dangaard Brouer <hawk@kernel.org>
4
5	# Kselftest framework requirement - SKIP code is 4.
6	readonly KSFT_SKIP=4
7	readonly NS1="ns1-$(mktemp -u XXXXXX)"
8	readonly NS2="ns2-$(mktemp -u XXXXXX)"
9
10	# Allow wrapper scripts to name test
11	if [ -z "$TESTNAME" ]; then
12	TESTNAME=xdp_vlan
13	fi
14
15	# Default XDP mode
16	XDP_MODE=xdpgeneric
17
18	usage() {
19	echo "Testing XDP + TC eBPF VLAN manipulations: $TESTNAME"
20	echo ""
21	echo "Usage: $0 [-vfh]"
22	echo " -v \| --verbose : Verbose"
23	echo " --flush : Flush before starting (e.g. after --interactive)"
24	echo " --interactive : Keep netns setup running after test-run"
25	echo " --mode=XXX : Choose XDP mode (xdp \| xdpgeneric \| xdpdrv)"
26	echo ""
27	}
28
29	valid_xdp_mode()
30	{
31	local mode=$1
32
33	case "$mode" in
34	xdpgeneric \| xdpdrv \| xdp)
35	return 0
36	;;
37	*)
38	return 1
39	esac
40	}
41
42	cleanup()
43	{
44	local status=$?
45
46	if [ "$status" = "0" ]; then
47	echo "selftests: $TESTNAME [PASS]";
48	else
49	echo "selftests: $TESTNAME [FAILED]";
50	fi
51
52	if [ -n "$INTERACTIVE" ]; then
53	echo "Namespace setup still active explore with:"
54	echo " ip netns exec ${NS1} bash"
55	echo " ip netns exec ${NS2} bash"
56	exit $status
57	fi
58
59	set +e
60	ip link del veth1 2> /dev/null
61	ip netns del ${NS1} 2> /dev/null
62	ip netns del ${NS2} 2> /dev/null
63	}
64
65	# Using external program "getopt" to get --long-options
66	OPTIONS=$(getopt -o hvfi: \
67	--long verbose,flush,help,interactive,debug,mode: -- "$@")
68	if (( $? != 0 )); then
69	usage
70	echo "selftests: $TESTNAME [FAILED] Error calling getopt, unknown option?"
71	exit 2
72	fi
73	eval set -- "$OPTIONS"
74
75	## --- Parse command line arguments / parameters ---
76	while true; do
77	case "$1" in
78	-v \| --verbose)
79	export VERBOSE=yes
80	shift
81	;;
82	-i \| --interactive \| --debug )
83	INTERACTIVE=yes
84	shift
85	;;
86	-f \| --flush )
87	cleanup
88	shift
89	;;
90	--mode )
91	shift
92	XDP_MODE=$1
93	shift
94	;;
95	-- )
96	shift
97	break
98	;;
99	-h \| --help )
100	usage;
101	echo "selftests: $TESTNAME [SKIP] usage help info requested"
102	exit $KSFT_SKIP
103	;;
104	* )
105	shift
106	break
107	;;
108	esac
109	done
110
111	if [ "$EUID" -ne 0 ]; then
112	echo "selftests: $TESTNAME [FAILED] need root privileges"
113	exit 1
114	fi
115
116	valid_xdp_mode $XDP_MODE
117	if [ $? -ne 0 ]; then
118	echo "selftests: $TESTNAME [FAILED] unknown XDP mode ($XDP_MODE)"
119	exit 1
120	fi
121
122	ip link set dev lo xdpgeneric off 2>/dev/null > /dev/null
123	if [ $? -ne 0 ]; then
124	echo "selftests: $TESTNAME [SKIP] need ip xdp support"
125	exit $KSFT_SKIP
126	fi
127
128	# Interactive mode likely require us to cleanup netns
129	if [ -n "$INTERACTIVE" ]; then
130	ip link del veth1 2> /dev/null
131	ip netns del ${NS1} 2> /dev/null
132	ip netns del ${NS2} 2> /dev/null
133	fi
134
135	# Exit on failure
136	set -e
137
138	# Some shell-tools dependencies
139	which ip > /dev/null
140	which tc > /dev/null
141	which ethtool > /dev/null
142
143	# Make rest of shell verbose, showing comments as doc/info
144	if [ -n "$VERBOSE" ]; then
145	set -v
146	fi
147
148	# Create two namespaces
149	ip netns add ${NS1}
150	ip netns add ${NS2}
151
152	# Run cleanup if failing or on kill
153	trap cleanup 0 2 3 6 9
154
155	# Create veth pair
156	ip link add veth1 type veth peer name veth2
157
158	# Move veth1 and veth2 into the respective namespaces
159	ip link set veth1 netns ${NS1}
160	ip link set veth2 netns ${NS2}
161
162	# NOTICE: XDP require VLAN header inside packet payload
163	# - Thus, disable VLAN offloading driver features
164	# - For veth REMEMBER TX side VLAN-offload
165	#
166	# Disable rx-vlan-offload (mostly needed on ns1)
167	ip netns exec ${NS1} ethtool -K veth1 rxvlan off
168	ip netns exec ${NS2} ethtool -K veth2 rxvlan off
169	#
170	# Disable tx-vlan-offload (mostly needed on ns2)
171	ip netns exec ${NS2} ethtool -K veth2 txvlan off
172	ip netns exec ${NS1} ethtool -K veth1 txvlan off
173
174	export IPADDR1=100.64.41.1
175	export IPADDR2=100.64.41.2
176
177	# In ns1/veth1 add IP-addr on plain net_device
178	ip netns exec ${NS1} ip addr add ${IPADDR1}/24 dev veth1
179	ip netns exec ${NS1} ip link set veth1 up
180
181	# In ns2/veth2 create VLAN device
182	export VLAN=4011
183	export DEVNS2=veth2
184	ip netns exec ${NS2} ip link add link $DEVNS2 name $DEVNS2.$VLAN type vlan id $VLAN
185	ip netns exec ${NS2} ip addr add ${IPADDR2}/24 dev $DEVNS2.$VLAN
186	ip netns exec ${NS2} ip link set $DEVNS2 up
187	ip netns exec ${NS2} ip link set $DEVNS2.$VLAN up
188
189	# Bringup lo in netns (to avoids confusing people using --interactive)
190	ip netns exec ${NS1} ip link set lo up
191	ip netns exec ${NS2} ip link set lo up
192
193	# At this point, the hosts cannot reach each-other,
194	# because ns2 are using VLAN tags on the packets.
195
196	ip netns exec ${NS2} sh -c 'ping -W 1 -c 1 100.64.41.1 \|\| echo "Success: First ping must fail"'
197
198
199	# Now we can use the test_xdp_vlan.c program to pop/push these VLAN tags
200	# ----------------------------------------------------------------------
201	# In ns1: ingress use XDP to remove VLAN tags
202	export DEVNS1=veth1
203	export BPF_FILE=test_xdp_vlan.bpf.o
204
205	# First test: Remove VLAN by setting VLAN ID 0, using "xdp_vlan_change"
206	export XDP_PROG=xdp_vlan_change
207	ip netns exec ${NS1} ip link set $DEVNS1 $XDP_MODE object $BPF_FILE section $XDP_PROG
208
209	# In ns1: egress use TC to add back VLAN tag 4011
210	# (del cmd)
211	# tc qdisc del dev $DEVNS1 clsact 2> /dev/null
212	#
213	ip netns exec ${NS1} tc qdisc add dev $DEVNS1 clsact
214	ip netns exec ${NS1} tc filter add dev $DEVNS1 egress \
215	prio 1 handle 1 bpf da obj $BPF_FILE sec tc_vlan_push
216
217	# Now the namespaces can reach each-other, test with ping:
218	ip netns exec ${NS2} ping -i 0.2 -W 2 -c 2 $IPADDR1
219	ip netns exec ${NS1} ping -i 0.2 -W 2 -c 2 $IPADDR2
220
221	# Second test: Replace xdp prog, that fully remove vlan header
222	#
223	# Catch kernel bug for generic-XDP, that does didn't allow us to
224	# remove a VLAN header, because skb->protocol still contain VLAN
225	# ETH_P_8021Q indication, and this cause overwriting of our changes.
226	#
227	export XDP_PROG=xdp_vlan_remove_outer2
228	ip netns exec ${NS1} ip link set $DEVNS1 $XDP_MODE off
229	ip netns exec ${NS1} ip link set $DEVNS1 $XDP_MODE object $BPF_FILE section $XDP_PROG
230
231	# Now the namespaces should still be able reach each-other, test with ping:
232	ip netns exec ${NS2} ping -i 0.2 -W 2 -c 2 $IPADDR1
233	ip netns exec ${NS1} ping -i 0.2 -W 2 -c 2 $IPADDR2

tools/testing/selftests/bpf/test_xdp_vlan_mode_generic.sh+0−9

@@ -1,9 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3
4	# Exit on failure
5	set -e
6
7	# Wrapper script to test generic-XDP
8	export TESTNAME=xdp_vlan_mode_generic
9	./test_xdp_vlan.sh --mode=xdpgeneric

tools/testing/selftests/bpf/test_xdp_vlan_mode_native.sh+0−9

@@ -1,9 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3
4	# Exit on failure
5	set -e
6
7	# Wrapper script to test native-XDP
8	export TESTNAME=xdp_vlan_mode_native
9	./test_xdp_vlan.sh --mode=xdpdrv

tools/testing/selftests/bpf/veristat.c+355−12

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	#define _GNU_SOURCE	3	#define _GNU_SOURCE
4	#include <argp.h>	4	#include <argp.h>
5	#include <libgen.h>	5	#include <libgen.h>
		6	#include <ctype.h>
6	#include <string.h>	7	#include <string.h>
7	#include <stdlib.h>	8	#include <stdlib.h>
8	#include <sched.h>	9	#include <sched.h>
@@ -154,6 +155,16 @@ struct filter { · 145 unmodified lines
154	bool abs;	155	bool abs;
155	};	156	};
156		157
		158	struct var_preset {
		159	char *name;
		160	enum { INTEGRAL, ENUMERATOR } type;
		161	union {
		162	long long ivalue;
		163	char *svalue;
		164	};
		165	bool applied;
		166	};
		167
157	static struct env {	168	static struct env {
158	char **filenames;	169	char **filenames;
159	int filename_cnt;	170	int filename_cnt;
@@ -195,6 +206,8 @@ static struct env { · 35 unmodified lines
195	int progs_processed;	206	int progs_processed;
196	int progs_skipped;	207	int progs_skipped;
197	int top_src_lines;	208	int top_src_lines;
		209	struct var_preset *presets;
		210	int npresets;
198	} env;	211	} env;
199		212
200	static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)	213	static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
@@ -246,16 +259,20 @@ static const struct argp_option opts[] = { · 45 unmodified lines
246	{ "test-reg-invariants", 'r', NULL, 0,	259	{ "test-reg-invariants", 'r', NULL, 0,
247	"Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },	260	"Force BPF verifier failure on register invariant violation (BPF_F_TEST_REG_INVARIANTS program flag)" },
248	{ "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" },	261	{ "top-src-lines", 'S', "N", 0, "Emit N most frequent source code lines" },
		262	{ "set-global-vars", 'G', "GLOBAL", 0, "Set global variables provided in the expression, for example \"var1 = 1\"" },
249	{},	263	{},
250	};	264	};
251		265
252	static int parse_stats(const char stats_str, struct stat_specs specs);	266	static int parse_stats(const char stats_str, struct stat_specs specs);
253	static int append_filter(struct filter *filters, int cnt, const char *str);	267	static int append_filter(struct filter *filters, int cnt, const char *str);
254	static int append_filter_file(const char *path);	268	static int append_filter_file(const char *path);
		269	static int append_var_preset(struct var_preset *presets, int cnt, const char *expr);
		270	static int append_var_preset_file(const char *filename);
		271	static int append_file(const char *path);
		272	static int append_file_from_file(const char *path);
255		273
256	static error_t parse_arg(int key, char arg, struct argp_state state)	274	static error_t parse_arg(int key, char arg, struct argp_state state)
257	{	275	{
258	void *tmp;
259	int err;	276	int err;
260		277
261	switch (key) {	278	switch (key) {
@@ -353,15 +370,26 @@ static error_t parse_arg(int key, char arg, struct argp_state state) · 91 unmodified lines
353	argp_usage(state);	370	argp_usage(state);
354	}	371	}
355	break;	372	break;
		373	case 'G': {
		374	if (arg[0] == '@')
		375	err = append_var_preset_file(arg + 1);
		376	else
		377	err = append_var_preset(&env.presets, &env.npresets, arg);
		378	if (err) {
		379	fprintf(stderr, "Failed to parse global variable presets: %s\n", arg);
		380	return err;
		381	}
		382	break;
		383	}
356	case ARGP_KEY_ARG:	384	case ARGP_KEY_ARG:
357	tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));	385	if (arg[0] == '@')
358	if (!tmp)	386	err = append_file_from_file(arg + 1);
359	return -ENOMEM;	387	else
360	env.filenames = tmp;	388	err = append_file(arg);
361	env.filenames[env.filename_cnt] = strdup(arg);	389	if (err) {
362	if (!env.filenames[env.filename_cnt])	390	fprintf(stderr, "Failed to collect BPF object files: %d\n", err);
363	return -ENOMEM;	391	return err;
364	env.filename_cnt++;	392	}
365	break;	393	break;
366	default:	394	default:
367	return ARGP_ERR_UNKNOWN;	395	return ARGP_ERR_UNKNOWN;
@@ -632,7 +660,7 @@ static int append_filter_file(const char *path) · 264 unmodified lines
632	f = fopen(path, "r");	660	f = fopen(path, "r");
633	if (!f) {	661	if (!f) {
634	err = -errno;	662	err = -errno;
635	fprintf(stderr, "Failed to open filters in '%s': %d\n", path, err);	663	fprintf(stderr, "Failed to open filters in '%s': %s\n", path, strerror(-err));
636	return err;	664	return err;
637	}	665	}
638		666
@@ -662,6 +690,49 @@ static const struct stat_specs default_output_spec = { · 23 unmodified lines
662	},	690	},
663	};	691	};
664		692
		693	static int append_file(const char *path)
		694	{
		695	void *tmp;
		696
		697	tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
		698	if (!tmp)
		699	return -ENOMEM;
		700	env.filenames = tmp;
		701	env.filenames[env.filename_cnt] = strdup(path);
		702	if (!env.filenames[env.filename_cnt])
		703	return -ENOMEM;
		704	env.filename_cnt++;
		705	return 0;
		706	}
		707
		708	static int append_file_from_file(const char *path)
		709	{
		710	char buf[1024];
		711	int err = 0;
		712	FILE *f;
		713
		714	f = fopen(path, "r");
		715	if (!f) {
		716	err = -errno;
		717	fprintf(stderr, "Failed to open object files list in '%s': %s\n",
		718	path, strerror(errno));
		719	return err;
		720	}
		721
		722	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
		723	/* lines starting with # are comments, skip them */
		724	if (buf[0] == '\0' \|\| buf[0] == '#')
		725	continue;
		726	err = append_file(buf);
		727	if (err)
		728	goto cleanup;
		729	}
		730
		731	cleanup:
		732	fclose(f);
		733	return err;
		734	}
		735
665	static const struct stat_specs default_csv_output_spec = {	736	static const struct stat_specs default_csv_output_spec = {
666	.spec_cnt = 14,	737	.spec_cnt = 14,
667	.ids = {	738	.ids = {
@@ -1163,13 +1234,13 @@ static void fixup_obj(struct bpf_object obj, struct bpf_program prog, const ch · 495 unmodified lines
1163	bpf_program__set_expected_attach_type(prog, attach_type);	1234	bpf_program__set_expected_attach_type(prog, attach_type);
1164		1235
1165	if (!env.quiet) {	1236	if (!env.quiet) {
1166	printf("Using guessed program type '%s' for %s/%s...\n",	1237	fprintf(stderr, "Using guessed program type '%s' for %s/%s...\n",
1167	libbpf_bpf_prog_type_str(prog_type),	1238	libbpf_bpf_prog_type_str(prog_type),
1168	filename, prog_name);	1239	filename, prog_name);
1169	}	1240	}
1170	} else {	1241	} else {
1171	if (!env.quiet) {	1242	if (!env.quiet) {
1172	printf("Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n",	1243	fprintf(stderr, "Failed to guess program type for freplace program with context type name '%s' for %s/%s. Consider using canonical type names to help veristat...\n",
1173	ctx_name, filename, prog_name);	1244	ctx_name, filename, prog_name);
1174	}	1245	}
1175	}	1246	}
@@ -1292,6 +1363,261 @@ static int process_prog(const char filename, struct bpf_object obj, struct bpf · 116 unmodified lines
1292	return 0;	1363	return 0;
1293	};	1364	};
1294		1365
		1366	static int append_var_preset(struct var_preset *presets, int cnt, const char *expr)
		1367	{
		1368	void *tmp;
		1369	struct var_preset *cur;
		1370	char var[256], val[256], *val_end;
		1371	long long value;
		1372	int n;
		1373
		1374	tmp = realloc(presets, (cnt + 1) * sizeof(**presets));
		1375	if (!tmp)
		1376	return -ENOMEM;
		1377	*presets = tmp;
		1378	cur = &(presets)[cnt];
		1379	memset(cur, 0, sizeof(*cur));
		1380	(*cnt)++;
		1381
		1382	if (sscanf(expr, "%s = %s %n", var, val, &n) != 2 \|\| n != strlen(expr)) {
		1383	fprintf(stderr, "Failed to parse expression '%s'\n", expr);
		1384	return -EINVAL;
		1385	}
		1386
		1387	if (val[0] == '-' \|\| isdigit(val[0])) {
		1388	/* must be a number */
		1389	errno = 0;
		1390	value = strtoll(val, &val_end, 0);
		1391	if (errno == ERANGE) {
		1392	errno = 0;
		1393	value = strtoull(val, &val_end, 0);
		1394	}
		1395	if (errno \|\| *val_end != '\0') {
		1396	fprintf(stderr, "Failed to parse value '%s'\n", val);
		1397	return -EINVAL;
		1398	}
		1399	cur->ivalue = value;
		1400	cur->type = INTEGRAL;
		1401	} else {
		1402	/* if not a number, consider it enum value */
		1403	cur->svalue = strdup(val);
		1404	if (!cur->svalue)
		1405	return -ENOMEM;
		1406	cur->type = ENUMERATOR;
		1407	}
		1408
		1409	cur->name = strdup(var);
		1410	if (!cur->name)
		1411	return -ENOMEM;
		1412
		1413	return 0;
		1414	}
		1415
		1416	static int append_var_preset_file(const char *filename)
		1417	{
		1418	char buf[1024];
		1419	FILE *f;
		1420	int err = 0;
		1421
		1422	f = fopen(filename, "rt");
		1423	if (!f) {
		1424	err = -errno;
		1425	fprintf(stderr, "Failed to open presets in '%s': %s\n", filename, strerror(-err));
		1426	return -EINVAL;
		1427	}
		1428
		1429	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
		1430	if (buf[0] == '\0' \|\| buf[0] == '#')
		1431	continue;
		1432
		1433	err = append_var_preset(&env.presets, &env.npresets, buf);
		1434	if (err)
		1435	goto cleanup;
		1436	}
		1437
		1438	cleanup:
		1439	fclose(f);
		1440	return err;
		1441	}
		1442
		1443	static bool is_signed_type(const struct btf_type *t)
		1444	{
		1445	if (btf_is_int(t))
		1446	return btf_int_encoding(t) & BTF_INT_SIGNED;
		1447	if (btf_is_any_enum(t))
		1448	return btf_kflag(t);
		1449	return true;
		1450	}
		1451
		1452	static int enum_value_from_name(const struct btf btf, const struct btf_type t,
		1453	const char evalue, long long retval)
		1454	{
		1455	if (btf_is_enum(t)) {
		1456	struct btf_enum *e = btf_enum(t);
		1457	int i, n = btf_vlen(t);
		1458
		1459	for (i = 0; i < n; ++i, ++e) {
		1460	const char *cur_name = btf__name_by_offset(btf, e->name_off);
		1461
		1462	if (strcmp(cur_name, evalue) == 0) {
		1463	*retval = e->val;
		1464	return 0;
		1465	}
		1466	}
		1467	} else if (btf_is_enum64(t)) {
		1468	struct btf_enum64 *e = btf_enum64(t);
		1469	int i, n = btf_vlen(t);
		1470
		1471	for (i = 0; i < n; ++i, ++e) {
		1472	const char *cur_name = btf__name_by_offset(btf, e->name_off);
		1473	__u64 value = btf_enum64_value(e);
		1474
		1475	if (strcmp(cur_name, evalue) == 0) {
		1476	*retval = value;
		1477	return 0;
		1478	}
		1479	}
		1480	}
		1481	return -EINVAL;
		1482	}
		1483
		1484	static bool is_preset_supported(const struct btf_type *t)
		1485	{
		1486	return btf_is_int(t) \|\| btf_is_enum(t) \|\| btf_is_enum64(t);
		1487	}
		1488
		1489	static int set_global_var(struct bpf_object obj, struct btf btf, const struct btf_type *t,
		1490	struct bpf_map map, struct btf_var_secinfo sinfo,
		1491	struct var_preset *preset)
		1492	{
		1493	const struct btf_type *base_type;
		1494	void *ptr;
		1495	long long value = preset->ivalue;
		1496	size_t size;
		1497
		1498	base_type = btf__type_by_id(btf, btf__resolve_type(btf, t->type));
		1499	if (!base_type) {
		1500	fprintf(stderr, "Failed to resolve type %d\n", t->type);
		1501	return -EINVAL;
		1502	}
		1503	if (!is_preset_supported(base_type)) {
		1504	fprintf(stderr, "Setting value for type %s is not supported\n",
		1505	btf__name_by_offset(btf, base_type->name_off));
		1506	return -EINVAL;
		1507	}
		1508
		1509	if (preset->type == ENUMERATOR) {
		1510	if (btf_is_any_enum(base_type)) {
		1511	if (enum_value_from_name(btf, base_type, preset->svalue, &value)) {
		1512	fprintf(stderr,
		1513	"Failed to find integer value for enum element %s\n",
		1514	preset->svalue);
		1515	return -EINVAL;
		1516	}
		1517	} else {
		1518	fprintf(stderr, "Value %s is not supported for type %s\n",
		1519	preset->svalue, btf__name_by_offset(btf, base_type->name_off));
		1520	return -EINVAL;
		1521	}
		1522	}
		1523
		1524	/* Check if value fits into the target variable size */
		1525	if (sinfo->size < sizeof(value)) {
		1526	bool is_signed = is_signed_type(base_type);
		1527	__u32 unsigned_bits = sinfo->size * 8 - (is_signed ? 1 : 0);
		1528	long long max_val = 1ll << unsigned_bits;
		1529
		1530	if (value >= max_val \|\| value < -max_val) {
		1531	fprintf(stderr,
		1532	"Variable %s value %lld is out of range [%lld; %lld]\n",
		1533	btf__name_by_offset(btf, t->name_off), value,
		1534	is_signed ? -max_val : 0, max_val - 1);
		1535	return -EINVAL;
		1536	}
		1537	}
		1538
		1539	ptr = bpf_map__initial_value(map, &size);
		1540	if (!ptr \|\| sinfo->offset + sinfo->size > size)
		1541	return -EINVAL;
		1542
		1543	if (__BYTE_ORDER == __LITTLE_ENDIAN) {
		1544	memcpy(ptr + sinfo->offset, &value, sinfo->size);
		1545	} else { /* __BYTE_ORDER == __BIG_ENDIAN */
		1546	__u8 src_offset = sizeof(value) - sinfo->size;
		1547
		1548	memcpy(ptr + sinfo->offset, (void *)&value + src_offset, sinfo->size);
		1549	}
		1550	return 0;
		1551	}
		1552
		1553	static int set_global_vars(struct bpf_object obj, struct var_preset presets, int npresets)
		1554	{
		1555	struct btf_var_secinfo *sinfo;
		1556	const char *sec_name;
		1557	const struct btf_type *t;
		1558	struct bpf_map *map;
		1559	struct btf *btf;
		1560	int i, j, k, n, cnt, err = 0;
		1561
		1562	if (npresets == 0)
		1563	return 0;
		1564
		1565	btf = bpf_object__btf(obj);
		1566	if (!btf)
		1567	return -EINVAL;
		1568
		1569	cnt = btf__type_cnt(btf);
		1570	for (i = 1; i != cnt; ++i) {
		1571	t = btf__type_by_id(btf, i);
		1572
		1573	if (!btf_is_datasec(t))
		1574	continue;
		1575
		1576	sinfo = btf_var_secinfos(t);
		1577	sec_name = btf__name_by_offset(btf, t->name_off);
		1578	map = bpf_object__find_map_by_name(obj, sec_name);
		1579	if (!map)
		1580	continue;
		1581
		1582	n = btf_vlen(t);
		1583	for (j = 0; j < n; ++j, ++sinfo) {
		1584	const struct btf_type *var_type = btf__type_by_id(btf, sinfo->type);
		1585	const char *var_name;
		1586
		1587	if (!btf_is_var(var_type))
		1588	continue;
		1589
		1590	var_name = btf__name_by_offset(btf, var_type->name_off);
		1591
		1592	for (k = 0; k < npresets; ++k) {
		1593	if (strcmp(var_name, presets[k].name) != 0)
		1594	continue;
		1595
		1596	if (presets[k].applied) {
		1597	fprintf(stderr, "Variable %s is set more than once",
		1598	var_name);
		1599	return -EINVAL;
		1600	}
		1601
		1602	err = set_global_var(obj, btf, var_type, map, sinfo, presets + k);
		1603	if (err)
		1604	return err;
		1605
		1606	presets[k].applied = true;
		1607	break;
		1608	}
		1609	}
		1610	}
		1611	for (i = 0; i < npresets; ++i) {
		1612	if (!presets[i].applied) {
		1613	fprintf(stderr, "Global variable preset %s has not been applied\n",
		1614	presets[i].name);
		1615	}
		1616	presets[i].applied = false;
		1617	}
		1618	return err;
		1619	}
		1620
1295	static int process_obj(const char *filename)	1621	static int process_obj(const char *filename)
1296	{	1622	{
1297	const char *base_filename = basename(strdupa(filename));	1623	const char *base_filename = basename(strdupa(filename));
@@ -1341,6 +1667,11 @@ static int process_obj(const char *filename) · 43 unmodified lines
1341	if (prog_cnt == 1) {	1667	if (prog_cnt == 1) {
1342	prog = bpf_object__next_program(obj, NULL);	1668	prog = bpf_object__next_program(obj, NULL);
1343	bpf_program__set_autoload(prog, true);	1669	bpf_program__set_autoload(prog, true);
		1670	err = set_global_vars(obj, env.presets, env.npresets);
		1671	if (err) {
		1672	fprintf(stderr, "Failed to set global variables %d\n", err);
		1673	goto cleanup;
		1674	}
1344	process_prog(filename, obj, prog);	1675	process_prog(filename, obj, prog);
1345	goto cleanup;	1676	goto cleanup;
1346	}	1677	}
@@ -1355,6 +1686,12 @@ static int process_obj(const char *filename) · 8 unmodified lines
1355	goto cleanup;	1686	goto cleanup;
1356	}	1687	}
1357		1688
		1689	err = set_global_vars(tobj, env.presets, env.npresets);
		1690	if (err) {
		1691	fprintf(stderr, "Failed to set global variables %d\n", err);
		1692	goto cleanup;
		1693	}
		1694
1358	lprog = NULL;	1695	lprog = NULL;
1359	bpf_object__for_each_program(tprog, tobj) {	1696	bpf_object__for_each_program(tprog, tobj) {
1360	const char *tprog_name = bpf_program__name(tprog);	1697	const char *tprog_name = bpf_program__name(tprog);
@@ -2460,5 +2797,11 @@ int main(int argc, char **argv) · 1099 unmodified lines
2460	free(env.deny_filters[i].prog_glob);	2797	free(env.deny_filters[i].prog_glob);
2461	}	2798	}
2462	free(env.deny_filters);	2799	free(env.deny_filters);
		2800	for (i = 0; i < env.npresets; ++i) {
		2801	free(env.presets[i].name);
		2802	if (env.presets[i].type == ENUMERATOR)
		2803	free(env.presets[i].svalue);
		2804	}
		2805	free(env.presets);
2463	return -err;	2806	return -err;
2464	}	2807	}

tools/testing/selftests/bpf/with_addr.sh+0−54

@@ -1,54 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3	#
4	# add private ipv4 and ipv6 addresses to loopback
5
6	readonly V6_INNER='100::a/128'
7	readonly V4_INNER='192.168.0.1/32'
8
9	if getopts ":s" opt; then
10	readonly SIT_DEV_NAME='sixtofourtest0'
11	readonly V6_SIT='2::/64'
12	readonly V4_SIT='172.17.0.1/32'
13	shift
14	fi
15
16	fail() {
17	echo "error: $*" 1>&2
18	exit 1
19	}
20
21	setup() {
22	ip -6 addr add "${V6_INNER}" dev lo \|\| fail 'failed to setup v6 address'
23	ip -4 addr add "${V4_INNER}" dev lo \|\| fail 'failed to setup v4 address'
24
25	if [[ -n "${V6_SIT}" ]]; then
26	ip link add "${SIT_DEV_NAME}" type sit remote any local any \
27	\|\| fail 'failed to add sit'
28	ip link set dev "${SIT_DEV_NAME}" up \
29	\|\| fail 'failed to bring sit device up'
30	ip -6 addr add "${V6_SIT}" dev "${SIT_DEV_NAME}" \
31	\|\| fail 'failed to setup v6 SIT address'
32	ip -4 addr add "${V4_SIT}" dev "${SIT_DEV_NAME}" \
33	\|\| fail 'failed to setup v4 SIT address'
34	fi
35
36	sleep 2 # avoid race causing bind to fail
37	}
38
39	cleanup() {
40	if [[ -n "${V6_SIT}" ]]; then
41	ip -4 addr del "${V4_SIT}" dev "${SIT_DEV_NAME}"
42	ip -6 addr del "${V6_SIT}" dev "${SIT_DEV_NAME}"
43	ip link del "${SIT_DEV_NAME}"
44	fi
45
46	ip -4 addr del "${V4_INNER}" dev lo
47	ip -6 addr del "${V6_INNER}" dev lo
48	}
49
50	trap cleanup EXIT
51
52	setup
53	"$@"
54	exit "$?"

tools/testing/selftests/bpf/with_tunnels.sh+0−36

@@ -1,36 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3	#
4	# setup tunnels for flow dissection test
5
6	readonly SUFFIX="test_$(mktemp -u XXXX)"
7	CONFIG="remote 127.0.0.2 local 127.0.0.1 dev lo"
8
9	setup() {
10	ip link add "ipip_${SUFFIX}" type ipip ${CONFIG}
11	ip link add "gre_${SUFFIX}" type gre ${CONFIG}
12	ip link add "sit_${SUFFIX}" type sit ${CONFIG}
13
14	echo "tunnels before test:"
15	ip tunnel show
16
17	ip link set "ipip_${SUFFIX}" up
18	ip link set "gre_${SUFFIX}" up
19	ip link set "sit_${SUFFIX}" up
20	}
21
22
23	cleanup() {
24	ip tunnel del "ipip_${SUFFIX}"
25	ip tunnel del "gre_${SUFFIX}"
26	ip tunnel del "sit_${SUFFIX}"
27
28	echo "tunnels after test:"
29	ip tunnel show
30	}
31
32	trap cleanup EXIT
33
34	setup
35	"$@"
36	exit "$?"

tools/testing/selftests/bpf/xdp_redirect_multi.c+0−226

@@ -1,226 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	#include <linux/bpf.h>
3	#include <linux/if_link.h>
4	#include <assert.h>
5	#include <errno.h>
6	#include <signal.h>
7	#include <stdio.h>
8	#include <stdlib.h>
9	#include <string.h>
10	#include <net/if.h>
11	#include <unistd.h>
12	#include <libgen.h>
13	#include <sys/ioctl.h>
14	#include <sys/types.h>
15	#include <sys/socket.h>
16	#include <netinet/in.h>
17
18	#include "bpf_util.h"
19	#include <bpf/bpf.h>
20	#include <bpf/libbpf.h>
21
22	#define MAX_IFACE_NUM 32
23	#define MAX_INDEX_NUM 1024
24
25	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
26	static int ifaces[MAX_IFACE_NUM] = {};
27
28	static void int_exit(int sig)
29	{
30	__u32 prog_id = 0;
31	int i;
32
33	for (i = 0; ifaces[i] > 0; i++) {
34	if (bpf_xdp_query_id(ifaces[i], xdp_flags, &prog_id)) {
35	printf("bpf_xdp_query_id failed\n");
36	exit(1);
37	}
38	if (prog_id)
39	bpf_xdp_detach(ifaces[i], xdp_flags, NULL);
40	}
41
42	exit(0);
43	}
44
45	static int get_mac_addr(unsigned int ifindex, void *mac_addr)
46	{
47	char ifname[IF_NAMESIZE];
48	struct ifreq ifr;
49	int fd, ret = -1;
50
51	fd = socket(AF_INET, SOCK_DGRAM, 0);
52	if (fd < 0)
53	return ret;
54
55	if (!if_indextoname(ifindex, ifname))
56	goto err_out;
57
58	strcpy(ifr.ifr_name, ifname);
59
60	if (ioctl(fd, SIOCGIFHWADDR, &ifr) != 0)
61	goto err_out;
62
63	memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
64	ret = 0;
65
66	err_out:
67	close(fd);
68	return ret;
69	}
70
71	static void usage(const char *prog)
72	{
73	fprintf(stderr,
74	"usage: %s [OPTS] <IFNAME\|IFINDEX> <IFNAME\|IFINDEX> ...\n"
75	"OPTS:\n"
76	" -S use skb-mode\n"
77	" -N enforce native mode\n"
78	" -F force loading prog\n"
79	" -X load xdp program on egress\n",
80	prog);
81	}
82
83	int main(int argc, char **argv)
84	{
85	int prog_fd, group_all, mac_map;
86	struct bpf_program ingress_prog, egress_prog;
87	int i, err, ret, opt, egress_prog_fd = 0;
88	struct bpf_devmap_val devmap_val;
89	bool attach_egress_prog = false;
90	unsigned char mac_addr[6];
91	char ifname[IF_NAMESIZE];
92	struct bpf_object *obj;
93	unsigned int ifindex;
94	char filename[256];
95
96	while ((opt = getopt(argc, argv, "SNFX")) != -1) {
97	switch (opt) {
98	case 'S':
99	xdp_flags \|= XDP_FLAGS_SKB_MODE;
100	break;
101	case 'N':
102	/* default, set below */
103	break;
104	case 'F':
105	xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
106	break;
107	case 'X':
108	attach_egress_prog = true;
109	break;
110	default:
111	usage(basename(argv[0]));
112	return 1;
113	}
114	}
115
116	if (!(xdp_flags & XDP_FLAGS_SKB_MODE)) {
117	xdp_flags \|= XDP_FLAGS_DRV_MODE;
118	} else if (attach_egress_prog) {
119	printf("Load xdp program on egress with SKB mode not supported yet\n");
120	goto err_out;
121	}
122
123	if (optind == argc) {
124	printf("usage: %s <IFNAME\|IFINDEX> <IFNAME\|IFINDEX> ...\n", argv[0]);
125	goto err_out;
126	}
127
128	printf("Get interfaces:");
129	for (i = 0; i < MAX_IFACE_NUM && argv[optind + i]; i++) {
130	ifaces[i] = if_nametoindex(argv[optind + i]);
131	if (!ifaces[i])
132	ifaces[i] = strtoul(argv[optind + i], NULL, 0);
133	if (!if_indextoname(ifaces[i], ifname)) {
134	perror("Invalid interface name or i");
135	goto err_out;
136	}
137	if (ifaces[i] > MAX_INDEX_NUM) {
138	printf(" interface index too large\n");
139	goto err_out;
140	}
141	printf(" %d", ifaces[i]);
142	}
143	printf("\n");
144
145	snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]);
146	obj = bpf_object__open_file(filename, NULL);
147	err = libbpf_get_error(obj);
148	if (err)
149	goto err_out;
150	err = bpf_object__load(obj);
151	if (err)
152	goto err_out;
153	prog_fd = bpf_program__fd(bpf_object__next_program(obj, NULL));
154
155	if (attach_egress_prog)
156	group_all = bpf_object__find_map_fd_by_name(obj, "map_egress");
157	else
158	group_all = bpf_object__find_map_fd_by_name(obj, "map_all");
159	mac_map = bpf_object__find_map_fd_by_name(obj, "mac_map");
160
161	if (group_all < 0 \|\| mac_map < 0) {
162	printf("bpf_object__find_map_fd_by_name failed\n");
163	goto err_out;
164	}
165
166	if (attach_egress_prog) {
167	/* Find ingress/egress prog for 2nd xdp prog */
168	ingress_prog = bpf_object__find_program_by_name(obj, "xdp_redirect_map_all_prog");
169	egress_prog = bpf_object__find_program_by_name(obj, "xdp_devmap_prog");
170	if (!ingress_prog \|\| !egress_prog) {
171	printf("finding ingress/egress_prog in obj file failed\n");
172	goto err_out;
173	}
174	prog_fd = bpf_program__fd(ingress_prog);
175	egress_prog_fd = bpf_program__fd(egress_prog);
176	if (prog_fd < 0 \|\| egress_prog_fd < 0) {
177	printf("find egress_prog fd failed\n");
178	goto err_out;
179	}
180	}
181
182	signal(SIGINT, int_exit);
183	signal(SIGTERM, int_exit);
184
185	/* Init forward multicast groups and exclude group */
186	for (i = 0; ifaces[i] > 0; i++) {
187	ifindex = ifaces[i];
188
189	if (attach_egress_prog) {
190	ret = get_mac_addr(ifindex, mac_addr);
191	if (ret < 0) {
192	printf("get interface %d mac failed\n", ifindex);
193	goto err_out;
194	}
195	ret = bpf_map_update_elem(mac_map, &ifindex, mac_addr, 0);
196	if (ret) {
197	perror("bpf_update_elem mac_map failed\n");
198	goto err_out;
199	}
200	}
201
202	/* Add all the interfaces to group all */
203	devmap_val.ifindex = ifindex;
204	devmap_val.bpf_prog.fd = egress_prog_fd;
205	ret = bpf_map_update_elem(group_all, &ifindex, &devmap_val, 0);
206	if (ret) {
207	perror("bpf_map_update_elem");
208	goto err_out;
209	}
210
211	/* bind prog_fd to each interface */
212	ret = bpf_xdp_attach(ifindex, prog_fd, xdp_flags, NULL);
213	if (ret) {
214	printf("Set xdp fd failed on %d\n", ifindex);
215	goto err_out;
216	}
217	}
218
219	/* sleep some time for testing */
220	sleep(999);
221
222	return 0;
223
224	err_out:
225	return 1;
226	}