Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull bpf fixes from Alexei Starovoitov: "Most of the diff stat comes from Xu Kuohai's fix to emit ENDBR/BTI, since all JITs had to be touched to move constant blinding out and pass bpf_verifier_env in. - Fix use-after-free in arena_vm_close on fork (Alexei Starovoitov) - Dissociate struct_ops program with map if map_update fails (Amery Hung) - Fix out-of-range and off-by-one bugs in arm64 JIT (Daniel Borkmann) - Fix precedence bug in convert_bpf_ld_abs alignment check (Daniel Borkmann) - Fix arg tracking for imprecise/multi-offset in BPF_ST/STX insns (Eduard Zingerman) - Copy token from main to subprogs to fix missing kallsyms (Eduard Zingerman) - Prevent double close and leak of btf objects in libbpf (Jiri Olsa) - Fix af_unix null-ptr-deref in sockmap (Michal Luczaj) - Fix NULL deref in map_kptr_match_type for scalar regs (Mykyta Yatsenko) - Avoid unnecessary IPIs. Remove redundant bpf_flush_icache() in arm64 and riscv JITs (Puranjay Mohan) - Fix out of bounds access. Validate node_id in arena_alloc_pages() (Puranjay Mohan) - Reject BPF-to-BPF calls and callbacks in arm32 JIT (Puranjay Mohan) - Refactor all JITs to pass bpf_verifier_env to emit ENDBR/BTI for indirect jump targets on x86-64, arm64 JITs (Xu Kuohai) - Allow UTF-8 literals in bpf_bprintf_prepare() (Yihan Ding)" * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (32 commits) bpf, arm32: Reject BPF-to-BPF calls and callbacks in the JIT bpf: Dissociate struct_ops program with map if map_update fails bpf: Validate node_id in arena_alloc_pages() libbpf: Prevent double close and leak of btf objects selftests/bpf: cover UTF-8 trace_printk output bpf: allow UTF-8 literals in bpf_bprintf_prepare() selftests/bpf: Reject scalar store into kptr slot bpf: Fix NULL deref in map_kptr_match_type for scalar regs bpf: Fix precedence bug in convert_bpf_ld_abs alignment check bpf, arm64: Emit BTI for indirect jump target bpf, x86: Emit ENDBR for indirect jump targets bpf: Add helper to detect indirect jump targets bpf: Pass bpf_verifier_env to JIT bpf: Move constants blinding out of arch-specific JITs bpf, sockmap: Take state lock for af_unix iter bpf, sockmap: Fix af_unix null-ptr-deref in proto update selftests/bpf: Extend bpf_iter_unix to attempt deadlocking bpf, sockmap: Fix af_unix iter deadlock bpf, sockmap: Annotate af_unix sock:: Sk_state data-races selftests/bpf: verify kallsyms entries for token-loaded subprograms ...

Linus Torvalds committed 7 months ago

Commit eb0d6d92 parents 12bffae e1d486446 files · +1173 −707Browse files

Unified · Split

46 files

arch/arc/net/bpf_jit_core.c +15 −26
arch/arm/net/bpf_jit_32.c +14 −35
arch/arm64/lib/insn.c +2 −0
arch/arm64/net/bpf_jit_comp.c +31 −72
arch/loongarch/net/bpf_jit.c +18 −43
arch/mips/net/bpf_jit_comp.c +2 −20
arch/parisc/net/bpf_jit_core.c +30 −45
arch/powerpc/net/bpf_jit_comp.c +27 −47
arch/riscv/net/bpf_jit.h +0 −6
arch/riscv/net/bpf_jit_core.c +23 −47
arch/s390/net/bpf_jit_comp.c +21 −40
arch/sparc/net/bpf_jit_comp_64.c +22 −41
arch/x86/net/bpf_jit_comp.c +23 −50
arch/x86/net/bpf_jit_comp32.c +5 −30
include/linux/bpf.h +2 −0
include/linux/bpf_verifier.h +5 −4
include/linux/filter.h +48 −2
kernel/bpf/arena.c +20 −3
kernel/bpf/bpf_struct_ops.c +4 −3
kernel/bpf/core.c +99 −39
kernel/bpf/fixups.c +138 −25
kernel/bpf/helpers.c +16 −1
kernel/bpf/liveness.c +62 −52
kernel/bpf/syscall.c +0 −4
kernel/bpf/verifier.c +19 −7
net/core/filter.c +1 −1
net/core/sock_map.c +2 −2
net/unix/af_unix.c +5 −4
net/unix/unix_bpf.c +3 −0
tools/lib/bpf/libbpf.c +11 −10
tools/testing/selftests/bpf/Makefile +1 −0
tools/testing/selftests/bpf/prog_tests/snprintf.c +2 −1
tools/testing/selftests/bpf/prog_tests/task_local_data.h +10 −3
tools/testing/selftests/bpf/prog_tests/test_task_local_data.c +87 −9
tools/testing/selftests/bpf/prog_tests/token.c +83 −3
tools/testing/selftests/bpf/prog_tests/trace_printk.c +22 −6
tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c +1 −20
tools/testing/selftests/bpf/progs/bpf_iter_unix.c +10 −0
tools/testing/selftests/bpf/progs/map_kptr_fail.c +15 −0
tools/testing/selftests/bpf/progs/task_local_data.bpf.h +3 −2
tools/testing/selftests/bpf/progs/timer_start_deadlock.c +4 −4
tools/testing/selftests/bpf/progs/token_kallsyms.c +19 −0
tools/testing/selftests/bpf/progs/trace_printk.c +10 −0
tools/testing/selftests/bpf/progs/verifier_live_stack.c +193 −0
tools/testing/selftests/bpf/sysctl_helpers.c +37 −0
tools/testing/selftests/bpf/sysctl_helpers.h +8 −0

arch/arc/net/bpf_jit_core.c+15−26

@@ -79,7 +79,6 @@ struct arc_jit_data { · 78 unmodified lines
79	79	* The JIT pertinent context that is used by different functions.
80	80	*
81	81	* prog: The current eBPF program being handled.
82		− * orig_prog: The original eBPF program before any possible change.
83	82	* jit: The JIT buffer and its length.
84	83	* bpf_header: The JITed program header. "jit.buf" points inside it.
85	84	* emit: If set, opcodes are written to memory; else, a dry-run.
@@ -94,12 +93,10 @@ struct arc_jit_data { · 8 unmodified lines
94	93	* need_extra_pass: A forecast if an "extra_pass" will occur.
95	94	* is_extra_pass: Indicates if the current pass is an extra pass.
96	95	* user_bpf_prog: True, if VM opcodes come from a real program.
97		− * blinded: True if "constant blinding" step returned a new "prog".
98	96	* success: Indicates if the whole JIT went OK.
99	97	*/
100	98	struct jit_context {
101	99	struct bpf_prog *prog;
102		− struct bpf_prog *orig_prog;
103	100	struct jit_buffer jit;
104	101	struct bpf_binary_header *bpf_header;
105	102	bool emit;
@@ -114,7 +111,6 @@ struct jit_context { · 8 unmodified lines
114	111	bool need_extra_pass;
115	112	bool is_extra_pass;
116	113	bool user_bpf_prog;
117		− bool blinded;
118	114	bool success;
119	115	};
120	116
@@ -161,13 +157,7 @@ static int jit_ctx_init(struct jit_context ctx, struct bpf_prog prog) · 40 unmodified lines
161	157	{
162	158	memset(ctx, 0, sizeof(*ctx));
163	159
164		− ctx->orig_prog = prog;
165		−
166		− /* If constant blinding was requested but failed, scram. */
167		− ctx->prog = bpf_jit_blind_constants(prog);
168		− if (IS_ERR(ctx->prog))
169		− return PTR_ERR(ctx->prog);
170		− ctx->blinded = (ctx->prog != ctx->orig_prog);
	160	+ ctx->prog = prog;
171	161
172	162	/* If the verifier doesn't zero-extend, then we have to do it. */
173	163	ctx->do_zext = !ctx->prog->aux->verifier_zext;
@@ -214,14 +204,6 @@ static inline void maybe_free(struct jit_context ctx, void *mem) · 40 unmodified lines
214	204	*/
215	205	static void jit_ctx_cleanup(struct jit_context *ctx)
216	206	{
217		− if (ctx->blinded) {
218		− /* if all went well, release the orig_prog. */
219		− if (ctx->success)
220		− bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog);
221		− else
222		− bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog);
223		− }
224		−
225	207	maybe_free(ctx, (void **)&ctx->bpf2insn);
226	208	maybe_free(ctx, (void **)&ctx->jit_data);
227	209
@@ -229,12 +211,19 @@ static void jit_ctx_cleanup(struct jit_context *ctx) · 1 unmodified line
229	211	ctx->bpf2insn_valid = false;
230	212
231	213	/* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */
232		− if (!ctx->success && ctx->bpf_header) {
233		− bpf_jit_binary_free(ctx->bpf_header);
234		− ctx->bpf_header = NULL;
235		− ctx->jit.buf = NULL;
236		− ctx->jit.index = 0;
237		− ctx->jit.len = 0;
	214	+ if (!ctx->success) {
	215	+ if (ctx->bpf_header) {
	216	+ bpf_jit_binary_free(ctx->bpf_header);
	217	+ ctx->bpf_header = NULL;
	218	+ ctx->jit.buf = NULL;
	219	+ ctx->jit.index = 0;
	220	+ ctx->jit.len = 0;
	221	+ }
	222	+ if (ctx->is_extra_pass) {
	223	+ ctx->prog->bpf_func = NULL;
	224	+ ctx->prog->jited = 0;
	225	+ ctx->prog->jited_len = 0;
	226	+ }
238	227	}
239	228
240	229	ctx->emit = false;
@@ -1411,7 +1400,7 @@ static struct bpf_prog do_extra_pass(struct bpf_prog prog) · 1170 unmodified lines
1411	1400	* (re)locations involved that their addresses are not known
1412	1401	* during the first run.
1413	1402	*/
1414		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	1403	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
1415	1404	{
1416	1405	vm_dump(prog);
1417	1406

arch/arm/net/bpf_jit_32.c+14−35

@@ -1852,6 +1852,9 @@ exit: · 1851 unmodified lines
1852	1852	{
1853	1853	u64 val = (u32)imm \| (u64)insn[1].imm << 32;
1854	1854
	1855	+ if (insn->src_reg == BPF_PSEUDO_FUNC)
	1856	+ goto notyet;
	1857	+
1855	1858	emit_a32_mov_i64(dst, val, ctx);
1856	1859
1857	1860	return 1;
@@ -2055,6 +2058,9 @@ go_jmp: · 197 unmodified lines
2055	2058	const s8 *r5 = bpf2a32[BPF_REG_5];
2056	2059	const u32 func = (u32)__bpf_call_base + (u32)imm;
2057	2060
	2061	+ if (insn->src_reg == BPF_PSEUDO_CALL)
	2062	+ goto notyet;
	2063	+
2058	2064	emit_a32_mov_r64(true, r0, r1, ctx);
2059	2065	emit_a32_mov_r64(true, r1, r2, ctx);
2060	2066	emit_push_r64(r5, ctx);
@@ -2142,11 +2148,9 @@ bool bpf_jit_needs_zext(void) · 81 unmodified lines
2142	2148	return true;
2143	2149	}
2144	2150
2145		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	2151	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
2146	2152	{
2147		− struct bpf_prog tmp, orig_prog = prog;
2148	2153	struct bpf_binary_header *header;
2149		− bool tmp_blinded = false;
2150	2154	struct jit_ctx ctx;
2151	2155	unsigned int tmp_idx;
2152	2156	unsigned int image_size;
@@ -2156,20 +2160,7 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 3 unmodified lines
2156	2160	* the interpreter.
2157	2161	*/
2158	2162	if (!prog->jit_requested)
2159		− return orig_prog;
2160		−
2161		− /* If constant blinding was enabled and we failed during blinding
2162		− * then we must fall back to the interpreter. Otherwise, we save
2163		− * the new JITed code.
2164		− */
2165		− tmp = bpf_jit_blind_constants(prog);
2166		−
2167		− if (IS_ERR(tmp))
2168		− return orig_prog;
2169		− if (tmp != prog) {
2170		− tmp_blinded = true;
2171		− prog = tmp;
2172		− }
	2163	+ return prog;
2173	2164
2174	2165	memset(&ctx, 0, sizeof(ctx));
2175	2166	ctx.prog = prog;
@@ -2179,10 +2170,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 3 unmodified lines
2179	2170	* we must fall back to the interpreter
2180	2171	*/
2181	2172	ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
2182		− if (ctx.offsets == NULL) {
2183		− prog = orig_prog;
2184		− goto out;
2185		− }
	2173	+ if (ctx.offsets == NULL)
	2174	+ return prog;
2186	2175
2187	2176	/* 1) fake pass to find in the length of the JITed code,
2188	2177	* to compute ctx->offsets and other context variables
@@ -2194,10 +2183,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 5 unmodified lines
2194	2183	* being successful in the second pass, so just fall back
2195	2184	* to the interpreter.
2196	2185	*/
2197		− if (build_body(&ctx)) {
2198		− prog = orig_prog;
	2186	+ if (build_body(&ctx))
2199	2187	goto out_off;
2200		− }
2201	2188
2202	2189	tmp_idx = ctx.idx;
2203	2190	build_prologue(&ctx);
@@ -2213,10 +2200,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 9 unmodified lines
2213	2200	ctx.idx += ctx.imm_count;
2214	2201	if (ctx.imm_count) {
2215	2202	ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);
2216		− if (ctx.imms == NULL) {
2217		− prog = orig_prog;
	2203	+ if (ctx.imms == NULL)
2218	2204	goto out_off;
2219		− }
2220	2205	}
2221	2206	#else
2222	2207	/* there's nothing about the epilogue on ARMv7 */
@@ -2238,10 +2223,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 15 unmodified lines
2238	2223	/* Not able to allocate memory for the structure then
2239	2224	* we must fall back to the interpretation
2240	2225	*/
2241		− if (header == NULL) {
2242		− prog = orig_prog;
	2226	+ if (header == NULL)
2243	2227	goto out_imms;
2244		− }
2245	2228
2246	2229	/* 2.) Actual pass to generate final JIT code */
2247	2230	ctx.target = (u32 *) image_ptr;
@@ -2278,16 +2261,12 @@ out_imms: · 30 unmodified lines
2278	2261	#endif
2279	2262	out_off:
2280	2263	kfree(ctx.offsets);
2281		−out:
2282		− if (tmp_blinded)
2283		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
2284		− tmp : orig_prog);
	2264	+
2285	2265	return prog;
2286	2266
2287	2267	out_free:
2288	2268	image_ptr = NULL;
2289	2269	bpf_jit_binary_free(header);
2290		− prog = orig_prog;
2291	2270	goto out_imms;
2292	2271	}
2293	2272

arch/arm64/lib/insn.c+2−0

@@ -338,6 +338,8 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, · 337 unmodified lines
338	338	long offset;
339	339
340	340	offset = label_imm_common(pc, addr, SZ_1M);
	341	+ if (offset >= SZ_1M)
	342	+ return AARCH64_BREAK_FAULT;
341	343
342	344	insn = aarch64_insn_get_bcond_value();
343	345

arch/arm64/net/bpf_jit_comp.c+31−72

@@ -18,7 +18,6 @@ · 17 unmodified lines
18	18
19	19	#include <asm/asm-extable.h>
20	20	#include <asm/byteorder.h>
21		−#include <asm/cacheflush.h>
22	21	#include <asm/cpufeature.h>
23	22	#include <asm/debug-monitors.h>
24	23	#include <asm/insn.h>
@@ -35,8 +34,8 @@ · 10 unmodified lines
35	34	#define ARENA_VM_START (MAX_BPF_JIT_REG + 5)
36	35
37	36	#define check_imm(bits, imm) do { \
38		− if ((((imm) > 0) && ((imm) >> (bits))) \|\| \
39		− (((imm) < 0) && (~(imm) >> (bits)))) { \
	37	+ if ((((imm) > 0) && ((imm) >> ((bits) - 1))) \|\| \
	38	+ (((imm) < 0) && (~(imm) >> ((bits) - 1)))) { \
40	39	pr_info("[%2d] imm=%d(0x%x) out of range\n", \
41	40	i, imm, imm); \
42	41	return -EINVAL; \
@@ -1198,8 +1197,8 @@ static int add_exception_handler(const struct bpf_insn *insn, · 1155 unmodified lines
1198	1197	* >0 - successfully JITed a 16-byte eBPF instruction.
1199	1198	* <0 - failed to JIT.
1200	1199	*/
1201		−static int build_insn(const struct bpf_insn insn, struct jit_ctx ctx,
1202		− bool extra_pass)
	1200	+static int build_insn(const struct bpf_verifier_env env, const struct bpf_insn insn,
	1201	+ struct jit_ctx *ctx, bool extra_pass)
1203	1202	{
1204	1203	const u8 code = insn->code;
1205	1204	u8 dst = bpf2a64[insn->dst_reg];
@@ -1224,6 +1223,9 @@ static int build_insn(const struct bpf_insn insn, struct jit_ctx ctx, · 18 unmodified lines
1224	1223	int ret;
1225	1224	bool sign_extend;
1226	1225
	1226	+ if (bpf_insn_is_indirect_target(env, ctx->prog, i))
	1227	+ emit_bti(A64_BTI_J, ctx);
	1228	+
1227	1229	switch (code) {
1228	1230	/* dst = src */
1229	1231	case BPF_ALU \| BPF_MOV \| BPF_X:
@@ -1899,7 +1901,7 @@ emit_cond_jmp: · 669 unmodified lines
1899	1901	return 0;
1900	1902	}
1901	1903
1902		−static int build_body(struct jit_ctx *ctx, bool extra_pass)
	1904	+static int build_body(struct bpf_verifier_env env, struct jit_ctx ctx, bool extra_pass)
1903	1905	{
1904	1906	const struct bpf_prog *prog = ctx->prog;
1905	1907	int i;
@@ -1918,7 +1920,7 @@ static int build_body(struct jit_ctx *ctx, bool extra_pass) · 12 unmodified lines
1918	1920	int ret;
1919	1921
1920	1922	ctx->offset[i] = ctx->idx;
1921		− ret = build_insn(insn, ctx, extra_pass);
	1923	+ ret = build_insn(env, insn, ctx, extra_pass);
1922	1924	if (ret > 0) {
1923	1925	i++;
1924	1926	ctx->offset[i] = ctx->idx;
@@ -1961,11 +1963,6 @@ static int validate_ctx(struct jit_ctx *ctx) · 36 unmodified lines
1961	1963	return 0;
1962	1964	}
1963	1965
1964		−static inline void bpf_flush_icache(void start, void end)
1965		−{
1966		− flush_icache_range((unsigned long)start, (unsigned long)end);
1967		−}
1968		−
1969	1966	static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
1970	1967	{
1971	1968	int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
@@ -2006,17 +2003,15 @@ struct arm64_jit_data { · 34 unmodified lines
2006	2003	struct jit_ctx ctx;
2007	2004	};
2008	2005
2009		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	2006	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
2010	2007	{
2011	2008	int image_size, prog_size, extable_size, extable_align, extable_offset;
2012		− struct bpf_prog tmp, orig_prog = prog;
2013	2009	struct bpf_binary_header *header;
2014	2010	struct bpf_binary_header *ro_header = NULL;
2015	2011	struct arm64_jit_data *jit_data;
2016	2012	void __percpu *priv_stack_ptr = NULL;
2017	2013	bool was_classic = bpf_prog_was_classic(prog);
2018	2014	int priv_stack_alloc_sz;
2019		− bool tmp_blinded = false;
2020	2015	bool extra_pass = false;
2021	2016	struct jit_ctx ctx;
2022	2017	u8 *image_ptr;
@@ -2025,26 +2020,13 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 2 unmodified lines
2025	2020	int exentry_idx;
2026	2021
2027	2022	if (!prog->jit_requested)
2028		− return orig_prog;
2029		−
2030		− tmp = bpf_jit_blind_constants(prog);
2031		− /* If blinding was requested and we failed during blinding,
2032		− * we must fall back to the interpreter.
2033		− */
2034		− if (IS_ERR(tmp))
2035		− return orig_prog;
2036		− if (tmp != prog) {
2037		− tmp_blinded = true;
2038		− prog = tmp;
2039		− }
	2023	+ return prog;
2040	2024
2041	2025	jit_data = prog->aux->jit_data;
2042	2026	if (!jit_data) {
2043	2027	jit_data = kzalloc_obj(*jit_data);
2044		− if (!jit_data) {
2045		− prog = orig_prog;
2046		− goto out;
2047		− }
	2028	+ if (!jit_data)
	2029	+ return prog;
2048	2030	prog->aux->jit_data = jit_data;
2049	2031	}
2050	2032	priv_stack_ptr = prog->aux->priv_stack_ptr;
@@ -2056,10 +2038,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 5 unmodified lines
2056	2038	priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +
2057	2039	2 * PRIV_STACK_GUARD_SZ;
2058	2040	priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL);
2059		− if (!priv_stack_ptr) {
2060		− prog = orig_prog;
	2041	+ if (!priv_stack_ptr)
2061	2042	goto out_priv_stack;
2062		− }
2063	2043
2064	2044	priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
2065	2045	prog->aux->priv_stack_ptr = priv_stack_ptr;
@@ -2079,10 +2059,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 13 unmodified lines
2079	2059	ctx.prog = prog;
2080	2060
2081	2061	ctx.offset = kvzalloc_objs(int, prog->len + 1);
2082		− if (ctx.offset == NULL) {
2083		− prog = orig_prog;
	2062	+ if (ctx.offset == NULL)
2084	2063	goto out_off;
2085		− }
2086	2064
2087	2065	ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
2088	2066	ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
@@ -2095,15 +2073,11 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 6 unmodified lines
2095	2073	* BPF line info needs ctx->offset[i] to be the offset of
2096	2074	* instruction[i] in jited image, so build prologue first.
2097	2075	*/
2098		− if (build_prologue(&ctx, was_classic)) {
2099		− prog = orig_prog;
	2076	+ if (build_prologue(&ctx, was_classic))
2100	2077	goto out_off;
2101		− }
2102	2078
2103		− if (build_body(&ctx, extra_pass)) {
2104		− prog = orig_prog;
	2079	+ if (build_body(env, &ctx, extra_pass))
2105	2080	goto out_off;
2106		− }
2107	2081
2108	2082	ctx.epilogue_offset = ctx.idx;
2109	2083	build_epilogue(&ctx, was_classic);
@@ -2121,10 +2095,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 11 unmodified lines
2121	2095	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,
2122	2096	sizeof(u64), &header, &image_ptr,
2123	2097	jit_fill_hole);
2124		− if (!ro_header) {
2125		− prog = orig_prog;
	2098	+ if (!ro_header)
2126	2099	goto out_off;
2127		− }
2128	2100
2129	2101	/* Pass 2: Determine jited position and result for each instruction */
2130	2102
@@ -2152,10 +2124,8 @@ skip_init_ctx: · 21 unmodified lines
2152	2124	/* Dont write body instructions to memory for now */
2153	2125	ctx.write = false;
2154	2126
2155		− if (build_body(&ctx, extra_pass)) {
2156		− prog = orig_prog;
	2127	+ if (build_body(env, &ctx, extra_pass))
2157	2128	goto out_free_hdr;
2158		− }
2159	2129
2160	2130	ctx.epilogue_offset = ctx.idx;
2161	2131	ctx.exentry_idx = exentry_idx;
@@ -2163,20 +2133,16 @@ skip_init_ctx: · 1 unmodified line
2163	2133	ctx.write = true;
2164	2134
2165	2135	/* Pass 3: Adjust jump offset and write final image */
2166		− if (build_body(&ctx, extra_pass) \|\|
2167		− WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) {
2168		− prog = orig_prog;
	2136	+ if (build_body(env, &ctx, extra_pass) \|\|
	2137	+ WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset))
2169	2138	goto out_free_hdr;
2170		− }
2171	2139
2172	2140	build_epilogue(&ctx, was_classic);
2173	2141	build_plt(&ctx);
2174	2142
2175	2143	/* Extra pass to validate JITed code. */
2176		− if (validate_ctx(&ctx)) {
2177		− prog = orig_prog;
	2144	+ if (validate_ctx(&ctx))
2178	2145	goto out_free_hdr;
2179		− }
2180	2146
2181	2147	/* update the real prog size */
2182	2148	prog_size = sizeof(u32) * ctx.idx;
@@ -2193,23 +2159,14 @@ skip_init_ctx: · 10 unmodified lines
2193	2159	if (extra_pass && ctx.idx > jit_data->ctx.idx) {
2194	2160	pr_err_once("multi-func JIT bug %d > %d\n",
2195	2161	ctx.idx, jit_data->ctx.idx);
2196		− prog->bpf_func = NULL;
2197		− prog->jited = 0;
2198		− prog->jited_len = 0;
2199	2162	goto out_free_hdr;
2200	2163	}
2201	2164	if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
2202		− /* ro_header has been freed */
	2165	+ /* ro_header and header has been freed */
2203	2166	ro_header = NULL;
2204		− prog = orig_prog;
2205		− goto out_off;
	2167	+ header = NULL;
	2168	+ goto out_free_hdr;
2206	2169	}
2207		− /*
2208		− * The instructions have now been copied to the ROX region from
2209		− * where they will execute. Now the data cache has to be cleaned to
2210		− * the PoU and the I-cache has to be invalidated for the VAs.
2211		− */
2212		− bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx);
2213	2170	} else {
2214	2171	jit_data->ctx = ctx;
2215	2172	jit_data->ro_image = ro_image_ptr;
@@ -2245,13 +2202,15 @@ out_priv_stack: · 29 unmodified lines
2245	2202	kfree(jit_data);
2246	2203	prog->aux->jit_data = NULL;
2247	2204	}
2248		−out:
2249		− if (tmp_blinded)
2250		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
2251		− tmp : orig_prog);
	2205	+
2252	2206	return prog;
2253	2207
2254	2208	out_free_hdr:
	2209	+ if (extra_pass) {
	2210	+ prog->bpf_func = NULL;
	2211	+ prog->jited = 0;
	2212	+ prog->jited_len = 0;
	2213	+ }
2255	2214	if (header) {
2256	2215	bpf_arch_text_copy(&ro_header->size, &header->size,
2257	2216	sizeof(header->size));

arch/loongarch/net/bpf_jit.c+18−43

@@ -1920,45 +1920,28 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 1919 unmodified lines
1920	1920	return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
1921	1921	}
1922	1922
1923		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	1923	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
1924	1924	{
1925		− bool tmp_blinded = false, extra_pass = false;
	1925	+ bool extra_pass = false;
1926	1926	u8 image_ptr, ro_image_ptr;
1927	1927	int image_size, prog_size, extable_size;
1928	1928	struct jit_ctx ctx;
1929	1929	struct jit_data *jit_data;
1930	1930	struct bpf_binary_header *header;
1931	1931	struct bpf_binary_header *ro_header;
1932		− struct bpf_prog tmp, orig_prog = prog;
1933	1932
1934	1933	/*
1935	1934	* If BPF JIT was not enabled then we must fall back to
1936	1935	* the interpreter.
1937	1936	*/
1938	1937	if (!prog->jit_requested)
1939		− return orig_prog;
1940		−
1941		− tmp = bpf_jit_blind_constants(prog);
1942		− /*
1943		− * If blinding was requested and we failed during blinding,
1944		− * we must fall back to the interpreter. Otherwise, we save
1945		− * the new JITed code.
1946		− */
1947		− if (IS_ERR(tmp))
1948		− return orig_prog;
1949		−
1950		− if (tmp != prog) {
1951		− tmp_blinded = true;
1952		− prog = tmp;
1953		− }
	1938	+ return prog;
1954	1939
1955	1940	jit_data = prog->aux->jit_data;
1956	1941	if (!jit_data) {
1957	1942	jit_data = kzalloc_obj(*jit_data);
1958		− if (!jit_data) {
1959		− prog = orig_prog;
1960		− goto out;
1961		− }
	1943	+ if (!jit_data)
	1944	+ return prog;
1962	1945	prog->aux->jit_data = jit_data;
1963	1946	}
1964	1947	if (jit_data->ctx.offset) {
@@ -1978,17 +1961,13 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 13 unmodified lines
1978	1961	ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
1979	1962
1980	1963	ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1981		− if (ctx.offset == NULL) {
1982		− prog = orig_prog;
	1964	+ if (ctx.offset == NULL)
1983	1965	goto out_offset;
1984		− }
1985	1966
1986	1967	/* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1987	1968	build_prologue(&ctx);
1988		− if (build_body(&ctx, extra_pass)) {
1989		− prog = orig_prog;
	1969	+ if (build_body(&ctx, extra_pass))
1990	1970	goto out_offset;
1991		− }
1992	1971	ctx.epilogue_offset = ctx.idx;
1993	1972	build_epilogue(&ctx);
1994	1973
@@ -2004,10 +1983,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 9 unmodified lines
2004	1983	/* Now we know the size of the structure to make */
2005	1984	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
2006	1985	&header, &image_ptr, jit_fill_hole);
2007		− if (!ro_header) {
2008		− prog = orig_prog;
	1986	+ if (!ro_header)
2009	1987	goto out_offset;
2010		− }
2011	1988
2012	1989	/* 2. Now, the actual pass to generate final JIT code */
2013	1990	/*
@@ -2027,17 +2004,13 @@ skip_init_ctx: · 13 unmodified lines
2027	2004	ctx.num_exentries = 0;
2028	2005
2029	2006	build_prologue(&ctx);
2030		− if (build_body(&ctx, extra_pass)) {
2031		− prog = orig_prog;
	2007	+ if (build_body(&ctx, extra_pass))
2032	2008	goto out_free;
2033		− }
2034	2009	build_epilogue(&ctx);
2035	2010
2036	2011	/* 3. Extra pass to validate JITed code */
2037		− if (validate_ctx(&ctx)) {
2038		− prog = orig_prog;
	2012	+ if (validate_ctx(&ctx))
2039	2013	goto out_free;
2040		− }
2041	2014
2042	2015	/* And we're done */
2043	2016	if (bpf_jit_enable > 1)
@@ -2050,9 +2023,9 @@ skip_init_ctx: · 6 unmodified lines
2050	2023	goto out_free;
2051	2024	}
2052	2025	if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
2053		− /* ro_header has been freed */
	2026	+ /* ro_header and header have been freed */
2054	2027	ro_header = NULL;
2055		− prog = orig_prog;
	2028	+ header = NULL;
2056	2029	goto out_free;
2057	2030	}
2058	2031	/*
@@ -2084,13 +2057,15 @@ out_offset: · 25 unmodified lines
2084	2057	prog->aux->jit_data = NULL;
2085	2058	}
2086	2059
2087		−out:
2088		− if (tmp_blinded)
2089		− bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
2090		−
2091	2060	return prog;
2092	2061
2093	2062	out_free:
	2063	+ if (extra_pass) {
	2064	+ prog->bpf_func = NULL;
	2065	+ prog->jited = 0;
	2066	+ prog->jited_len = 0;
	2067	+ }
	2068	+
2094	2069	if (header) {
2095	2070	bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size));
2096	2071	bpf_jit_binary_pack_free(ro_header, header);

arch/mips/net/bpf_jit_comp.c+2−20

@@ -909,12 +909,10 @@ bool bpf_jit_needs_zext(void) · 908 unmodified lines
909	909	return true;
910	910	}
911	911
912		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	912	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
913	913	{
914		− struct bpf_prog tmp, orig_prog = prog;
915	914	struct bpf_binary_header *header = NULL;
916	915	struct jit_context ctx;
917		− bool tmp_blinded = false;
918	916	unsigned int tmp_idx;
919	917	unsigned int image_size;
920	918	u8 *image_ptr;
@@ -925,19 +923,7 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 4 unmodified lines
925	923	* the interpreter.
926	924	*/
927	925	if (!prog->jit_requested)
928		− return orig_prog;
929		− /*
930		− * If constant blinding was enabled and we failed during blinding
931		− * then we must fall back to the interpreter. Otherwise, we save
932		− * the new JITed code.
933		− */
934		− tmp = bpf_jit_blind_constants(prog);
935		− if (IS_ERR(tmp))
936		− return orig_prog;
937		− if (tmp != prog) {
938		− tmp_blinded = true;
939		− prog = tmp;
940		− }
	926	+ return prog;
941	927
942	928	memset(&ctx, 0, sizeof(ctx));
943	929	ctx.program = prog;
@@ -1025,14 +1011,10 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 81 unmodified lines
1025	1011	prog->jited_len = image_size;
1026	1012
1027	1013	out:
1028		− if (tmp_blinded)
1029		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
1030		− tmp : orig_prog);
1031	1014	kfree(ctx.descriptors);
1032	1015	return prog;
1033	1016
1034	1017	out_err:
1035		− prog = orig_prog;
1036	1018	if (header)
1037	1019	bpf_jit_binary_free(header);
1038	1020	goto out;

arch/parisc/net/bpf_jit_core.c+30−45

@@ -41,33 +41,22 @@ bool bpf_jit_needs_zext(void) · 40 unmodified lines
41	41	return true;
42	42	}
43	43
44		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	44	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
45	45	{
46	46	unsigned int prog_size = 0, extable_size = 0;
47		− bool tmp_blinded = false, extra_pass = false;
48		− struct bpf_prog tmp, orig_prog = prog;
	47	+ bool extra_pass = false;
49	48	int pass = 0, prev_ninsns = 0, prologue_len, i;
50	49	struct hppa_jit_data *jit_data;
51	50	struct hppa_jit_context *ctx;
52	51
53	52	if (!prog->jit_requested)
54		− return orig_prog;
55		−
56		− tmp = bpf_jit_blind_constants(prog);
57		− if (IS_ERR(tmp))
58		− return orig_prog;
59		− if (tmp != prog) {
60		− tmp_blinded = true;
61		− prog = tmp;
62		− }
	53	+ return prog;
63	54
64	55	jit_data = prog->aux->jit_data;
65	56	if (!jit_data) {
66	57	jit_data = kzalloc_obj(*jit_data);
67		− if (!jit_data) {
68		− prog = orig_prog;
69		− goto out;
70		− }
	58	+ if (!jit_data)
	59	+ return prog;
71	60	prog->aux->jit_data = jit_data;
72	61	}
73	62
@@ -81,10 +70,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 7 unmodified lines
81	70
82	71	ctx->prog = prog;
83	72	ctx->offset = kzalloc_objs(int, prog->len);
84		− if (!ctx->offset) {
85		− prog = orig_prog;
86		− goto out_offset;
87		− }
	73	+ if (!ctx->offset)
	74	+ goto out_err;
88	75	for (i = 0; i < prog->len; i++) {
89	76	prev_ninsns += 20;
90	77	ctx->offset[i] = prev_ninsns;
@@ -93,10 +80,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 2 unmodified lines
93	80	for (i = 0; i < NR_JIT_ITERATIONS; i++) {
94	81	pass++;
95	82	ctx->ninsns = 0;
96		− if (build_body(ctx, extra_pass, ctx->offset)) {
97		− prog = orig_prog;
98		− goto out_offset;
99		− }
	83	+ if (build_body(ctx, extra_pass, ctx->offset))
	84	+ goto out_err;
100	85	ctx->body_len = ctx->ninsns;
101	86	bpf_jit_build_prologue(ctx);
102	87	ctx->prologue_len = ctx->ninsns - ctx->body_len;
@@ -116,10 +101,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 13 unmodified lines
116	101	&jit_data->image,
117	102	sizeof(long),
118	103	bpf_fill_ill_insns);
119		− if (!jit_data->header) {
120		− prog = orig_prog;
121		− goto out_offset;
122		− }
	104	+ if (!jit_data->header)
	105	+ goto out_err;
123	106
124	107	ctx->insns = (u32 *)jit_data->image;
125	108	/*
@@ -134,8 +117,7 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 8 unmodified lines
134	117	pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
135	118	if (jit_data->header)
136	119	bpf_jit_binary_free(jit_data->header);
137		− prog = orig_prog;
138		− goto out_offset;
	120	+ goto out_err;
139	121	}
140	122
141	123	if (extable_size)
@@ -148,8 +130,7 @@ skip_init_ctx: · 6 unmodified lines
148	130	bpf_jit_build_prologue(ctx);
149	131	if (build_body(ctx, extra_pass, NULL)) {
150	132	bpf_jit_binary_free(jit_data->header);
151		− prog = orig_prog;
152		− goto out_offset;
	133	+ goto out_err;
153	134	}
154	135	bpf_jit_build_epilogue(ctx);
155	136
@@ -160,20 +141,19 @@ skip_init_ctx: · 4 unmodified lines
160	141	{ extern int machine_restart(char *); machine_restart(""); }
161	142	}
162	143
	144	+ if (!prog->is_func \|\| extra_pass) {
	145	+ if (bpf_jit_binary_lock_ro(jit_data->header)) {
	146	+ bpf_jit_binary_free(jit_data->header);
	147	+ goto out_err;
	148	+ }
	149	+ bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
	150	+ }
	151	+
163	152	prog->bpf_func = (void *)ctx->insns;
164	153	prog->jited = 1;
165	154	prog->jited_len = prog_size;
166	155
167		− bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
168		−
169	156	if (!prog->is_func \|\| extra_pass) {
170		− if (bpf_jit_binary_lock_ro(jit_data->header)) {
171		− bpf_jit_binary_free(jit_data->header);
172		− prog->bpf_func = NULL;
173		− prog->jited = 0;
174		− prog->jited_len = 0;
175		− goto out_offset;
176		− }
177	157	prologue_len = ctx->epilogue_offset - ctx->body_len;
178	158	for (i = 0; i < prog->len; i++)
179	159	ctx->offset[i] += prologue_len;
@@ -183,14 +163,19 @@ out_offset: · 3 unmodified lines
183	163	kfree(jit_data);
184	164	prog->aux->jit_data = NULL;
185	165	}
186		−out:
	166	+
187	167	if (HPPA_JIT_REBOOT)
188	168	{ extern int machine_restart(char *); machine_restart(""); }
189	169
190		− if (tmp_blinded)
191		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
192		− tmp : orig_prog);
193	170	return prog;
	171	+
	172	+out_err:
	173	+ if (extra_pass) {
	174	+ prog->bpf_func = NULL;
	175	+ prog->jited = 0;
	176	+ prog->jited_len = 0;
	177	+ }
	178	+ goto out_offset;
194	179	}
195	180
196	181	u64 hppa_div64(u64 div, u64 divisor)

arch/powerpc/net/bpf_jit_comp.c+27−47

@@ -162,7 +162,7 @@ static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size · 161 unmodified lines
162	162	}
163	163	}
164	164
165		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp)
	165	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *fp)
166	166	{
167	167	u32 proglen;
168	168	u32 alloclen;
@@ -177,9 +177,6 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 8 unmodified lines
177	177	void __percpu *priv_stack_ptr = NULL;
178	178	struct bpf_binary_header *fhdr = NULL;
179	179	struct bpf_binary_header *hdr = NULL;
180		− struct bpf_prog *org_fp = fp;
181		− struct bpf_prog *tmp_fp = NULL;
182		− bool bpf_blinded = false;
183	180	bool extra_pass = false;
184	181	u8 *fimage = NULL;
185	182	u32 *fcode_base = NULL;
@@ -187,24 +184,13 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 1 unmodified line
187	184	u32 fixup_len;
188	185
189	186	if (!fp->jit_requested)
190		− return org_fp;
191		−
192		− tmp_fp = bpf_jit_blind_constants(org_fp);
193		− if (IS_ERR(tmp_fp))
194		− return org_fp;
195		−
196		− if (tmp_fp != org_fp) {
197		− bpf_blinded = true;
198		− fp = tmp_fp;
199		− }
	187	+ return fp;
200	188
201	189	jit_data = fp->aux->jit_data;
202	190	if (!jit_data) {
203	191	jit_data = kzalloc_obj(*jit_data);
204		− if (!jit_data) {
205		− fp = org_fp;
206		− goto out;
207		− }
	192	+ if (!jit_data)
	193	+ return fp;
208	194	fp->aux->jit_data = jit_data;
209	195	}
210	196
@@ -219,10 +205,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 8 unmodified lines
219	205	priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) +
220	206	2 * PRIV_STACK_GUARD_SZ;
221	207	priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL);
222		− if (!priv_stack_ptr) {
223		− fp = org_fp;
	208	+ if (!priv_stack_ptr)
224	209	goto out_priv_stack;
225		− }
226	210
227	211	priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size);
228	212	fp->aux->priv_stack_ptr = priv_stack_ptr;
@@ -249,10 +233,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 20 unmodified lines
249	233	}
250	234
251	235	addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
252		− if (addrs == NULL) {
253		− fp = org_fp;
254		− goto out_addrs;
255		− }
	236	+ if (addrs == NULL)
	237	+ goto out_err;
256	238
257	239	memset(&cgctx, 0, sizeof(struct codegen_context));
258	240	bpf_jit_init_reg_mapping(&cgctx);
@@ -279,11 +261,9 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 20 unmodified lines
279	261	}
280	262
281	263	/* Scouting faux-generate pass 0 */
282		− if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
	264	+ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
283	265	/* We hit something illegal or unsupported. */
284		− fp = org_fp;
285		− goto out_addrs;
286		− }
	266	+ goto out_err;
287	267
288	268	/*
289	269	* If we have seen a tail call, we need a second pass.
@@ -294,10 +274,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 4 unmodified lines
294	274	*/
295	275	if (cgctx.seen & SEEN_TAILCALL \|\| !is_offset_in_branch_range((long)cgctx.idx * 4)) {
296	276	cgctx.idx = 0;
297		− if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {
298		− fp = org_fp;
299		− goto out_addrs;
300		− }
	277	+ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
	278	+ goto out_err;
301	279	}
302	280
303	281	bpf_jit_realloc_regs(&cgctx);
@@ -318,10 +296,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 14 unmodified lines
318	296
319	297	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
320	298	bpf_jit_fill_ill_insns);
321		− if (!fhdr) {
322		− fp = org_fp;
323		− goto out_addrs;
324		− }
	299	+ if (!fhdr)
	300	+ goto out_err;
325	301
326	302	if (extable_len)
327	303	fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
@@ -340,8 +316,7 @@ skip_init_ctx: · 12 unmodified lines
340	316	extra_pass)) {
341	317	bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
342	318	bpf_jit_binary_pack_free(fhdr, hdr);
343		− fp = org_fp;
344		− goto out_addrs;
	319	+ goto out_err;
345	320	}
346	321	bpf_jit_build_epilogue(code_base, &cgctx);
347	322
@@ -363,15 +338,16 @@ skip_init_ctx: · 15 unmodified lines
363	338	((u64 *)image)[1] = local_paca->kernel_toc;
364	339	#endif
365	340
	341	+ if (!fp->is_func \|\| extra_pass) {
	342	+ if (bpf_jit_binary_pack_finalize(fhdr, hdr))
	343	+ goto out_err;
	344	+ }
	345	+
366	346	fp->bpf_func = (void *)fimage;
367	347	fp->jited = 1;
368	348	fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE;
369	349
370	350	if (!fp->is_func \|\| extra_pass) {
371		− if (bpf_jit_binary_pack_finalize(fhdr, hdr)) {
372		− fp = org_fp;
373		− goto out_addrs;
374		− }
375	351	bpf_prog_fill_jited_linfo(fp, addrs);
376	352	/*
377	353	* On ABI V1, executable code starts after the function
@@ -398,11 +374,15 @@ out_priv_stack: · 20 unmodified lines
398	374	jit_data->hdr = hdr;
399	375	}
400	376
401		−out:
402		− if (bpf_blinded)
403		− bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
404		−
405	377	return fp;
	378	+
	379	+out_err:
	380	+ if (extra_pass) {
	381	+ fp->bpf_func = NULL;
	382	+ fp->jited = 0;
	383	+ fp->jited_len = 0;
	384	+ }
	385	+ goto out_addrs;
406	386	}
407	387
408	388	/*

arch/riscv/net/bpf_jit.h+0−6

@@ -11,7 +11,6 @@ · 10 unmodified lines
11	11
12	12	#include <linux/bpf.h>
13	13	#include <linux/filter.h>
14		−#include <asm/cacheflush.h>
15	14
16	15	/* verify runtime detection extension status */
17	16	#define rv_ext_enabled(ext) \
@@ -105,11 +104,6 @@ static inline void bpf_fill_ill_insns(void *area, unsigned int size) · 87 unmodified lines
105	104	memset(area, 0, size);
106	105	}
107	106
108		−static inline void bpf_flush_icache(void start, void end)
109		−{
110		− flush_icache_range((unsigned long)start, (unsigned long)end);
111		−}
112		−
113	107	/* Emit a 4-byte riscv instruction. */
114	108	static inline void emit(const u32 insn, struct rv_jit_context *ctx)
115	109	{

arch/riscv/net/bpf_jit_core.c+23−47

@@ -41,32 +41,22 @@ bool bpf_jit_needs_zext(void) · 40 unmodified lines
41	41	return true;
42	42	}
43	43
44		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	44	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
45	45	{
46	46	unsigned int prog_size = 0, extable_size = 0;
47		− bool tmp_blinded = false, extra_pass = false;
48		− struct bpf_prog tmp, orig_prog = prog;
	47	+ bool extra_pass = false;
49	48	int pass = 0, prev_ninsns = 0, i;
50	49	struct rv_jit_data *jit_data;
51	50	struct rv_jit_context *ctx;
52	51
53	52	if (!prog->jit_requested)
54		− return orig_prog;
55		−
56		− tmp = bpf_jit_blind_constants(prog);
57		− if (IS_ERR(tmp))
58		− return orig_prog;
59		− if (tmp != prog) {
60		− tmp_blinded = true;
61		− prog = tmp;
62		− }
	53	+ return prog;
63	54
64	55	jit_data = prog->aux->jit_data;
65	56	if (!jit_data) {
66	57	jit_data = kzalloc_obj(*jit_data);
67	58	if (!jit_data) {
68		− prog = orig_prog;
69		− goto out;
	59	+ return prog;
70	60	}
71	61	prog->aux->jit_data = jit_data;
72	62	}
@@ -83,15 +73,11 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 10 unmodified lines
83	73	ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
84	74	ctx->prog = prog;
85	75	ctx->offset = kzalloc_objs(int, prog->len);
86		− if (!ctx->offset) {
87		− prog = orig_prog;
	76	+ if (!ctx->offset)
88	77	goto out_offset;
89		− }
90	78
91		− if (build_body(ctx, extra_pass, NULL)) {
92		− prog = orig_prog;
	79	+ if (build_body(ctx, extra_pass, NULL))
93	80	goto out_offset;
94		− }
95	81
96	82	for (i = 0; i < prog->len; i++) {
97	83	prev_ninsns += 32;
@@ -105,10 +91,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 7 unmodified lines
105	91	bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
106	92	ctx->prologue_len = ctx->ninsns;
107	93
108		− if (build_body(ctx, extra_pass, ctx->offset)) {
109		− prog = orig_prog;
	94	+ if (build_body(ctx, extra_pass, ctx->offset))
110	95	goto out_offset;
111		− }
112	96
113	97	ctx->epilogue_offset = ctx->ninsns;
114	98	bpf_jit_build_epilogue(ctx);
@@ -126,10 +110,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 11 unmodified lines
126	110	&jit_data->ro_image, sizeof(u32),
127	111	&jit_data->header, &jit_data->image,
128	112	bpf_fill_ill_insns);
129		− if (!jit_data->ro_header) {
130		− prog = orig_prog;
	113	+ if (!jit_data->ro_header)
131	114	goto out_offset;
132		− }
133	115
134	116	/*
135	117	* Use the image(RW) for writing the JITed instructions. But also save
@@ -150,7 +132,6 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 14 unmodified lines
150	132
151	133	if (i == NR_JIT_ITERATIONS) {
152	134	pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
153		− prog = orig_prog;
154	135	goto out_free_hdr;
155	136	}
156	137
@@ -163,33 +144,27 @@ skip_init_ctx: · 6 unmodified lines
163	144	ctx->nexentries = 0;
164	145
165	146	bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
166		− if (build_body(ctx, extra_pass, NULL)) {
167		− prog = orig_prog;
	147	+ if (build_body(ctx, extra_pass, NULL))
168	148	goto out_free_hdr;
169		− }
170	149	bpf_jit_build_epilogue(ctx);
171	150
172	151	if (bpf_jit_enable > 1)
173	152	bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
174	153
175		− prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset();
176		− prog->jited = 1;
177		− prog->jited_len = prog_size - cfi_get_offset();
178		−
179	154	if (!prog->is_func \|\| extra_pass) {
180	155	if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) {
181	156	/* ro_header has been freed */
182	157	jit_data->ro_header = NULL;
183		− prog = orig_prog;
184		− goto out_offset;
	158	+ jit_data->header = NULL;
	159	+ goto out_free_hdr;
185	160	}
186		− /*
187		− * The instructions have now been copied to the ROX region from
188		− * where they will execute.
189		− * Write any modified data cache blocks out to memory and
190		− * invalidate the corresponding blocks in the instruction cache.
191		− */
192		− bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
	161	+ }
	162	+
	163	+ prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset();
	164	+ prog->jited = 1;
	165	+ prog->jited_len = prog_size - cfi_get_offset();
	166	+
	167	+ if (!prog->is_func \|\| extra_pass) {
193	168	for (i = 0; i < prog->len; i++)
194	169	ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
195	170	bpf_prog_fill_jited_linfo(prog, ctx->offset);
@@ -198,14 +173,15 @@ out_offset: · 2 unmodified lines
198	173	kfree(jit_data);
199	174	prog->aux->jit_data = NULL;
200	175	}
201		−out:
202	176
203		− if (tmp_blinded)
204		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
205		− tmp : orig_prog);
206	177	return prog;
207	178
208	179	out_free_hdr:
	180	+ if (extra_pass) {
	181	+ prog->bpf_func = NULL;
	182	+ prog->jited = 0;
	183	+ prog->jited_len = 0;
	184	+ }
209	185	if (jit_data->header) {
210	186	bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
211	187	sizeof(jit_data->header->size));

arch/s390/net/bpf_jit_comp.c+21−40

@@ -2312,38 +2312,22 @@ static struct bpf_binary_header bpf_jit_alloc(struct bpf_jit jit, · 2311 unmodified lines
2312	2312	/*
2313	2313	* Compile eBPF program "fp"
2314	2314	*/
2315		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp)
	2315	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *fp)
2316	2316	{
2317		− struct bpf_prog tmp, orig_fp = fp;
2318	2317	struct bpf_binary_header *header;
2319	2318	struct s390_jit_data *jit_data;
2320		− bool tmp_blinded = false;
2321	2319	bool extra_pass = false;
2322	2320	struct bpf_jit jit;
2323	2321	int pass;
2324	2322
2325	2323	if (!fp->jit_requested)
2326		− return orig_fp;
2327		−
2328		− tmp = bpf_jit_blind_constants(fp);
2329		− /*
2330		− * If blinding was requested and we failed during blinding,
2331		− * we must fall back to the interpreter.
2332		− */
2333		− if (IS_ERR(tmp))
2334		− return orig_fp;
2335		− if (tmp != fp) {
2336		− tmp_blinded = true;
2337		− fp = tmp;
2338		− }
	2324	+ return fp;
2339	2325
2340	2326	jit_data = fp->aux->jit_data;
2341	2327	if (!jit_data) {
2342	2328	jit_data = kzalloc_obj(*jit_data);
2343		− if (!jit_data) {
2344		− fp = orig_fp;
2345		− goto out;
2346		− }
	2329	+ if (!jit_data)
	2330	+ return fp;
2347	2331	fp->aux->jit_data = jit_data;
2348	2332	}
2349	2333	if (jit_data->ctx.addrs) {
@@ -2356,34 +2340,27 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog fp) · 6 unmodified lines
2356	2340
2357	2341	memset(&jit, 0, sizeof(jit));
2358	2342	jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
2359		− if (jit.addrs == NULL) {
2360		− fp = orig_fp;
2361		− goto free_addrs;
2362		− }
	2343	+ if (jit.addrs == NULL)
	2344	+ goto out_err;
2363	2345	/*
2364	2346	* Three initial passes:
2365	2347	* - 1/2: Determine clobbered registers
2366	2348	* - 3: Calculate program size and addrs array
2367	2349	*/
2368	2350	for (pass = 1; pass <= 3; pass++) {
2369		− if (bpf_jit_prog(&jit, fp, extra_pass)) {
2370		− fp = orig_fp;
2371		− goto free_addrs;
2372		− }
	2351	+ if (bpf_jit_prog(&jit, fp, extra_pass))
	2352	+ goto out_err;
2373	2353	}
2374	2354	/*
2375	2355	* Final pass: Allocate and generate program
2376	2356	*/
2377	2357	header = bpf_jit_alloc(&jit, fp);
2378		− if (!header) {
2379		− fp = orig_fp;
2380		− goto free_addrs;
2381		− }
	2358	+ if (!header)
	2359	+ goto out_err;
2382	2360	skip_init_ctx:
2383	2361	if (bpf_jit_prog(&jit, fp, extra_pass)) {
2384	2362	bpf_jit_binary_free(header);
2385		− fp = orig_fp;
2386		− goto free_addrs;
	2363	+ goto out_err;
2387	2364	}
2388	2365	if (bpf_jit_enable > 1) {
2389	2366	bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
@@ -2392,8 +2369,7 @@ skip_init_ctx: · 2 unmodified lines
2392	2369	if (!fp->is_func \|\| extra_pass) {
2393	2370	if (bpf_jit_binary_lock_ro(header)) {
2394	2371	bpf_jit_binary_free(header);
2395		− fp = orig_fp;
2396		− goto free_addrs;
	2372	+ goto out_err;
2397	2373	}
2398	2374	} else {
2399	2375	jit_data->header = header;
@@ -2411,11 +2387,16 @@ free_addrs: · 11 unmodified lines
2411	2387	kfree(jit_data);
2412	2388	fp->aux->jit_data = NULL;
2413	2389	}
2414		−out:
2415		− if (tmp_blinded)
2416		− bpf_jit_prog_release_other(fp, fp == orig_fp ?
2417		− tmp : orig_fp);
	2390	+
2418	2391	return fp;
	2392	+
	2393	+out_err:
	2394	+ if (extra_pass) {
	2395	+ fp->bpf_func = NULL;
	2396	+ fp->jited = 0;
	2397	+ fp->jited_len = 0;
	2398	+ }
	2399	+ goto free_addrs;
2419	2400	}
2420	2401
2421	2402	bool bpf_jit_supports_kfunc_call(void)

arch/sparc/net/bpf_jit_comp_64.c+22−41

@@ -1477,39 +1477,24 @@ struct sparc64_jit_data { · 1476 unmodified lines
1477	1477	struct jit_ctx ctx;
1478	1478	};
1479	1479
1480		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	1480	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
1481	1481	{
1482		− struct bpf_prog tmp, orig_prog = prog;
1483	1482	struct sparc64_jit_data *jit_data;
1484	1483	struct bpf_binary_header *header;
1485	1484	u32 prev_image_size, image_size;
1486		− bool tmp_blinded = false;
1487	1485	bool extra_pass = false;
1488	1486	struct jit_ctx ctx;
1489	1487	u8 *image_ptr;
1490	1488	int pass, i;
1491	1489
1492	1490	if (!prog->jit_requested)
1493		− return orig_prog;
1494		−
1495		− tmp = bpf_jit_blind_constants(prog);
1496		− /* If blinding was requested and we failed during blinding,
1497		− * we must fall back to the interpreter.
1498		− */
1499		− if (IS_ERR(tmp))
1500		− return orig_prog;
1501		− if (tmp != prog) {
1502		− tmp_blinded = true;
1503		− prog = tmp;
1504		− }
	1491	+ return prog;
1505	1492
1506	1493	jit_data = prog->aux->jit_data;
1507	1494	if (!jit_data) {
1508	1495	jit_data = kzalloc_obj(*jit_data);
1509		− if (!jit_data) {
1510		− prog = orig_prog;
1511		− goto out;
1512		− }
	1496	+ if (!jit_data)
	1497	+ return prog;
1513	1498	prog->aux->jit_data = jit_data;
1514	1499	}
1515	1500	if (jit_data->ctx.offset) {
@@ -1527,10 +1512,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 11 unmodified lines
1527	1512	ctx.prog = prog;
1528	1513
1529	1514	ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);
1530		− if (ctx.offset == NULL) {
1531		− prog = orig_prog;
1532		− goto out_off;
1533		− }
	1515	+ if (ctx.offset == NULL)
	1516	+ goto out_err;
1534	1517
1535	1518	/* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook
1536	1519	* the offset array so that we converge faster.
@@ -1543,10 +1526,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 6 unmodified lines
1543	1526	ctx.idx = 0;
1544	1527
1545	1528	build_prologue(&ctx);
1546		− if (build_body(&ctx)) {
1547		− prog = orig_prog;
1548		− goto out_off;
1549		− }
	1529	+ if (build_body(&ctx))
	1530	+ goto out_err;
1550	1531	build_epilogue(&ctx);
1551	1532
1552	1533	if (bpf_jit_enable > 1)
@@ -1569,10 +1550,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 16 unmodified lines
1569	1550	image_size = sizeof(u32) * ctx.idx;
1570	1551	header = bpf_jit_binary_alloc(image_size, &image_ptr,
1571	1552	sizeof(u32), jit_fill_hole);
1572		− if (header == NULL) {
1573		− prog = orig_prog;
1574		− goto out_off;
1575		− }
	1553	+ if (header == NULL)
	1554	+ goto out_err;
1576	1555
1577	1556	ctx.image = (u32 *)image_ptr;
1578	1557	skip_init_ctx:
@@ -1582,8 +1561,7 @@ skip_init_ctx: · 3 unmodified lines
1582	1561
1583	1562	if (build_body(&ctx)) {
1584	1563	bpf_jit_binary_free(header);
1585		− prog = orig_prog;
1586		− goto out_off;
	1564	+ goto out_err;
1587	1565	}
1588	1566
1589	1567	build_epilogue(&ctx);
@@ -1592,8 +1570,7 @@ skip_init_ctx: · 2 unmodified lines
1592	1570	pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
1593	1571	prev_image_size, ctx.idx * 4);
1594	1572	bpf_jit_binary_free(header);
1595		− prog = orig_prog;
1596		− goto out_off;
	1573	+ goto out_err;
1597	1574	}
1598	1575
1599	1576	if (bpf_jit_enable > 1)
@@ -1604,8 +1581,7 @@ skip_init_ctx: · 4 unmodified lines
1604	1581	if (!prog->is_func \|\| extra_pass) {
1605	1582	if (bpf_jit_binary_lock_ro(header)) {
1606	1583	bpf_jit_binary_free(header);
1607		− prog = orig_prog;
1608		− goto out_off;
	1584	+ goto out_err;
1609	1585	}
1610	1586	} else {
1611	1587	jit_data->ctx = ctx;
@@ -1624,9 +1600,14 @@ out_off: · 12 unmodified lines
1624	1600	kfree(jit_data);
1625	1601	prog->aux->jit_data = NULL;
1626	1602	}
1627		−out:
1628		− if (tmp_blinded)
1629		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
1630		− tmp : orig_prog);
	1603	+
1631	1604	return prog;
	1605	+
	1606	+out_err:
	1607	+ if (extra_pass) {
	1608	+ prog->bpf_func = NULL;
	1609	+ prog->jited = 0;
	1610	+ prog->jited_len = 0;
	1611	+ }
	1612	+ goto out_off;
1632	1613	}

arch/x86/net/bpf_jit_comp.c+23−50

@@ -58,8 +58,8 @@ static u8 emit_code(u8 ptr, u32 bytes, unsigned int len) · 57 unmodified lines
58	58	#define EMIT_ENDBR() EMIT(gen_endbr(), 4)
59	59	#define EMIT_ENDBR_POISON() EMIT(gen_endbr_poison(), 4)
60	60	#else
61		−#define EMIT_ENDBR()
62		−#define EMIT_ENDBR_POISON()
	61	+#define EMIT_ENDBR() do { } while (0)
	62	+#define EMIT_ENDBR_POISON() do { } while (0)
63	63	#endif
64	64
65	65	static bool is_imm8(int value)
@@ -1649,8 +1649,8 @@ static int emit_spectre_bhb_barrier(u8 *pprog, u8 ip, · 1583 unmodified lines
1649	1649	return 0;
1650	1650	}
1651	1651
1652		−static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 image, u8 rw_image,
1653		− int oldproglen, struct jit_context *ctx, bool jmp_padding)
	1652	+static int do_jit(struct bpf_verifier_env env, struct bpf_prog bpf_prog, int addrs, u8 image,
	1653	+ u8 rw_image, int oldproglen, struct jit_context ctx, bool jmp_padding)
1654	1654	{
1655	1655	bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
1656	1656	struct bpf_insn *insn = bpf_prog->insnsi;
@@ -1663,7 +1663,7 @@ static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 image, u8 rw_image · 6 unmodified lines
1663	1663	void __percpu *priv_stack_ptr;
1664	1664	int i, excnt = 0;
1665	1665	int ilen, proglen = 0;
1666		− u8 *prog = temp;
	1666	+ u8 ip, prog = temp;
1667	1667	u32 stack_depth;
1668	1668	int err;
1669	1669
@@ -1734,6 +1734,11 @@ static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 image, u8 rw_image · 64 unmodified lines
1734	1734	dst_reg = X86_REG_R9;
1735	1735	}
1736	1736
	1737	+ if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1))
	1738	+ EMIT_ENDBR();
	1739	+
	1740	+ ip = image + addrs[i - 1] + (prog - temp);
	1741	+
1737	1742	switch (insn->code) {
1738	1743	/* ALU */
1739	1744	case BPF_ALU \| BPF_ADD \| BPF_X:
@@ -2440,8 +2445,6 @@ populate_extable: · 700 unmodified lines
2440	2445
2441	2446	/* call */
2442	2447	case BPF_JMP \| BPF_CALL: {
2443		− u8 *ip = image + addrs[i - 1];
2444		−
2445	2448	func = (u8 *) __bpf_call_base + imm32;
2446	2449	if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
2447	2450	LOAD_TAIL_CALL_CNT_PTR(stack_depth);
@@ -2465,7 +2468,8 @@ populate_extable: · 17 unmodified lines
2465	2468	if (imm32)
2466	2469	emit_bpf_tail_call_direct(bpf_prog,
2467	2470	&bpf_prog->aux->poke_tab[imm32 - 1],
2468		− &prog, image + addrs[i - 1],
	2471	+ &prog,
	2472	+ ip,
2469	2473	callee_regs_used,
2470	2474	stack_depth,
2471	2475	ctx);
@@ -2474,7 +2478,7 @@ populate_extable: · 2 unmodified lines
2474	2478	&prog,
2475	2479	callee_regs_used,
2476	2480	stack_depth,
2477		− image + addrs[i - 1],
	2481	+ ip,
2478	2482	ctx);
2479	2483	break;
2480	2484
@@ -2639,7 +2643,7 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 158 unmodified lines
2639	2643	break;
2640	2644
2641	2645	case BPF_JMP \| BPF_JA \| BPF_X:
2642		− emit_indirect_jump(&prog, insn->dst_reg, image + addrs[i - 1]);
	2646	+ emit_indirect_jump(&prog, insn->dst_reg, ip);
2643	2647	break;
2644	2648	case BPF_JMP \| BPF_JA:
2645	2649	case BPF_JMP32 \| BPF_JA:
@@ -2729,8 +2733,6 @@ emit_jmp: · 83 unmodified lines
2729	2733	ctx->cleanup_addr = proglen;
2730	2734	if (bpf_prog_was_classic(bpf_prog) &&
2731	2735	!ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) {
2732		− u8 *ip = image + addrs[i - 1];
2733		−
2734	2736	if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog))
2735	2737	return -EINVAL;
2736	2738	}
@@ -3713,17 +3715,15 @@ struct x64_jit_data { · 976 unmodified lines
3713	3715	#define MAX_PASSES 20
3714	3716	#define PADDING_PASSES (MAX_PASSES - 5)
3715	3717
3716		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	3718	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
3717	3719	{
3718	3720	struct bpf_binary_header *rw_header = NULL;
3719	3721	struct bpf_binary_header *header = NULL;
3720		− struct bpf_prog tmp, orig_prog = prog;
3721	3722	void __percpu *priv_stack_ptr = NULL;
3722	3723	struct x64_jit_data *jit_data;
3723	3724	int priv_stack_alloc_sz;
3724	3725	int proglen, oldproglen = 0;
3725	3726	struct jit_context ctx = {};
3726		− bool tmp_blinded = false;
3727	3727	bool extra_pass = false;
3728	3728	bool padding = false;
3729	3729	u8 *rw_image = NULL;
@@ -3733,27 +3733,13 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 3 unmodified lines
3733	3733	int i;
3734	3734
3735	3735	if (!prog->jit_requested)
3736		− return orig_prog;
3737		−
3738		− tmp = bpf_jit_blind_constants(prog);
3739		− /*
3740		− * If blinding was requested and we failed during blinding,
3741		− * we must fall back to the interpreter.
3742		− */
3743		− if (IS_ERR(tmp))
3744		− return orig_prog;
3745		− if (tmp != prog) {
3746		− tmp_blinded = true;
3747		− prog = tmp;
3748		− }
	3736	+ return prog;
3749	3737
3750	3738	jit_data = prog->aux->jit_data;
3751	3739	if (!jit_data) {
3752	3740	jit_data = kzalloc_obj(*jit_data);
3753		− if (!jit_data) {
3754		− prog = orig_prog;
3755		− goto out;
3756		− }
	3741	+ if (!jit_data)
	3742	+ return prog;
3757	3743	prog->aux->jit_data = jit_data;
3758	3744	}
3759	3745	priv_stack_ptr = prog->aux->priv_stack_ptr;
@@ -3765,10 +3751,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 5 unmodified lines
3765	3751	priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
3766	3752	2 * PRIV_STACK_GUARD_SZ;
3767	3753	priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL);
3768		− if (!priv_stack_ptr) {
3769		− prog = orig_prog;
	3754	+ if (!priv_stack_ptr)
3770	3755	goto out_priv_stack;
3771		− }
3772	3756
3773	3757	priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
3774	3758	prog->aux->priv_stack_ptr = priv_stack_ptr;
@@ -3786,10 +3770,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 11 unmodified lines
3786	3770	goto skip_init_addrs;
3787	3771	}
3788	3772	addrs = kvmalloc_objs(*addrs, prog->len + 1);
3789		− if (!addrs) {
3790		− prog = orig_prog;
	3773	+ if (!addrs)
3791	3774	goto out_addrs;
3792		− }
3793	3775
3794	3776	/*
3795	3777	* Before first pass, make a rough estimation of addrs[]
@@ -3811,7 +3793,7 @@ skip_init_addrs: · 15 unmodified lines
3811	3793	for (pass = 0; pass < MAX_PASSES \|\| image; pass++) {
3812	3794	if (!padding && pass >= PADDING_PASSES)
3813	3795	padding = true;
3814		− proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding);
	3796	+ proglen = do_jit(env, prog, addrs, image, rw_image, oldproglen, &ctx, padding);
3815	3797	if (proglen <= 0) {
3816	3798	out_image:
3817	3799	image = NULL;
@@ -3820,8 +3802,6 @@ out_image: · 2 unmodified lines
3820	3802	sizeof(rw_header->size));
3821	3803	bpf_jit_binary_pack_free(header, rw_header);
3822	3804	}
3823		− /* Fall back to interpreter mode */
3824		− prog = orig_prog;
3825	3805	if (extra_pass) {
3826	3806	prog->bpf_func = NULL;
3827	3807	prog->jited = 0;
@@ -3852,10 +3832,8 @@ out_image: · 24 unmodified lines
3852	3832	header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
3853	3833	&image, align, &rw_header, &rw_image,
3854	3834	jit_fill_hole);
3855		− if (!header) {
3856		− prog = orig_prog;
	3835	+ if (!header)
3857	3836	goto out_addrs;
3858		− }
3859	3837	prog->aux->extable = (void *) image + roundup(proglen, align);
3860	3838	}
3861	3839	oldproglen = proglen;
@@ -3908,8 +3886,6 @@ out_image: · 46 unmodified lines
3908	3886	prog->bpf_func = (void *)image + cfi_get_offset();
3909	3887	prog->jited = 1;
3910	3888	prog->jited_len = proglen - cfi_get_offset();
3911		− } else {
3912		− prog = orig_prog;
3913	3889	}
3914	3890
3915	3891	if (!image \|\| !prog->is_func \|\| extra_pass) {
@@ -3925,10 +3901,7 @@ out_priv_stack: · 9 unmodified lines
3925	3901	kfree(jit_data);
3926	3902	prog->aux->jit_data = NULL;
3927	3903	}
3928		−out:
3929		− if (tmp_blinded)
3930		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
3931		− tmp : orig_prog);
	3904	+
3932	3905	return prog;
3933	3906	}
3934	3907

arch/x86/net/bpf_jit_comp32.c+5−30

@@ -2518,38 +2518,22 @@ bool bpf_jit_needs_zext(void) · 2517 unmodified lines
2518	2518	return true;
2519	2519	}
2520	2520
2521		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog)
	2521	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
2522	2522	{
2523	2523	struct bpf_binary_header *header = NULL;
2524		− struct bpf_prog tmp, orig_prog = prog;
2525	2524	int proglen, oldproglen = 0;
2526	2525	struct jit_context ctx = {};
2527		− bool tmp_blinded = false;
2528	2526	u8 *image = NULL;
2529	2527	int *addrs;
2530	2528	int pass;
2531	2529	int i;
2532	2530
2533	2531	if (!prog->jit_requested)
2534		− return orig_prog;
2535		−
2536		− tmp = bpf_jit_blind_constants(prog);
2537		− /*
2538		− * If blinding was requested and we failed during blinding,
2539		− * we must fall back to the interpreter.
2540		− */
2541		− if (IS_ERR(tmp))
2542		− return orig_prog;
2543		− if (tmp != prog) {
2544		− tmp_blinded = true;
2545		− prog = tmp;
2546		− }
	2532	+ return prog;
2547	2533
2548	2534	addrs = kmalloc_objs(*addrs, prog->len);
2549		− if (!addrs) {
2550		− prog = orig_prog;
2551		− goto out;
2552		− }
	2535	+ if (!addrs)
	2536	+ return prog;
2553	2537
2554	2538	/*
2555	2539	* Before first pass, make a rough estimation of addrs[]
@@ -2574,7 +2558,6 @@ out_image: · 18 unmodified lines
2574	2558	image = NULL;
2575	2559	if (header)
2576	2560	bpf_jit_binary_free(header);
2577		− prog = orig_prog;
2578	2561	goto out_addrs;
2579	2562	}
2580	2563	if (image) {
@@ -2588,10 +2571,8 @@ out_image: · 7 unmodified lines
2588	2571	if (proglen == oldproglen) {
2589	2572	header = bpf_jit_binary_alloc(proglen, &image,
2590	2573	1, jit_fill_hole);
2591		− if (!header) {
2592		− prog = orig_prog;
	2574	+ if (!header)
2593	2575	goto out_addrs;
2594		− }
2595	2576	}
2596	2577	oldproglen = proglen;
2597	2578	cond_resched();
@@ -2604,16 +2585,10 @@ out_image: · 6 unmodified lines
2604	2585	prog->bpf_func = (void *)image;
2605	2586	prog->jited = 1;
2606	2587	prog->jited_len = proglen;
2607		− } else {
2608		− prog = orig_prog;
2609	2588	}
2610	2589
2611	2590	out_addrs:
2612	2591	kfree(addrs);
2613		−out:
2614		− if (tmp_blinded)
2615		− bpf_jit_prog_release_other(prog, prog == orig_prog ?
2616		− tmp : orig_prog);
2617	2592	return prog;
2618	2593	}
2619	2594

include/linux/bpf.h+2−0

@@ -1541,6 +1541,8 @@ bool bpf_has_frame_pointer(unsigned long ip); · 1540 unmodified lines
1541	1541	int bpf_jit_charge_modmem(u32 size);
1542	1542	void bpf_jit_uncharge_modmem(u32 size);
1543	1543	bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
	1544	+bool bpf_insn_is_indirect_target(const struct bpf_verifier_env env, const struct bpf_prog prog,
	1545	+ int insn_idx);
1544	1546	#else
1545	1547	static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1546	1548	struct bpf_trampoline *tr,

include/linux/bpf_verifier.h+5−4

@@ -630,16 +630,17 @@ struct bpf_insn_aux_data { · 629 unmodified lines
630	630
631	631	/* below fields are initialized once */
632	632	unsigned int orig_idx; /* original instruction index */
633		− bool jmp_point;
634		− bool prune_point;
	633	+ u32 jmp_point:1;
	634	+ u32 prune_point:1;
635	635	/* ensure we check state equivalence and save state checkpoint and
636	636	* this instruction, regardless of any heuristics
637	637	*/
638		− bool force_checkpoint;
	638	+ u32 force_checkpoint:1;
639	639	/* true if instruction is a call to a helper function that
640	640	* accepts callback function as a parameter.
641	641	*/
642		− bool calls_callback;
	642	+ u32 calls_callback:1;
	643	+ u32 indirect_target:1; /* if it is an indirect jump target */
643	644	/*
644	645	* CFG strongly connected component this instruction belongs to,
645	646	* zero if it is a singleton SCC.

include/linux/filter.h+48−2

@@ -1108,6 +1108,8 @@ sk_filter_reason(struct sock sk, struct sk_buff skb) · 1107 unmodified lines
1108	1108	return sk_filter_trim_cap(sk, skb, 1);
1109	1109	}
1110	1110
	1111	+struct bpf_prog __bpf_prog_select_runtime(struct bpf_verifier_env env, struct bpf_prog *fp,
	1112	+ int *err);
1111	1113	struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err);
1112	1114	void bpf_prog_free(struct bpf_prog *fp);
1113	1115
@@ -1153,7 +1155,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); · 39 unmodified lines
1153	1155	((u64 ()(u64, u64, u64, u64, u64, const struct bpf_insn )) \
1154	1156	(void *)__bpf_call_base)
1155	1157
1156		−struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog);
	1158	+struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog);
1157	1159	void bpf_jit_compile(struct bpf_prog *prog);
1158	1160	bool bpf_jit_needs_zext(void);
1159	1161	bool bpf_jit_inlines_helper_call(s32 imm);
@@ -1184,6 +1186,31 @@ static inline bool bpf_dump_raw_ok(const struct cred *cred) · 24 unmodified lines
1184	1186
1185	1187	struct bpf_prog bpf_patch_insn_single(struct bpf_prog prog, u32 off,
1186	1188	const struct bpf_insn *patch, u32 len);
	1189	+
	1190	+#ifdef CONFIG_BPF_SYSCALL
	1191	+struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,
	1192	+ const struct bpf_insn *patch, u32 len);
	1193	+struct bpf_insn_aux_data bpf_dup_insn_aux_data(struct bpf_verifier_env env);
	1194	+void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
	1195	+ struct bpf_insn_aux_data *orig_insn_aux);
	1196	+#else
	1197	+static inline struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,
	1198	+ const struct bpf_insn *patch, u32 len)
	1199	+{
	1200	+ return ERR_PTR(-ENOTSUPP);
	1201	+}
	1202	+
	1203	+static inline struct bpf_insn_aux_data bpf_dup_insn_aux_data(struct bpf_verifier_env env)
	1204	+{
	1205	+ return NULL;
	1206	+}
	1207	+
	1208	+static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
	1209	+ struct bpf_insn_aux_data *orig_insn_aux)
	1210	+{
	1211	+}
	1212	+#endif /* CONFIG_BPF_SYSCALL */
	1213	+
1187	1214	int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
1188	1215
1189	1216	static inline bool xdp_return_frame_no_direct(void)
@@ -1310,9 +1337,14 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog, · 120 unmodified lines
1310	1337
1311	1338	const char bpf_jit_get_prog_name(struct bpf_prog prog);
1312	1339
1313		−struct bpf_prog bpf_jit_blind_constants(struct bpf_prog fp);
	1340	+struct bpf_prog bpf_jit_blind_constants(struct bpf_verifier_env env, struct bpf_prog *prog);
1314	1341	void bpf_jit_prog_release_other(struct bpf_prog fp, struct bpf_prog fp_other);
1315	1342
	1343	+static inline bool bpf_prog_need_blind(const struct bpf_prog *prog)
	1344	+{
	1345	+ return prog->blinding_requested && !prog->blinded;
	1346	+}
	1347	+
1316	1348	static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
1317	1349	u32 pass, void *image)
1318	1350	{
@@ -1451,6 +1483,20 @@ static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp) · 132 unmodified lines
1451	1483	{
1452	1484	}
1453	1485
	1486	+static inline bool bpf_prog_need_blind(const struct bpf_prog *prog)
	1487	+{
	1488	+ return false;
	1489	+}
	1490	+
	1491	+static inline
	1492	+struct bpf_prog bpf_jit_blind_constants(struct bpf_verifier_env env, struct bpf_prog *prog)
	1493	+{
	1494	+ return prog;
	1495	+}
	1496	+
	1497	+static inline void bpf_jit_prog_release_other(struct bpf_prog fp, struct bpf_prog fp_other)
	1498	+{
	1499	+}
1454	1500	#endif /* CONFIG_BPF_JIT */
1455	1501
1456	1502	void bpf_prog_kallsyms_del_all(struct bpf_prog *fp);

kernel/bpf/arena.c+20−3

@@ -341,6 +341,16 @@ static void arena_vm_open(struct vm_area_struct *vma) · 340 unmodified lines
341	341	refcount_inc(&vml->mmap_count);
342	342	}
343	343
	344	+static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr)
	345	+{
	346	+ return -EINVAL;
	347	+}
	348	+
	349	+static int arena_vm_mremap(struct vm_area_struct *vma)
	350	+{
	351	+ return -EINVAL;
	352	+}
	353	+
344	354	static void arena_vm_close(struct vm_area_struct *vma)
345	355	{
346	356	struct bpf_map *map = vma->vm_file->private_data;
@@ -417,6 +427,8 @@ out_unlock_sigsegv: · 70 unmodified lines
417	427
418	428	static const struct vm_operations_struct arena_vm_ops = {
419	429	.open = arena_vm_open,
	430	+ .may_split = arena_vm_may_split,
	431	+ .mremap = arena_vm_mremap,
420	432	.close = arena_vm_close,
421	433	.fault = arena_vm_fault,
422	434	};
@@ -486,10 +498,11 @@ static int arena_map_mmap(struct bpf_map map, struct vm_area_struct vma) · 63 unmodified lines
486	498	arena->user_vm_end = vma->vm_end;
487	499	/*
488	500	* bpf_map_mmap() checks that it's being mmaped as VM_SHARED and
489		− * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid
490		− * potential change of user_vm_start.
	501	+ * clears VM_MAYEXEC. Set VM_DONTEXPAND to avoid potential change
	502	+ * of user_vm_start. Set VM_DONTCOPY to prevent arena VMA from
	503	+ * being copied into the child process on fork.
491	504	*/
492		− vm_flags_set(vma, VM_DONTEXPAND);
	505	+ vm_flags_set(vma, VM_DONTEXPAND \| VM_DONTCOPY);
493	506	vma->vm_ops = &arena_vm_ops;
494	507	return 0;
495	508	}
@@ -549,6 +562,10 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt · 53 unmodified lines
549	562	u32 uaddr32;
550	563	int ret, i;
551	564
	565	+ if (node_id != NUMA_NO_NODE &&
	566	+ ((unsigned int)node_id >= nr_node_ids \|\| !node_online(node_id)))
	567	+ return 0;
	568	+
552	569	if (page_cnt > page_cnt_max)
553	570	return 0;
554	571

kernel/bpf/bpf_struct_ops.c+4−3

@@ -811,9 +811,6 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map map, void key, · 810 unmodified lines
811	811	goto reset_unlock;
812	812	}
813	813
814		− /* Poison pointer on error instead of return for backward compatibility */
815		− bpf_prog_assoc_struct_ops(prog, &st_map->map);
816		−
817	814	link = kzalloc_obj(*link, GFP_USER);
818	815	if (!link) {
819	816	bpf_prog_put(prog);
@@ -824,6 +821,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map map, void key, · 4 unmodified lines
824	821	&bpf_struct_ops_link_lops, prog, prog->expected_attach_type);
825	822	*plink++ = &link->link;
826	823
	824	+ /* Poison pointer on error instead of return for backward compatibility */
	825	+ bpf_prog_assoc_struct_ops(prog, &st_map->map);
	826	+
827	827	ksym = kzalloc_obj(*ksym, GFP_USER);
828	828	if (!ksym) {
829	829	err = -ENOMEM;
@@ -906,6 +906,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map map, void key, · 76 unmodified lines
906	906	reset_unlock:
907	907	bpf_struct_ops_map_free_ksyms(st_map);
908	908	bpf_struct_ops_map_free_image(st_map);
	909	+ bpf_struct_ops_map_dissoc_progs(st_map);
909	910	bpf_struct_ops_map_put_progs(st_map);
910	911	memset(uvalue, 0, map->value_size);
911	912	memset(kvalue, 0, map->value_size);

kernel/bpf/core.c+99−39

@@ -1491,24 +1491,11 @@ void bpf_jit_prog_release_other(struct bpf_prog fp, struct bpf_prog fp_other) · 1490 unmodified lines
1491	1491	bpf_prog_clone_free(fp_other);
1492	1492	}
1493	1493
1494		−static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
1495		−{
1496		−#ifdef CONFIG_BPF_SYSCALL
1497		− struct bpf_map *map;
1498		− int i;
1499		−
1500		− if (len <= 1)
1501		− return;
1502		−
1503		− for (i = 0; i < prog->aux->used_map_cnt; i++) {
1504		− map = prog->aux->used_maps[i];
1505		− if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
1506		− bpf_insn_array_adjust(map, off, len);
1507		− }
1508		−#endif
1509		−}
1510		−
1511		−struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog)
	1494	+/*
	1495	+ * Now this function is used only to blind the main prog and must be invoked only when
	1496	+ * bpf_prog_need_blind() returns true.
	1497	+ */
	1498	+struct bpf_prog bpf_jit_blind_constants(struct bpf_verifier_env env, struct bpf_prog *prog)
1512	1499	{
1513	1500	struct bpf_insn insn_buff[16], aux[2];
1514	1501	struct bpf_prog clone, tmp;
@@ -1516,13 +1503,17 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 1 unmodified line
1516	1503	struct bpf_insn *insn;
1517	1504	int i, rewritten;
1518	1505
1519		− if (!prog->blinding_requested \|\| prog->blinded)
1520		− return prog;
	1506	+ if (WARN_ON_ONCE(env && env->prog != prog))
	1507	+ return ERR_PTR(-EINVAL);
1521	1508
1522	1509	clone = bpf_prog_clone_create(prog, GFP_USER);
1523	1510	if (!clone)
1524	1511	return ERR_PTR(-ENOMEM);
1525	1512
	1513	+ /* make sure bpf_patch_insn_data() patches the correct prog */
	1514	+ if (env)
	1515	+ env->prog = clone;
	1516	+
1526	1517	insn_cnt = clone->len;
1527	1518	insn = clone->insnsi;
1528	1519
@@ -1550,21 +1541,28 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 21 unmodified lines
1550	1541	if (!rewritten)
1551	1542	continue;
1552	1543
1553		− tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
1554		− if (IS_ERR(tmp)) {
	1544	+ if (env)
	1545	+ tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
	1546	+ else
	1547	+ tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
	1548	+
	1549	+ if (IS_ERR_OR_NULL(tmp)) {
	1550	+ if (env)
	1551	+ /* restore the original prog */
	1552	+ env->prog = prog;
1555	1553	/* Patching may have repointed aux->prog during
1556	1554	* realloc from the original one, so we need to
1557	1555	* fix it up here on error.
1558	1556	*/
1559	1557	bpf_jit_prog_release_other(prog, clone);
1560		− return tmp;
	1558	+ return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM);
1561	1559	}
1562	1560
1563	1561	clone = tmp;
1564	1562	insn_delta = rewritten - 1;
1565	1563
1566		− /* Instructions arrays must be updated using absolute xlated offsets */
1567		− adjust_insn_arrays(clone, prog->aux->subprog_start + i, rewritten);
	1564	+ if (env)
	1565	+ env->prog = clone;
1568	1566
1569	1567	/* Walk new program and skip insns we just inserted. */
1570	1568	insn = clone->insnsi + i + insn_delta;
@@ -1575,6 +1573,15 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 4 unmodified lines
1575	1573	clone->blinded = 1;
1576	1574	return clone;
1577	1575	}
	1576	+
	1577	+bool bpf_insn_is_indirect_target(const struct bpf_verifier_env env, const struct bpf_prog prog,
	1578	+ int insn_idx)
	1579	+{
	1580	+ if (!env)
	1581	+ return false;
	1582	+ insn_idx += prog->aux->subprog_start;
	1583	+ return env->insn_aux_data[insn_idx].indirect_target;
	1584	+}
1578	1585	#endif /* CONFIG_BPF_JIT */
1579	1586
1580	1587	/* Base function for offset calculation. Needs to go into .text section,
@@ -2533,18 +2540,55 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) · 952 unmodified lines
2533	2540	return select_interpreter;
2534	2541	}
2535	2542
2536		−/**
2537		− * bpf_prog_select_runtime - select exec runtime for BPF program
2538		− * @fp: bpf_prog populated with BPF program
2539		− * @err: pointer to error variable
2540		− *
2541		− * Try to JIT eBPF program, if JIT is not available, use interpreter.
2542		− * The BPF program will be executed via bpf_prog_run() function.
2543		− *
2544		− * Return: the &fp argument along with &err set to 0 for success or
2545		− * a negative errno code on failure
2546		− */
2547		−struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err)
	2543	+static struct bpf_prog bpf_prog_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
	2544	+{
	2545	+#ifdef CONFIG_BPF_JIT
	2546	+ struct bpf_prog *orig_prog;
	2547	+ struct bpf_insn_aux_data *orig_insn_aux;
	2548	+
	2549	+ if (!bpf_prog_need_blind(prog))
	2550	+ return bpf_int_jit_compile(env, prog);
	2551	+
	2552	+ if (env) {
	2553	+ /*
	2554	+ * If env is not NULL, we are called from the end of bpf_check(), at this
	2555	+ * point, only insn_aux_data is used after failure, so it should be restored
	2556	+ * on failure.
	2557	+ */
	2558	+ orig_insn_aux = bpf_dup_insn_aux_data(env);
	2559	+ if (!orig_insn_aux)
	2560	+ return prog;
	2561	+ }
	2562	+
	2563	+ orig_prog = prog;
	2564	+ prog = bpf_jit_blind_constants(env, prog);
	2565	+ /*
	2566	+ * If blinding was requested and we failed during blinding, we must fall
	2567	+ * back to the interpreter.
	2568	+ */
	2569	+ if (IS_ERR(prog))
	2570	+ goto out_restore;
	2571	+
	2572	+ prog = bpf_int_jit_compile(env, prog);
	2573	+ if (prog->jited) {
	2574	+ bpf_jit_prog_release_other(prog, orig_prog);
	2575	+ if (env)
	2576	+ vfree(orig_insn_aux);
	2577	+ return prog;
	2578	+ }
	2579	+
	2580	+ bpf_jit_prog_release_other(orig_prog, prog);
	2581	+
	2582	+out_restore:
	2583	+ prog = orig_prog;
	2584	+ if (env)
	2585	+ bpf_restore_insn_aux_data(env, orig_insn_aux);
	2586	+#endif
	2587	+ return prog;
	2588	+}
	2589	+
	2590	+struct bpf_prog __bpf_prog_select_runtime(struct bpf_verifier_env env, struct bpf_prog *fp,
	2591	+ int *err)
2548	2592	{
2549	2593	/* In case of BPF to BPF calls, verifier did all the prep
2550	2594	* work with regards to JITing, etc.
@@ -2572,7 +2616,7 @@ struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err) · 21 unmodified lines
2572	2616	if (*err)
2573	2617	return fp;
2574	2618
2575		− fp = bpf_int_jit_compile(fp);
	2619	+ fp = bpf_prog_jit_compile(env, fp);
2576	2620	bpf_prog_jit_attempt_done(fp);
2577	2621	if (!fp->jited && jit_needed) {
2578	2622	*err = -ENOTSUPP;
@@ -2598,6 +2642,22 @@ finalize: · 19 unmodified lines
2598	2642
2599	2643	return fp;
2600	2644	}
	2645	+
	2646	+/**
	2647	+ * bpf_prog_select_runtime - select exec runtime for BPF program
	2648	+ * @fp: bpf_prog populated with BPF program
	2649	+ * @err: pointer to error variable
	2650	+ *
	2651	+ * Try to JIT eBPF program, if JIT is not available, use interpreter.
	2652	+ * The BPF program will be executed via bpf_prog_run() function.
	2653	+ *
	2654	+ * Return: the &fp argument along with &err set to 0 for success or
	2655	+ * a negative errno code on failure
	2656	+ */
	2657	+struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err)
	2658	+{
	2659	+ return __bpf_prog_select_runtime(NULL, fp, err);
	2660	+}
2601	2661	EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
2602	2662
2603	2663	static unsigned int __bpf_prog_ret1(const void *ctx,
@@ -3085,7 +3145,7 @@ const struct bpf_func_proto bpf_tail_call_proto = { · 481 unmodified lines
3085	3145	* It is encouraged to implement bpf_int_jit_compile() instead, so that
3086	3146	* eBPF and implicitly also cBPF can get JITed!
3087	3147	*/
3088		−struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
	3148	+struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog prog)
3089	3149	{
3090	3150	return prog;
3091	3151	}

kernel/bpf/fixups.c+138−25

@@ -183,6 +183,18 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, · 182 unmodified lines
183	183	data[i].seen = old_seen;
184	184	data[i].zext_dst = insn_has_def32(insn + i);
185	185	}
	186	+
	187	+ /*
	188	+ * The indirect_target flag of the original instruction was moved to the last of the
	189	+ * new instructions by the above memmove and memset, but the indirect jump target is
	190	+ * actually the first instruction, so move it back. This also matches with the behavior
	191	+ * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new
	192	+ * instruction.
	193	+ */
	194	+ if (data[off + cnt - 1].indirect_target) {
	195	+ data[off].indirect_target = 1;
	196	+ data[off + cnt - 1].indirect_target = 0;
	197	+ }
186	198	}
187	199
188	200	static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
@@ -232,8 +244,8 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) · 43 unmodified lines
232	244	}
233	245	}
234	246
235		−static struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,
236		− const struct bpf_insn *patch, u32 len)
	247	+struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,
	248	+ const struct bpf_insn *patch, u32 len)
237	249	{
238	250	struct bpf_prog *new_prog;
239	251	struct bpf_insn_aux_data *new_data = NULL;
@@ -973,7 +985,47 @@ patch_insn_buf: · 733 unmodified lines
973	985	return 0;
974	986	}
975	987
976		−int bpf_jit_subprogs(struct bpf_verifier_env *env)
	988	+static u32 bpf_dup_subprog_starts(struct bpf_verifier_env env)
	989	+{
	990	+ u32 *starts = NULL;
	991	+
	992	+ starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT);
	993	+ if (starts) {
	994	+ for (int i = 0; i < env->subprog_cnt; i++)
	995	+ starts[i] = env->subprog_info[i].start;
	996	+ }
	997	+ return starts;
	998	+}
	999	+
	1000	+static void bpf_restore_subprog_starts(struct bpf_verifier_env env, u32 orig_starts)
	1001	+{
	1002	+ for (int i = 0; i < env->subprog_cnt; i++)
	1003	+ env->subprog_info[i].start = orig_starts[i];
	1004	+ /* restore the start of fake 'exit' subprog as well */
	1005	+ env->subprog_info[env->subprog_cnt].start = env->prog->len;
	1006	+}
	1007	+
	1008	+struct bpf_insn_aux_data bpf_dup_insn_aux_data(struct bpf_verifier_env env)
	1009	+{
	1010	+ size_t size;
	1011	+ void *new_aux;
	1012	+
	1013	+ size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);
	1014	+ new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT);
	1015	+ if (new_aux)
	1016	+ memcpy(new_aux, env->insn_aux_data, size);
	1017	+ return new_aux;
	1018	+}
	1019	+
	1020	+void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
	1021	+ struct bpf_insn_aux_data *orig_insn_aux)
	1022	+{
	1023	+ /* the expanded elements are zero-filled, so no special handling is required */
	1024	+ vfree(env->insn_aux_data);
	1025	+ env->insn_aux_data = orig_insn_aux;
	1026	+}
	1027	+
	1028	+static int jit_subprogs(struct bpf_verifier_env *env)
977	1029	{
978	1030	struct bpf_prog prog = env->prog, func, tmp;
979	1031	int i, j, subprog_start, subprog_end = 0, len, subprog;
@@ -981,10 +1033,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 1 unmodified line
981	1033	struct bpf_insn *insn;
982	1034	void *old_bpf_func;
983	1035	int err, num_exentries;
984		− int old_len, subprog_start_adjustment = 0;
985		−
986		− if (env->subprog_cnt <= 1)
987		− return 0;
988	1036
989	1037	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
990	1038	if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
@@ -1053,10 +1101,11 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 62 unmodified lines
1053	1101	goto out_free;
1054	1102	func[i]->is_func = 1;
1055	1103	func[i]->sleepable = prog->sleepable;
	1104	+ func[i]->blinded = prog->blinded;
1056	1105	func[i]->aux->func_idx = i;
1057	1106	/* Below members will be freed only at prog->aux */
1058	1107	func[i]->aux->btf = prog->aux->btf;
1059		− func[i]->aux->subprog_start = subprog_start + subprog_start_adjustment;
	1108	+ func[i]->aux->subprog_start = subprog_start;
1060	1109	func[i]->aux->func_info = prog->aux->func_info;
1061	1110	func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
1062	1111	func[i]->aux->poke_tab = prog->aux->poke_tab;
@@ -1110,17 +1159,10 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 47 unmodified lines
1110	1159	func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
1111	1160	func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
1112	1161	func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
	1162	+ func[i]->aux->token = prog->aux->token;
1113	1163	if (!i)
1114	1164	func[i]->aux->exception_boundary = env->seen_exception;
1115		−
1116		− /*
1117		− * To properly pass the absolute subprog start to jit
1118		− * all instruction adjustments should be accumulated
1119		− */
1120		− old_len = func[i]->len;
1121		− func[i] = bpf_int_jit_compile(func[i]);
1122		− subprog_start_adjustment += func[i]->len - old_len;
1123		−
	1165	+ func[i] = bpf_int_jit_compile(env, func[i]);
1124	1166	if (!func[i]->jited) {
1125	1167	err = -ENOTSUPP;
1126	1168	goto out_free;
@@ -1164,7 +1206,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 37 unmodified lines
1164	1206	}
1165	1207	for (i = 0; i < env->subprog_cnt; i++) {
1166	1208	old_bpf_func = func[i]->bpf_func;
1167		− tmp = bpf_int_jit_compile(func[i]);
	1209	+ tmp = bpf_int_jit_compile(env, func[i]);
1168	1210	if (tmp != func[i] \|\| func[i]->bpf_func != old_bpf_func) {
1169	1211	verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
1170	1212	err = -ENOTSUPP;
@@ -1246,16 +1288,87 @@ out_free: · 75 unmodified lines
1246	1288	}
1247	1289	kfree(func);
1248	1290	out_undo_insn:
	1291	+ bpf_prog_jit_attempt_done(prog);
	1292	+ return err;
	1293	+}
	1294	+
	1295	+int bpf_jit_subprogs(struct bpf_verifier_env *env)
	1296	+{
	1297	+ int err, i;
	1298	+ bool blinded = false;
	1299	+ struct bpf_insn *insn;
	1300	+ struct bpf_prog prog, orig_prog;
	1301	+ struct bpf_insn_aux_data *orig_insn_aux;
	1302	+ u32 *orig_subprog_starts;
	1303	+
	1304	+ if (env->subprog_cnt <= 1)
	1305	+ return 0;
	1306	+
	1307	+ prog = orig_prog = env->prog;
	1308	+ if (bpf_prog_need_blind(prog)) {
	1309	+ orig_insn_aux = bpf_dup_insn_aux_data(env);
	1310	+ if (!orig_insn_aux) {
	1311	+ err = -ENOMEM;
	1312	+ goto out_cleanup;
	1313	+ }
	1314	+ orig_subprog_starts = bpf_dup_subprog_starts(env);
	1315	+ if (!orig_subprog_starts) {
	1316	+ vfree(orig_insn_aux);
	1317	+ err = -ENOMEM;
	1318	+ goto out_cleanup;
	1319	+ }
	1320	+ prog = bpf_jit_blind_constants(env, prog);
	1321	+ if (IS_ERR(prog)) {
	1322	+ err = -ENOMEM;
	1323	+ prog = orig_prog;
	1324	+ goto out_restore;
	1325	+ }
	1326	+ blinded = true;
	1327	+ }
	1328	+
	1329	+ err = jit_subprogs(env);
	1330	+ if (err)
	1331	+ goto out_jit_err;
	1332	+
	1333	+ if (blinded) {
	1334	+ bpf_jit_prog_release_other(prog, orig_prog);
	1335	+ kvfree(orig_subprog_starts);
	1336	+ vfree(orig_insn_aux);
	1337	+ }
	1338	+
	1339	+ return 0;
	1340	+
	1341	+out_jit_err:
	1342	+ if (blinded) {
	1343	+ bpf_jit_prog_release_other(orig_prog, prog);
	1344	+ /* roll back to the clean original prog */
	1345	+ prog = env->prog = orig_prog;
	1346	+ goto out_restore;
	1347	+ } else {
	1348	+ if (err != -EFAULT) {
	1349	+ /*
	1350	+ * We will fall back to interpreter mode when err is not -EFAULT, before
	1351	+ * that, insn->off and insn->imm should be restored to their original
	1352	+ * values since they were modified by jit_subprogs.
	1353	+ */
	1354	+ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
	1355	+ if (!bpf_pseudo_call(insn))
	1356	+ continue;
	1357	+ insn->off = 0;
	1358	+ insn->imm = env->insn_aux_data[i].call_imm;
	1359	+ }
	1360	+ }
	1361	+ goto out_cleanup;
	1362	+ }
	1363	+
	1364	+out_restore:
	1365	+ bpf_restore_subprog_starts(env, orig_subprog_starts);
	1366	+ bpf_restore_insn_aux_data(env, orig_insn_aux);
	1367	+ kvfree(orig_subprog_starts);
	1368	+out_cleanup:
1249	1369	/* cleanup main prog to be interpreted */
1250	1370	prog->jit_requested = 0;
1251	1371	prog->blinding_requested = 0;
1252		− for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
1253		− if (!bpf_pseudo_call(insn))
1254		− continue;
1255		− insn->off = 0;
1256		− insn->imm = env->insn_aux_data[i].call_imm;
1257		− }
1258		− bpf_prog_jit_attempt_done(prog);
1259	1372	return err;
1260	1373	}
1261	1374

kernel/bpf/helpers.c+16−1

@@ -845,7 +845,13 @@ int bpf_bprintf_prepare(const char fmt, u32 fmt_size, const u64 raw_args, · 844 unmodified lines
845	845	data->buf = buffers->buf;
846	846
847	847	for (i = 0; i < fmt_size; i++) {
848		− if ((!isprint(fmt[i]) && !isspace(fmt[i])) \|\| !isascii(fmt[i])) {
	848	+ unsigned char c = fmt[i];
	849	+
	850	+ /*
	851	+ * Permit bytes >= 0x80 in plain text so UTF-8 literals can pass
	852	+ * through unchanged, while still rejecting ASCII control bytes.
	853	+ */
	854	+ if (isascii(c) && !isprint(c) && !isspace(c)) {
849	855	err = -EINVAL;
850	856	goto out;
851	857	}
@@ -867,6 +873,15 @@ int bpf_bprintf_prepare(const char fmt, u32 fmt_size, const u64 raw_args, · 15 unmodified lines
867	873	* always access fmt[i + 1], in the worst case it will be a 0
868	874	*/
869	875	i++;
	876	+ c = fmt[i];
	877	+ /*
	878	+ * The format parser below only understands ASCII conversion
	879	+ * specifiers and modifiers, so reject non-ASCII after '%'.
	880	+ */
	881	+ if (!isascii(c)) {
	882	+ err = -EINVAL;
	883	+ goto out;
	884	+ }
870	885
871	886	/* skip optional "[0 +-][num]" width formatting field */
872	887	while (fmt[i] == '0' \|\| fmt[i] == '+' \|\| fmt[i] == '-' \|\|

kernel/bpf/liveness.c+62−52

@@ -574,7 +574,7 @@ static int print_instances(struct bpf_verifier_env *env) · 573 unmodified lines
574	574	*
575	575	* precise {frame=N, off=V} -- known absolute frame index and byte offset
576	576	* \|
577		− * offset-imprecise {frame=N, off=OFF_IMPRECISE}
	577	+ * offset-imprecise {frame=N, cnt=0}
578	578	* \| -- known frame identity, unknown offset
579	579	* fully-imprecise {frame=ARG_IMPRECISE, mask=bitmask}
580	580	* -- unknown frame identity; .mask is a
@@ -607,8 +607,6 @@ enum arg_track_state { · 26 unmodified lines
607	607	ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */
608	608	};
609	609
610		−#define OFF_IMPRECISE S16_MIN /* arg identity known but offset unknown */
611		−
612	610	/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */
613	611	#define MAX_ARG_SPILL_SLOTS 64
614	612
@@ -622,28 +620,6 @@ static bool arg_is_fp(const struct arg_track *at) · 7 unmodified lines
622	620	return at->frame >= 0 \|\| at->frame == ARG_IMPRECISE;
623	621	}
624	622
625		−/*
626		− * Clear all tracked callee stack slots overlapping the byte range
627		− * [off, off+sz-1] where off is a negative FP-relative offset.
628		− */
629		−static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz)
630		−{
631		− struct arg_track none = { .frame = ARG_NONE };
632		−
633		− if (off == OFF_IMPRECISE) {
634		− for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
635		− at_stack[i] = none;
636		− return;
637		− }
638		− for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
639		− int slot_start = -((i + 1) * 8);
640		− int slot_end = slot_start + 8;
641		−
642		− if (slot_start < off + (int)sz && slot_end > off)
643		− at_stack[i] = none;
644		− }
645		−}
646		−
647	623	static void verbose_arg_track(struct bpf_verifier_env env, struct arg_track at)
648	624	{
649	625	int i;
@@ -863,16 +839,13 @@ static void arg_track_alu64(struct arg_track dst, const struct arg_track src) · 213 unmodified lines
863	839	dst = arg_join_imprecise(dst, *src);
864	840	}
865	841
866		−static s16 arg_add(s16 off, s64 delta)
	842	+static bool arg_add(s16 off, s64 delta, s16 *out)
867	843	{
868		− s64 res;
869		−
870		− if (off == OFF_IMPRECISE)
871		− return OFF_IMPRECISE;
872		− res = (s64)off + delta;
873		− if (res < S16_MIN + 1 \|\| res > S16_MAX)
874		− return OFF_IMPRECISE;
875		− return res;
	844	+ s16 d = delta;
	845	+
	846	+ if (d != delta)
	847	+ return true;
	848	+ return check_add_overflow(off, d, out);
876	849	}
877	850
878	851	static void arg_padd(struct arg_track *at, s64 delta)
@@ -882,9 +855,9 @@ static void arg_padd(struct arg_track *at, s64 delta) · 3 unmodified lines
882	855	if (at->off_cnt == 0)
883	856	return;
884	857	for (i = 0; i < at->off_cnt; i++) {
885		− s16 new_off = arg_add(at->off[i], delta);
	858	+ s16 new_off;
886	859
887		− if (new_off == OFF_IMPRECISE) {
	860	+ if (arg_add(at->off[i], delta, &new_off)) {
888	861	at->off_cnt = 0;
889	862	return;
890	863	}
@@ -899,8 +872,6 @@ static void arg_padd(struct arg_track *at, s64 delta) · 8 unmodified lines
899	872	*/
900	873	static int fp_off_to_slot(s16 off)
901	874	{
902		− if (off == OFF_IMPRECISE)
903		− return -1;
904	875	if (off >= 0 \|\| off < -(int)(MAX_ARG_SPILL_SLOTS * 8))
905	876	return -1;
906	877	if (off % 8)
@@ -930,9 +901,11 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, · 23 unmodified lines
930	901	return imp;
931	902
932	903	for (i = 0; i < cnt; i++) {
933		− s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
934		− int slot = fp_off_to_slot(fp_off);
	904	+ s16 fp_off, slot;
935	905
	906	+ if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
	907	+ return imp;
	908	+ slot = fp_off_to_slot(fp_off);
936	909	if (slot < 0)
937	910	return imp;
938	911	result = __arg_track_join(result, at_stack_out[slot]);
@@ -968,9 +941,12 @@ static void spill_to_stack(struct bpf_insn insn, struct arg_track at_out, · 29 unmodified lines
968	941	return;
969	942	}
970	943	for (i = 0; i < cnt; i++) {
971		− s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
972		− int slot = fp_off_to_slot(fp_off);
	944	+ s16 fp_off;
	945	+ int slot;
973	946
	947	+ if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
	948	+ continue;
	949	+ slot = fp_off_to_slot(fp_off);
974	950	if (slot < 0)
975	951	continue;
976	952	if (cnt == 1)
@@ -980,6 +956,32 @@ static void spill_to_stack(struct bpf_insn insn, struct arg_track at_out, · 3 unmodified lines
980	956	}
981	957	}
982	958
	959	+/*
	960	+ * Clear all tracked callee stack slots overlapping the byte range
	961	+ * [off, off+sz-1] where off is a negative FP-relative offset.
	962	+ */
	963	+static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt)
	964	+{
	965	+ struct arg_track none = { .frame = ARG_NONE };
	966	+
	967	+ if (cnt == 0) {
	968	+ for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
	969	+ at_stack[i] = __arg_track_join(at_stack[i], none);
	970	+ return;
	971	+ }
	972	+ for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
	973	+ int slot_start = -((i + 1) * 8);
	974	+ int slot_end = slot_start + 8;
	975	+
	976	+ if (slot_start < off + (int)sz && slot_end > off) {
	977	+ if (cnt == 1)
	978	+ at_stack[i] = none;
	979	+ else
	980	+ at_stack[i] = __arg_track_join(at_stack[i], none);
	981	+ }
	982	+ }
	983	+}
	984	+
983	985	/*
984	986	* Clear stack slots overlapping all possible FP offsets in @reg.
985	987	*/
@@ -990,18 +992,22 @@ static void clear_stack_for_all_offs(struct bpf_insn *insn, · 4 unmodified lines
990	992	int cnt, i;
991	993
992	994	if (reg == BPF_REG_FP) {
993		− clear_overlapping_stack_slots(at_stack_out, insn->off, sz);
	995	+ clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1);
994	996	return;
995	997	}
996	998	cnt = at_out[reg].off_cnt;
997	999	if (cnt == 0) {
998		− clear_overlapping_stack_slots(at_stack_out, OFF_IMPRECISE, sz);
	1000	+ clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt);
999	1001	return;
1000	1002	}
1001	1003	for (i = 0; i < cnt; i++) {
1002		− s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
	1004	+ s16 fp_off;
1003	1005
1004		− clear_overlapping_stack_slots(at_stack_out, fp_off, sz);
	1006	+ if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) {
	1007	+ clear_overlapping_stack_slots(at_stack_out, 0, sz, 0);
	1008	+ break;
	1009	+ }
	1010	+ clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt);
1005	1011	}
1006	1012	}
1007	1013
@@ -1042,6 +1048,12 @@ static void arg_track_log(struct bpf_verifier_env env, struct bpf_insn insn, i · 34 unmodified lines
1042	1048	verbose(env, "\n");
1043	1049	}
1044	1050
	1051	+static bool can_be_local_fp(int depth, int regno, struct arg_track *at)
	1052	+{
	1053	+ return regno == BPF_REG_FP \|\| at->frame == depth \|\|
	1054	+ (at->frame == ARG_IMPRECISE && (at->mask & BIT(depth)));
	1055	+}
	1056	+
1045	1057	/*
1046	1058	* Pure dataflow transfer function for arg_track state.
1047	1059	* Updates at_out[] based on how the instruction modifies registers.
@@ -1111,8 +1123,7 @@ static void arg_track_xfer(struct bpf_verifier_env env, struct bpf_insn insn, · 63 unmodified lines
1111	1123	at_out[r] = none;
1112	1124	} else if (class == BPF_LDX) {
1113	1125	u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1114		− bool src_is_local_fp = insn->src_reg == BPF_REG_FP \|\| src->frame == depth \|\|
1115		− (src->frame == ARG_IMPRECISE && (src->mask & BIT(depth)));
	1126	+ bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src);
1116	1127
1117	1128	/*
1118	1129	* Reload from callee stack: if src is current-frame FP-derived
@@ -1147,7 +1158,7 @@ static void arg_track_xfer(struct bpf_verifier_env env, struct bpf_insn insn, · 28 unmodified lines
1147	1158	bool dst_is_local_fp;
1148	1159
1149	1160	/* Track spills to current-frame FP-derived callee stack */
1150		− dst_is_local_fp = insn->dst_reg == BPF_REG_FP \|\| dst->frame == depth;
	1161	+ dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
1151	1162	if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM)
1152	1163	spill_to_stack(insn, at_out, insn->dst_reg,
1153	1164	at_stack_out, src, sz);
@@ -1166,7 +1177,7 @@ static void arg_track_xfer(struct bpf_verifier_env env, struct bpf_insn insn, · 12 unmodified lines
1166	1177	}
1167	1178	} else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) {
1168	1179	u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1169		− bool dst_is_local_fp = insn->dst_reg == BPF_REG_FP \|\| dst->frame == depth;
	1180	+ bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
1170	1181
1171	1182	/* BPF_ST to FP-derived dst: clear overlapping stack slots */
1172	1183	if (dst_is_local_fp)
@@ -1316,8 +1327,7 @@ static int record_load_store_access(struct bpf_verifier_env *env, · 143 unmodified lines
1316	1327	resolved.off_cnt = ptr->off_cnt;
1317	1328	resolved.frame = ptr->frame;
1318	1329	for (oi = 0; oi < ptr->off_cnt; oi++) {
1319		− resolved.off[oi] = arg_add(ptr->off[oi], insn->off);
1320		− if (resolved.off[oi] == OFF_IMPRECISE) {
	1330	+ if (arg_add(ptr->off[oi], insn->off, &resolved.off[oi])) {
1321	1331	resolved.off_cnt = 0;
1322	1332	break;
1323	1333	}

kernel/bpf/syscall.c+0−4

@@ -3083,10 +3083,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) · 3082 unmodified lines
3083	3083	if (err < 0)
3084	3084	goto free_used_maps;
3085	3085
3086		− prog = bpf_prog_select_runtime(prog, &err);
3087		− if (err < 0)
3088		− goto free_used_maps;
3089		−
3090	3086	err = bpf_prog_mark_insn_arrays_ready(prog);
3091	3087	if (err < 0)
3092	3088	goto free_used_maps;

kernel/bpf/verifier.c+19−7

@@ -3497,6 +3497,11 @@ static int insn_stack_access_flags(int frameno, int spi) · 3496 unmodified lines
3497	3497	return INSN_F_STACK_ACCESS \| (spi << INSN_F_SPI_SHIFT) \| frameno;
3498	3498	}
3499	3499
	3500	+static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
	3501	+{
	3502	+ env->insn_aux_data[idx].indirect_target = true;
	3503	+}
	3504	+
3500	3505	#define LR_FRAMENO_BITS 3
3501	3506	#define LR_SPI_BITS 6
3502	3507	#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)
@@ -4544,6 +4549,9 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, · 1041 unmodified lines
4544	4549	int perm_flags;
4545	4550	const char *reg_name = "";
4546	4551
	4552	+ if (base_type(reg->type) != PTR_TO_BTF_ID)
	4553	+ goto bad_type;
	4554	+
4547	4555	if (btf_is_kernel(reg->btf)) {
4548	4556	perm_flags = PTR_MAYBE_NULL \| PTR_TRUSTED \| MEM_RCU;
4549	4557
@@ -4556,7 +4564,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, · 6 unmodified lines
4556	4564	perm_flags \|= MEM_PERCPU;
4557	4565	}
4558	4566
4559		− if (base_type(reg->type) != PTR_TO_BTF_ID \|\| (type_flag(reg->type) & ~perm_flags))
	4567	+ if (type_flag(reg->type) & ~perm_flags)
4560	4568	goto bad_type;
4561	4569
4562	4570	/* We need to verify reg->type and reg->btf, before accessing reg->btf */
@@ -17545,12 +17553,14 @@ static int check_indirect_jump(struct bpf_verifier_env env, struct bpf_insn in · 12982 unmodified lines
17545	17553	}
17546	17554
17547	17555	for (i = 0; i < n - 1; i++) {
	17556	+ mark_indirect_target(env, env->gotox_tmp_buf->items[i]);
17548	17557	other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
17549	17558	env->insn_idx, env->cur_state->speculative);
17550	17559	if (IS_ERR(other_branch))
17551	17560	return PTR_ERR(other_branch);
17552	17561	}
17553	17562	env->insn_idx = env->gotox_tmp_buf->items[n-1];
	17563	+ mark_indirect_target(env, env->insn_idx);
17554	17564	return INSN_IDX_UPDATED;
17555	17565	}
17556	17566
@@ -20155,6 +20165,14 @@ skip_full_check: · 2598 unmodified lines
20155	20165
20156	20166	adjust_btf_func(env);
20157	20167
	20168	+ /* extension progs temporarily inherit the attach_type of their targets
	20169	+ for verification purposes, so set it back to zero before returning
	20170	+ */
	20171	+ if (env->prog->type == BPF_PROG_TYPE_EXT)
	20172	+ env->prog->expected_attach_type = 0;
	20173	+
	20174	+ env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
	20175	+
20158	20176	err_release_maps:
20159	20177	if (ret)
20160	20178	release_insn_arrays(env);
@@ -20166,12 +20184,6 @@ err_release_maps: · 5 unmodified lines
20166	20184	if (!env->prog->aux->used_btfs)
20167	20185	release_btfs(env);
20168	20186
20169		− /* extension progs temporarily inherit the attach_type of their targets
20170		− for verification purposes, so set it back to zero before returning
20171		− */
20172		− if (env->prog->type == BPF_PROG_TYPE_EXT)
20173		− env->prog->expected_attach_type = 0;
20174		−
20175	20187	*prog = env->prog;
20176	20188
20177	20189	module_put(env->attach_btf_mod);

net/core/filter.c+1−1

@@ -503,7 +503,7 @@ static bool convert_bpf_ld_abs(struct sock_filter fp, struct bpf_insn *insnp) · 502 unmodified lines
503	503	((unaligned_ok && offset >= 0) \|\|
504	504	(!unaligned_ok && offset >= 0 &&
505	505	offset + ip_align >= 0 &&
506		− offset + ip_align % size == 0))) {
	506	+ (offset + ip_align) % size == 0))) {
507	507	bool ldx_off_ok = offset <= S16_MAX;
508	508
509	509	*insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);

net/core/sock_map.c+2−2

@@ -530,7 +530,7 @@ static bool sock_map_redirect_allowed(const struct sock *sk) · 529 unmodified lines
530	530	if (sk_is_tcp(sk))
531	531	return sk->sk_state != TCP_LISTEN;
532	532	else
533		− return sk->sk_state == TCP_ESTABLISHED;
	533	+ return READ_ONCE(sk->sk_state) == TCP_ESTABLISHED;
534	534	}
535	535
536	536	static bool sock_map_sk_is_suitable(const struct sock *sk)
@@ -543,7 +543,7 @@ static bool sock_map_sk_state_allowed(const struct sock *sk) · 6 unmodified lines
543	543	if (sk_is_tcp(sk))
544	544	return (1 << sk->sk_state) & (TCPF_ESTABLISHED \| TCPF_LISTEN);
545	545	if (sk_is_stream_unix(sk))
546		− return (1 << sk->sk_state) & TCPF_ESTABLISHED;
	546	+ return (1 << READ_ONCE(sk->sk_state)) & TCPF_ESTABLISHED;
547	547	if (sk_is_vsock(sk) &&
548	548	(sk->sk_type == SOCK_STREAM \|\| sk->sk_type == SOCK_SEQPACKET))
549	549	return (1 << sk->sk_state) & TCPF_ESTABLISHED;

net/unix/af_unix.c+5−4

@@ -3735,15 +3735,15 @@ static int bpf_iter_unix_seq_show(struct seq_file seq, void v) · 3734 unmodified lines
3735	3735	struct bpf_prog *prog;
3736	3736	struct sock *sk = v;
3737	3737	uid_t uid;
3738		− bool slow;
3739	3738	int ret;
3740	3739
3741	3740	if (v == SEQ_START_TOKEN)
3742	3741	return 0;
3743	3742
3744		− slow = lock_sock_fast(sk);
	3743	+ lock_sock(sk);
	3744	+ unix_state_lock(sk);
3745	3745
3746		− if (unlikely(sk_unhashed(sk))) {
	3746	+ if (unlikely(sock_flag(sk, SOCK_DEAD))) {
3747	3747	ret = SEQ_SKIP;
3748	3748	goto unlock;
3749	3749	}
@@ -3753,7 +3753,8 @@ static int bpf_iter_unix_seq_show(struct seq_file seq, void v) · 3 unmodified lines
3753	3753	prog = bpf_iter_get_info(&meta, false);
3754	3754	ret = unix_prog_seq_show(prog, &meta, v, uid);
3755	3755	unlock:
3756		− unlock_sock_fast(sk, slow);
	3756	+ unix_state_unlock(sk);
	3757	+ release_sock(sk);
3757	3758	return ret;
3758	3759	}
3759	3760

net/unix/unix_bpf.c+3−0

@@ -185,6 +185,9 @@ int unix_stream_bpf_update_proto(struct sock sk, struct sk_psock psock, bool r · 184 unmodified lines
185	185	*/
186	186	if (!psock->sk_pair) {
187	187	sk_pair = unix_peer(sk);
	188	+ if (unlikely(!sk_pair))
	189	+ return -EINVAL;
	190	+
188	191	sock_hold(sk_pair);
189	192	psock->sk_pair = sk_pair;
190	193	}

tools/lib/bpf/libbpf.c+11−10

@@ -5852,11 +5852,12 @@ static int load_module_btfs(struct bpf_object *obj) · 5851 unmodified lines
5852	5852	info.name = ptr_to_u64(name);
5853	5853	info.name_len = sizeof(name);
5854	5854
	5855	+ btf = NULL;
5855	5856	err = bpf_btf_get_info_by_fd(fd, &info, &len);
5856	5857	if (err) {
5857	5858	err = -errno;
5858	5859	pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err));
5859		− goto err_out;
	5860	+ break;
5860	5861	}
5861	5862
5862	5863	/* ignore non-module BTFs */
@@ -5870,15 +5871,15 @@ static int load_module_btfs(struct bpf_object *obj) · 7 unmodified lines
5870	5871	if (err) {
5871	5872	pr_warn("failed to load module [%s]'s BTF object #%d: %s\n",
5872	5873	name, id, errstr(err));
5873		− goto err_out;
	5874	+ break;
5874	5875	}
5875	5876
5876	5877	err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5877	5878	sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5878	5879	if (err)
5879		− goto err_out;
	5880	+ break;
5880	5881
5881		− mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
	5882	+ mod_btf = &obj->btf_modules[obj->btf_module_cnt];
5882	5883
5883	5884	mod_btf->btf = btf;
5884	5885	mod_btf->id = id;
@@ -5886,16 +5887,16 @@ static int load_module_btfs(struct bpf_object *obj) · 1 unmodified line
5886	5887	mod_btf->name = strdup(name);
5887	5888	if (!mod_btf->name) {
5888	5889	err = -ENOMEM;
5889		− goto err_out;
	5890	+ break;
5890	5891	}
5891		− continue;
	5892	+ obj->btf_module_cnt++;
	5893	+ }
5892	5894
5893		−err_out:
	5895	+ if (err) {
	5896	+ btf__free(btf);
5894	5897	close(fd);
5895		− return err;
5896	5898	}
5897		−
5898		− return 0;
	5899	+ return err;
5899	5900	}
5900	5901
5901	5902	static struct bpf_core_cand_list *

tools/testing/selftests/bpf/Makefile+1−0

@@ -751,6 +751,7 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \ · 750 unmodified lines
751	751	btf_helpers.c \
752	752	cap_helpers.c \
753	753	unpriv_helpers.c \
	754	+ sysctl_helpers.c \
754	755	netlink_helpers.c \
755	756	jit_disasm_helpers.c \
756	757	io_helpers.c \

tools/testing/selftests/bpf/prog_tests/snprintf.c+2−1

@@ -114,7 +114,8 @@ static void test_snprintf_negative(void) · 113 unmodified lines
114	114	ASSERT_ERR(load_single_snprintf("%--------"), "invalid specifier 5");
115	115	ASSERT_ERR(load_single_snprintf("%lc"), "invalid specifier 6");
116	116	ASSERT_ERR(load_single_snprintf("%llc"), "invalid specifier 7");
117		− ASSERT_ERR(load_single_snprintf("\x80"), "non ascii character");
	117	+ ASSERT_OK(load_single_snprintf("\x80"), "non ascii plain text");
	118	+ ASSERT_ERR(load_single_snprintf("%\x80"), "non ascii in specifier");
118	119	ASSERT_ERR(load_single_snprintf("\x1"), "non printable character");
119	120	ASSERT_ERR(load_single_snprintf("%p%"), "invalid specifier 8");
120	121	ASSERT_ERR(load_single_snprintf("%s%"), "invalid specifier 9");

tools/testing/selftests/bpf/prog_tests/task_local_data.h+10−3

@@ -99,14 +99,20 @@ struct tld_meta_u { · 98 unmodified lines
99	99	struct tld_metadata metadata[];
100	100	};
101	101
	102	+/*
	103	+ * The unused field ensures map_val.start > 0. On the BPF side, __tld_fetch_key()
	104	+ * calculates off by summing map_val.start and tld_key_t.off and treats off == 0
	105	+ * as key not cached.
	106	+ */
102	107	struct tld_data_u {
103		− __u64 start; /* offset of tld_data_u->data in a page */
	108	+ __u64 unused;
104	109	char data[] __attribute__((aligned(8)));
105	110	};
106	111
107	112	struct tld_map_value {
108	113	void *data;
109	114	struct tld_meta_u *meta;
	115	+ __u16 start; /* offset of tld_data_u->data in a page */
110	116	};
111	117
112	118	struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak));
@@ -182,7 +188,7 @@ static int __tld_init_data_p(int map_fd) · 69 unmodified lines
182	188	* is a page in BTF.
183	189	*/
184	190	map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data);
185		− data->start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u);
	191	+ map_val.start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u);
186	192	map_val.meta = tld_meta_p;
187	193
188	194	err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0);
@@ -241,7 +247,8 @@ retry: · 52 unmodified lines
241	247	* TLD_DYN_DATA_SIZE is allocated for tld_create_key()
242	248	*/
243	249	if (dyn_data) {
244		− if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size)
	250	+ if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size \|\|
	251	+ tld_meta_p->size > TLD_PAGE_SIZE - sizeof(struct tld_data_u))
245	252	return (tld_key_t){-E2BIG};
246	253	} else {
247	254	if (off + TLD_ROUND_UP(size, 8) > TLD_PAGE_SIZE - sizeof(struct tld_data_u))

tools/testing/selftests/bpf/prog_tests/test_task_local_data.c+87−9

@@ -3,8 +3,14 @@ · 2 unmodified lines
3	3	#include <bpf/btf.h>
4	4	#include <test_progs.h>
5	5
	6	+/*
	7	+ * Only a page is pinned to kernel, so the maximum amount of dynamic data
	8	+ * allowed is page_size - sizeof(struct tld_data_u) - static TLD fields.
	9	+ */
	10	+#define TLD_DYN_DATA_SIZE_MAX (getpagesize() - sizeof(struct tld_data_u) - 8)
	11	+
6	12	#define TLD_FREE_DATA_ON_THREAD_EXIT
7		−#define TLD_DYN_DATA_SIZE (getpagesize() - 8)
	13	+#define TLD_DYN_DATA_SIZE TLD_DYN_DATA_SIZE_MAX
8	14	#include "task_local_data.h"
9	15
10	16	struct test_tld_struct {
@@ -24,12 +30,12 @@ TLD_DEFINE_KEY(value0_key, "value0", sizeof(int)); · 13 unmodified lines
24	30	* sequentially. Users of task local data library should not touch
25	31	* library internal.
26	32	*/
27		−static void reset_tld(void)
	33	+static void reset_tld(__u16 dyn_data_size)
28	34	{
29	35	if (tld_meta_p) {
30	36	/* Remove TLDs created by tld_create_key() */
31	37	tld_meta_p->cnt = 1;
32		− tld_meta_p->size = TLD_DYN_DATA_SIZE;
	38	+ tld_meta_p->size = dyn_data_size + 8;
33	39	memset(&tld_meta_p->metadata[1], 0,
34	40	(TLD_MAX_DATA_CNT - 1) * sizeof(struct tld_metadata));
35	41	}
@@ -127,7 +133,7 @@ static void test_task_local_data_basic(void) · 91 unmodified lines
127	133	tld_key_t key;
128	134	int i, err;
129	135
130		− reset_tld();
	136	+ reset_tld(TLD_DYN_DATA_SIZE_MAX);
131	137
132	138	ASSERT_OK(pthread_mutex_init(&global_mutex, NULL), "pthread_mutex_init");
133	139
@@ -147,11 +153,13 @@ static void test_task_local_data_basic(void) · 13 unmodified lines
147	153
148	154	/*
149	155	* Shouldn't be able to store data exceed a page. Create a TLD just big
150		− * enough to exceed a page. TLDs already created are int value0, int
151		− * value1, and struct test_tld_struct value2.
	156	+ * enough to exceed a page. Data already contains struct tld_data_u,
	157	+ * value0 and value1 of int type, and value 2 of struct test_tld_struct.
152	158	*/
153		− key = tld_create_key("value_not_exist",
154		− TLD_PAGE_SIZE - 2 * sizeof(int) - sizeof(struct test_tld_struct) + 1);
	159	+ key = tld_create_key("value_not_exist", TLD_PAGE_SIZE + 1 -
	160	+ sizeof(struct tld_data_u) -
	161	+ TLD_ROUND_UP(sizeof(int), 8) * 2 -
	162	+ TLD_ROUND_UP(sizeof(struct test_tld_struct), 8));
155	163	ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key");
156	164
157	165	key = tld_create_key("value2", sizeof(struct test_tld_struct));
@@ -239,7 +247,7 @@ static void test_task_local_data_race(void) · 81 unmodified lines
239	247	tld_keys[0] = value0_key;
240	248
241	249	for (j = 0; j < 100; j++) {
242		− reset_tld();
	250	+ reset_tld(TLD_DYN_DATA_SIZE_MAX);
243	251
244	252	for (i = 0; i < TEST_RACE_THREAD_NUM; i++) {
245	253	/*
@@ -288,10 +296,80 @@ out: · 42 unmodified lines
288	296	test_task_local_data__destroy(skel);
289	297	}
290	298
	299	+static void test_task_local_data_dyn_size(__u16 dyn_data_size)
	300	+{
	301	+ LIBBPF_OPTS(bpf_test_run_opts, opts);
	302	+ struct test_task_local_data *skel;
	303	+ int max_keys, i, err, fd, *data;
	304	+ char name[TLD_NAME_LEN];
	305	+ tld_key_t key;
	306	+
	307	+ reset_tld(dyn_data_size);
	308	+
	309	+ skel = test_task_local_data__open_and_load();
	310	+ if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
	311	+ return;
	312	+
	313	+ tld_keys = calloc(TLD_MAX_DATA_CNT, sizeof(tld_key_t));
	314	+ if (!ASSERT_OK_PTR(tld_keys, "calloc tld_keys"))
	315	+ goto out;
	316	+
	317	+ fd = bpf_map__fd(skel->maps.tld_data_map);
	318	+
	319	+ /* Create as many int-sized TLDs as the dynamic data size allows */
	320	+ max_keys = dyn_data_size / TLD_ROUND_UP(sizeof(int), 8);
	321	+ for (i = 0; i < max_keys; i++) {
	322	+ snprintf(name, TLD_NAME_LEN, "value_%d", i);
	323	+ tld_keys[i] = tld_create_key(name, sizeof(int));
	324	+ if (!ASSERT_FALSE(tld_key_is_err(tld_keys[i]), "tld_create_key"))
	325	+ goto out;
	326	+
	327	+ data = tld_get_data(fd, tld_keys[i]);
	328	+ if (!ASSERT_OK_PTR(data, "tld_get_data"))
	329	+ goto out;
	330	+ *data = i;
	331	+ }
	332	+
	333	+ /* The next key should fail with E2BIG */
	334	+ key = tld_create_key("overflow", sizeof(int));
	335	+ ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key overflow");
	336	+
	337	+ /* Verify data for value_i do not overlap */
	338	+ for (i = 0; i < max_keys; i++) {
	339	+ data = tld_get_data(fd, tld_keys[i]);
	340	+ if (!ASSERT_OK_PTR(data, "tld_get_data"))
	341	+ goto out;
	342	+
	343	+ ASSERT_EQ(*data, i, "tld_get_data value_i");
	344	+ }
	345	+
	346	+ /* Verify BPF side can still read the static key */
	347	+ data = tld_get_data(fd, value0_key);
	348	+ if (!ASSERT_OK_PTR(data, "tld_get_data value0"))
	349	+ goto out;
	350	+ *data = 0xdeadbeef;
	351	+
	352	+ err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.task_main), &opts);
	353	+ ASSERT_OK(err, "run task_main");
	354	+ ASSERT_EQ(skel->bss->test_value0, 0xdeadbeef, "tld_get_data value0");
	355	+
	356	+out:
	357	+ if (tld_keys) {
	358	+ free(tld_keys);
	359	+ tld_keys = NULL;
	360	+ }
	361	+ tld_free();
	362	+ test_task_local_data__destroy(skel);
	363	+}
	364	+
291	365	void test_task_local_data(void)
292	366	{
293	367	if (test__start_subtest("task_local_data_basic"))
294	368	test_task_local_data_basic();
295	369	if (test__start_subtest("task_local_data_race"))
296	370	test_task_local_data_race();
	371	+ if (test__start_subtest("task_local_data_dyn_size_small"))
	372	+ test_task_local_data_dyn_size(64);
	373	+ if (test__start_subtest("task_local_data_dyn_size_zero"))
	374	+ test_task_local_data_dyn_size(0);
297	375	}

tools/testing/selftests/bpf/prog_tests/token.c+83−3

@@ -1,9 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
3	3	#define _GNU_SOURCE
4		−#include <test_progs.h>
5	4	#include <bpf/btf.h>
6		−#include "cap_helpers.h"
7	5	#include <fcntl.h>
8	6	#include <sched.h>
9	7	#include <signal.h>
@@ -15,9 +13,17 @@ · 5 unmodified lines
15	13	#include <sys/stat.h>
16	14	#include <sys/syscall.h>
17	15	#include <sys/un.h>
	16	+
	17	+#include "bpf_util.h"
	18	+#include "cap_helpers.h"
	19	+#include "sysctl_helpers.h"
	20	+#include "test_progs.h"
	21	+#include "trace_helpers.h"
	22	+
18	23	#include "priv_map.skel.h"
19	24	#include "priv_prog.skel.h"
20	25	#include "dummy_st_ops_success.skel.h"
	26	+#include "token_kallsyms.skel.h"
21	27	#include "token_lsm.skel.h"
22	28	#include "priv_freplace_prog.skel.h"
23	29
@@ -1045,6 +1051,58 @@ err_out: · 1021 unmodified lines
1045	1051	return -EINVAL;
1046	1052	}
1047	1053
	1054	+static bool kallsyms_has_bpf_func(struct ksyms ksyms, const char func_name)
	1055	+{
	1056	+ char name[256];
	1057	+ int i;
	1058	+
	1059	+ for (i = 0; i < ksyms->sym_cnt; i++) {
	1060	+ if (sscanf(ksyms->syms[i].name, "bpf_prog_%*[^_]_%255s", name) == 1 &&
	1061	+ strcmp(name, func_name) == 0)
	1062	+ return true;
	1063	+ }
	1064	+ return false;
	1065	+}
	1066	+
	1067	+static int userns_obj_priv_prog_kallsyms(int mnt_fd, struct token_lsm *lsm_skel)
	1068	+{
	1069	+ const char *func_names[] = { "xdp_main", "token_ksym_subprog" };
	1070	+ LIBBPF_OPTS(bpf_object_open_opts, opts);
	1071	+ struct token_kallsyms *skel;
	1072	+ struct ksyms *ksyms = NULL;
	1073	+ char buf[256];
	1074	+ int i, err;
	1075	+
	1076	+ snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd);
	1077	+ opts.bpf_token_path = buf;
	1078	+ skel = token_kallsyms__open_opts(&opts);
	1079	+ if (!ASSERT_OK_PTR(skel, "token_kallsyms__open_opts"))
	1080	+ return -EINVAL;
	1081	+
	1082	+ err = token_kallsyms__load(skel);
	1083	+ if (!ASSERT_OK(err, "token_kallsyms__load"))
	1084	+ goto cleanup;
	1085	+
	1086	+ ksyms = load_kallsyms_local();
	1087	+ if (!ASSERT_OK_PTR(ksyms, "load_kallsyms_local")) {
	1088	+ err = -EINVAL;
	1089	+ goto cleanup;
	1090	+ }
	1091	+
	1092	+ for (i = 0; i < ARRAY_SIZE(func_names); i++) {
	1093	+ if (!ASSERT_TRUE(kallsyms_has_bpf_func(ksyms, func_names[i]),
	1094	+ func_names[i])) {
	1095	+ err = -EINVAL;
	1096	+ break;
	1097	+ }
	1098	+ }
	1099	+
	1100	+cleanup:
	1101	+ free_kallsyms_local(ksyms);
	1102	+ token_kallsyms__destroy(skel);
	1103	+ return err;
	1104	+}
	1105	+
1048	1106	#define bit(n) (1ULL << (n))
1049	1107
1050	1108	static int userns_bpf_token_info(int mnt_fd, struct token_lsm *lsm_skel)
@@ -1082,7 +1140,7 @@ cleanup: · 31 unmodified lines
1082	1140	return err;
1083	1141	}
1084	1142
1085		−void test_token(void)
	1143	+void serial_test_token(void)
1086	1144	{
1087	1145	if (test__start_subtest("map_token")) {
1088	1146	struct bpffs_opts opts = {
@@ -1194,4 +1252,26 @@ void test_token(void) · 105 unmodified lines
1194	1252
1195	1253	subtest_userns(&opts, userns_bpf_token_info);
1196	1254	}
	1255	+ if (test__start_subtest("obj_priv_prog_kallsyms")) {
	1256	+ char perf_paranoid_orig[32] = {};
	1257	+ char kptr_restrict_orig[32] = {};
	1258	+ struct bpffs_opts opts = {
	1259	+ .cmds = bit(BPF_BTF_LOAD) \| bit(BPF_PROG_LOAD),
	1260	+ .progs = bit(BPF_PROG_TYPE_XDP),
	1261	+ .attachs = ~0ULL,
	1262	+ };
	1263	+
	1264	+ if (sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", perf_paranoid_orig, "0"))
	1265	+ goto cleanup;
	1266	+ if (sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", kptr_restrict_orig, "0"))
	1267	+ goto cleanup;
	1268	+
	1269	+ subtest_userns(&opts, userns_obj_priv_prog_kallsyms);
	1270	+
	1271	+cleanup:
	1272	+ if (perf_paranoid_orig[0])
	1273	+ sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", NULL, perf_paranoid_orig);
	1274	+ if (kptr_restrict_orig[0])
	1275	+ sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", NULL, kptr_restrict_orig);
	1276	+ }
1197	1277	}

tools/testing/selftests/bpf/prog_tests/trace_printk.c+22−6

@@ -6,18 +6,21 @@ · 5 unmodified lines
6	6	#include "trace_printk.lskel.h"
7	7
8	8	#define SEARCHMSG "testing,testing"
	9	+#define SEARCHMSG_UTF8 "中文,测试"
9	10
10	11	static void trace_pipe_cb(const char str, void data)
11	12	{
12	13	if (strstr(str, SEARCHMSG) != NULL)
13		− ((int )data)++;
	14	+ ((int *)data)[0]++;
	15	+ if (strstr(str, SEARCHMSG_UTF8))
	16	+ ((int *)data)[1]++;
14	17	}
15	18
16	19	void serial_test_trace_printk(void)
17	20	{
18	21	struct trace_printk_lskel__bss *bss;
19	22	struct trace_printk_lskel *skel;
20		− int err = 0, found = 0;
	23	+ int err = 0, found[2] = {};
21	24
22	25	skel = trace_printk_lskel__open();
23	26	if (!ASSERT_OK_PTR(skel, "trace_printk__open"))
@@ -46,11 +49,24 @@ void serial_test_trace_printk(void) · 22 unmodified lines
46	49	if (!ASSERT_GT(bss->trace_printk_ret, 0, "bss->trace_printk_ret"))
47	50	goto cleanup;
48	51
49		− /* verify our search string is in the trace buffer */
50		− ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, &found, 1000),
51		− "read_trace_pipe_iter");
	52	+ if (!ASSERT_GT(bss->trace_printk_utf8_ran, 0, "bss->trace_printk_utf8_ran"))
	53	+ goto cleanup;
	54	+
	55	+ if (!ASSERT_GT(bss->trace_printk_utf8_ret, 0, "bss->trace_printk_utf8_ret"))
	56	+ goto cleanup;
	57	+
	58	+ if (!ASSERT_LT(bss->trace_printk_invalid_spec_ret, 0,
	59	+ "bss->trace_printk_invalid_spec_ret"))
	60	+ goto cleanup;
	61	+
	62	+ /* verify our search strings are in the trace buffer */
	63	+ ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, found, 1000),
	64	+ "read_trace_pipe_iter");
	65	+
	66	+ if (!ASSERT_EQ(found[0], bss->trace_printk_ran, "found"))
	67	+ goto cleanup;
52	68
53		− if (!ASSERT_EQ(found, bss->trace_printk_ran, "found"))
	69	+ if (!ASSERT_EQ(found[1], bss->trace_printk_utf8_ran, "found_utf8"))
54	70	goto cleanup;
55	71
56	72	cleanup:

tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c+1−20

@@ -8,6 +8,7 @@ · 7 unmodified lines
8	8
9	9	#include "cap_helpers.h"
10	10	#include "bpf_util.h"
	11	+#include "sysctl_helpers.h"
11	12
12	13	/* Using CAP_LAST_CAP is risky here, since it can get pulled in from
13	14	* an old /usr/include/linux/capability.h and be < CAP_BPF; as a result
@@ -36,26 +37,6 @@ static void process_perfbuf(void ctx, int cpu, void data, __u32 len) · 22 unmodified lines
36	37	got_perfbuf_val = (__u32 )data;
37	38	}
38	39
39		−static int sysctl_set(const char sysctl_path, char old_val, const char *new_val)
40		−{
41		− int ret = 0;
42		− FILE *fp;
43		−
44		− fp = fopen(sysctl_path, "r+");
45		− if (!fp)
46		− return -errno;
47		− if (old_val && fscanf(fp, "%s", old_val) <= 0) {
48		− ret = -ENOENT;
49		− } else if (!old_val \|\| strcmp(old_val, new_val) != 0) {
50		− fseek(fp, 0, SEEK_SET);
51		− if (fprintf(fp, "%s", new_val) < 0)
52		− ret = -errno;
53		− }
54		− fclose(fp);
55		−
56		− return ret;
57		−}
58		−
59	40	static void test_unpriv_bpf_disabled_positive(struct test_unpriv_bpf_disabled *skel,
60	41	__u32 prog_id, int prog_fd, int perf_fd,
61	42	char *map_paths, int map_fds)

tools/testing/selftests/bpf/progs/bpf_iter_unix.c+10−0

@@ -7,6 +7,13 @@ · 6 unmodified lines
7	7
8	8	char _license[] SEC("license") = "GPL";
9	9
	10	+SEC(".maps") struct {
	11	+ __uint(type, BPF_MAP_TYPE_SOCKMAP);
	12	+ __uint(max_entries, 1);
	13	+ __type(key, __u32);
	14	+ __type(value, __u64);
	15	+} sockmap;
	16	+
10	17	static long sock_i_ino(const struct sock *sk)
11	18	{
12	19	const struct socket *sk_socket = sk->sk_socket;
@@ -76,5 +83,8 @@ int dump_unix(struct bpf_iter__unix *ctx) · 63 unmodified lines
76	83
77	84	BPF_SEQ_PRINTF(seq, "\n");
78	85
	86	+ /* Test for deadlock. */
	87	+ bpf_map_update_elem(&sockmap, &(int){0}, sk, 0);
	88	+
79	89	return 0;
80	90	}

tools/testing/selftests/bpf/progs/map_kptr_fail.c+15−0

@@ -385,4 +385,19 @@ int kptr_xchg_possibly_null(struct __sk_buff *ctx) · 384 unmodified lines
385	385	return 0;
386	386	}
387	387
	388	+SEC("?tc")
	389	+__failure __msg("invalid kptr access, R")
	390	+int reject_scalar_store_to_kptr(struct __sk_buff *ctx)
	391	+{
	392	+ struct map_value *v;
	393	+ int key = 0;
	394	+
	395	+ v = bpf_map_lookup_elem(&array_map, &key);
	396	+ if (!v)
	397	+ return 0;
	398	+
	399	+ (volatile u64 )&v->unref_ptr = 0xBADC0DE;
	400	+ return 0;
	401	+}
	402	+
388	403	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/task_local_data.bpf.h+3−2

@@ -86,13 +86,14 @@ struct tld_meta_u { · 85 unmodified lines
86	86	};
87	87
88	88	struct tld_data_u {
89		− __u64 start; /* offset of tld_data_u->data in a page */
	89	+ __u64 unused;
90	90	char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8)));
91	91	};
92	92
93	93	struct tld_map_value {
94	94	struct tld_data_u __uptr *data;
95	95	struct tld_meta_u __uptr *meta;
	96	+ __u16 start; /* offset of tld_data_u->data in a page */
96	97	};
97	98
98	99	typedef struct tld_uptr_dummy {
@@ -176,7 +177,7 @@ static int __tld_fetch_key(struct tld_object tld_obj, const char name, int i_s · 77 unmodified lines
176	177	if (!tld_obj->data_map \|\| !tld_obj->data_map->data \|\| !tld_obj->data_map->meta)
177	178	return 0;
178	179
179		− start = tld_obj->data_map->data->start;
	180	+ start = tld_obj->data_map->start;
180	181	cnt = tld_obj->data_map->meta->cnt;
181	182	metadata = tld_obj->data_map->meta->metadata;
182	183

tools/testing/selftests/bpf/progs/timer_start_deadlock.c+4−4

@@ -27,13 +27,13 @@ static int timer_cb(void map, int key, struct elem *value) · 26 unmodified lines
27	27	return 0;
28	28	}
29	29
30		−SEC("tp_btf/hrtimer_cancel")
31		−int BPF_PROG(tp_hrtimer_cancel, struct hrtimer *hrtimer)
	30	+SEC("tp_btf/hrtimer_start")
	31	+int BPF_PROG(tp_hrtimer_start, struct hrtimer *hrtimer, enum hrtimer_mode mode, bool was_armed)
32	32	{
33	33	struct bpf_timer *timer;
34	34	int key = 0;
35	35
36		− if (!in_timer_start)
	36	+ if (!in_timer_start \|\| !was_armed)
37	37	return 0;
38	38
39	39	tp_called = 1;
@@ -60,7 +60,7 @@ int start_timer(void *ctx) · 20 unmodified lines
60	60
61	61	/*
62	62	* call hrtimer_start() twice, so that 2nd call does
63		− * remove_hrtimer() and trace_hrtimer_cancel() tracepoint.
	63	+ * trace_hrtimer_start(was_armed=1) tracepoint.
64	64	*/
65	65	in_timer_start = 1;
66	66	bpf_timer_start(timer, 1000000000, 0);

tools/testing/selftests/bpf/progs/token_kallsyms.c+19−0

@@ -0,0 +1,19 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
	3	+
	4	+#include "vmlinux.h"
	5	+#include <bpf/bpf_helpers.h>
	6	+
	7	+char _license[] SEC("license") = "GPL";
	8	+
	9	+__weak
	10	+int token_ksym_subprog(void)
	11	+{
	12	+ return 0;
	13	+}
	14	+
	15	+SEC("xdp")
	16	+int xdp_main(struct xdp_md *xdp)
	17	+{
	18	+ return token_ksym_subprog();
	19	+}

tools/testing/selftests/bpf/progs/trace_printk.c+10−0

@@ -10,13 +10,23 @@ char _license[] SEC("license") = "GPL"; · 9 unmodified lines
10	10
11	11	int trace_printk_ret = 0;
12	12	int trace_printk_ran = 0;
	13	+int trace_printk_invalid_spec_ret = 0;
	14	+int trace_printk_utf8_ret = 0;
	15	+int trace_printk_utf8_ran = 0;
13	16
14	17	const char fmt[] = "Testing,testing %d\n";
	18	+static const char utf8_fmt[] = "中文,测试 %d\n";
	19	+/* Non-ASCII bytes after '%' must still be rejected. */
	20	+static const char invalid_spec_fmt[] = "%\x80\n";
15	21
16	22	SEC("fentry/" SYS_PREFIX "sys_nanosleep")
17	23	int sys_enter(void *ctx)
18	24	{
19	25	trace_printk_ret = bpf_trace_printk(fmt, sizeof(fmt),
20	26	++trace_printk_ran);
	27	+ trace_printk_utf8_ret = bpf_trace_printk(utf8_fmt, sizeof(utf8_fmt),
	28	+ ++trace_printk_utf8_ran);
	29	+ trace_printk_invalid_spec_ret = bpf_trace_printk(invalid_spec_fmt,
	30	+ sizeof(invalid_spec_fmt));
21	31	return 0;
22	32	}

tools/testing/selftests/bpf/progs/verifier_live_stack.c+193−0

@@ -2647,3 +2647,196 @@ __naked void spill_join_with_imprecise_off(void) · 2646 unmodified lines
2647	2647	"exit;"
2648	2648	::: __clobber_all);
2649	2649	}
	2650	+
	2651	+/*
	2652	+ * Same as spill_join_with_multi_off but the write is BPF_ST (store
	2653	+ * immediate) instead of BPF_STX. BPF_ST goes through
	2654	+ * clear_stack_for_all_offs() rather than spill_to_stack(), and that
	2655	+ * path also needs to join instead of overwriting.
	2656	+ *
	2657	+ * fp-8 = &fp-24
	2658	+ * fp-16 = &fp-32
	2659	+ * r1 = fp-8 or fp-16 (two offsets from branch)
	2660	+ * (u64 )(r1 + 0) = 0 -- BPF_ST with immediate
	2661	+ * r0 = (u64 )(r10 - 16) -- fill from fp-16
	2662	+ * r0 = (u64 )(r0 + 0) -- deref: should produce use
	2663	+ */
	2664	+SEC("socket")
	2665	+__log_level(2)
	2666	+__failure
	2667	+__msg("15: (7a) (u64 )(r1 +0) = 0 fp-8: fp0-24 -> fp0-24\|fp0+0 fp-16: fp0-32 -> fp0-32\|fp0+0")
	2668	+__msg("17: (79) r0 = (u64 )(r0 +0) ; use: fp0-32")
	2669	+__naked void st_imm_join_with_multi_off(void)
	2670	+{
	2671	+ asm volatile (
	2672	+ "(u64 )(r10 - 24) = 0;"
	2673	+ "(u64 )(r10 - 32) = 0;"
	2674	+ "r1 = r10;"
	2675	+ "r1 += -24;"
	2676	+ "(u64 )(r10 - 8) = r1;"
	2677	+ "r1 = r10;"
	2678	+ "r1 += -32;"
	2679	+ "(u64 )(r10 - 16) = r1;"
	2680	+ /* create r1 with two candidate offsets: fp-8 or fp-16 */
	2681	+ "call %[bpf_get_prandom_u32];"
	2682	+ "if r0 == 0 goto 1f;"
	2683	+ "r1 = r10;"
	2684	+ "r1 += -8;"
	2685	+ "goto 2f;"
	2686	+"1:"
	2687	+ "r1 = r10;"
	2688	+ "r1 += -16;"
	2689	+"2:"
	2690	+ /* BPF_ST: store immediate through multi-offset r1 */
	2691	+ "(u64 )(r1 + 0) = 0;"
	2692	+ /* read back fp-16 and deref */
	2693	+ "r0 = (u64 )(r10 - 16);"
	2694	+ "r0 = (u64 )(r0 + 0);"
	2695	+ "r0 = 0;"
	2696	+ "exit;"
	2697	+ :: __imm(bpf_get_prandom_u32)
	2698	+ : __clobber_all);
	2699	+}
	2700	+
	2701	+/*
	2702	+ * Check that BPF_ST with a known offset fully overwrites stack slot
	2703	+ * from the arg tracking point of view.
	2704	+ */
	2705	+SEC("socket")
	2706	+__log_level(2)
	2707	+__success
	2708	+__msg("5: (7a) (u64 )(r1 +0) = 0 fp-8: fp0-16 -> _{{$}}")
	2709	+__naked void st_imm_join_with_single_off(void)
	2710	+{
	2711	+ asm volatile (
	2712	+ "r2 = r10;"
	2713	+ "r2 += -16;"
	2714	+ "(u64 )(r10 - 8) = r2;"
	2715	+ "r1 = r10;"
	2716	+ "r1 += -8;"
	2717	+ "(u64 )(r1 + 0) = 0;"
	2718	+ "r0 = 0;"
	2719	+ "exit;"
	2720	+ ::: __clobber_all);
	2721	+}
	2722	+
	2723	+/*
	2724	+ * Same as spill_join_with_imprecise_off but the write is BPF_ST.
	2725	+ * Use "r2 = -8; r1 += r2" to make arg tracking lose offset
	2726	+ * precision while the main verifier keeps r1 as fixed-offset.
	2727	+ *
	2728	+ * fp-8 = &fp-24
	2729	+ * fp-16 = &fp-32
	2730	+ * r1 = fp-8 (imprecise to arg tracking)
	2731	+ * (u64 )(r1 + 0) = 0 -- BPF_ST with immediate
	2732	+ * r0 = (u64 )(r10 - 16) -- fill from fp-16
	2733	+ * r0 = (u64 )(r0 + 0) -- deref: should produce use
	2734	+ */
	2735	+SEC("socket")
	2736	+__log_level(2)
	2737	+__success
	2738	+__msg("13: (79) r0 = (u64 )(r0 +0) ; use: fp0-32")
	2739	+__naked void st_imm_join_with_imprecise_off(void)
	2740	+{
	2741	+ asm volatile (
	2742	+ "(u64 )(r10 - 24) = 0;"
	2743	+ "(u64 )(r10 - 32) = 0;"
	2744	+ "r1 = r10;"
	2745	+ "r1 += -24;"
	2746	+ "(u64 )(r10 - 8) = r1;"
	2747	+ "r1 = r10;"
	2748	+ "r1 += -32;"
	2749	+ "(u64 )(r10 - 16) = r1;"
	2750	+ /* r1 = fp-8 but arg tracking sees off_cnt == 0 */
	2751	+ "r1 = r10;"
	2752	+ "r2 = -8;"
	2753	+ "r1 += r2;"
	2754	+ /* store immediate through imprecise r1 */
	2755	+ "(u64 )(r1 + 0) = 0;"
	2756	+ /* read back fp-16 */
	2757	+ "r0 = (u64 )(r10 - 16);"
	2758	+ /* deref: should produce use */
	2759	+ "r0 = (u64 )(r0 + 0);"
	2760	+ "r0 = 0;"
	2761	+ "exit;"
	2762	+ ::: __clobber_all);
	2763	+}
	2764	+
	2765	+/*
	2766	+ * Test that spilling through an ARG_IMPRECISE pointer joins with
	2767	+ * existing at_stack values. Subprog receives r1 = fp0-24 and
	2768	+ * r2 = map_value, creates an ARG_IMPRECISE pointer by joining caller
	2769	+ * and callee FP on two branches.
	2770	+ *
	2771	+ * Setup: callee spills &fp1-16 to fp1-8 (precise, tracked).
	2772	+ * Then writes map_value through ARG_IMPRECISE r1 — on path A
	2773	+ * this hits fp1-8, on path B it hits caller stack.
	2774	+ * Since spill_to_stack is skipped for ARG_IMPRECISE dst,
	2775	+ * fp1-8 tracking isn't joined with none.
	2776	+ *
	2777	+ * Expected after the imprecise write:
	2778	+ * - arg tracking should show fp1-8 = fp1-16\|fp1+0 (joined with none)
	2779	+ * - read from fp1-8 and deref should produce use for fp1-16
	2780	+ * - write through it should NOT produce def for fp1-16
	2781	+ */
	2782	+SEC("socket")
	2783	+__log_level(2)
	2784	+__success
	2785	+__msg("26: (79) r0 = (u64 )(r10 -8) // r1=IMP3 r6=fp0-24 r7=fp1-16 fp-8=fp1-16\|fp1+0")
	2786	+__naked void imprecise_dst_spill_join(void)
	2787	+{
	2788	+ asm volatile (
	2789	+ "(u64 )(r10 - 24) = 0;"
	2790	+ /* map lookup for a valid non-FP pointer */
	2791	+ "(u32 )(r10 - 32) = 0;"
	2792	+ "r1 = %[map] ll;"
	2793	+ "r2 = r10;"
	2794	+ "r2 += -32;"
	2795	+ "call %[bpf_map_lookup_elem];"
	2796	+ "if r0 == 0 goto 1f;"
	2797	+ /* r1 = &caller_fp-24, r2 = map_value */
	2798	+ "r1 = r10;"
	2799	+ "r1 += -24;"
	2800	+ "r2 = r0;"
	2801	+ "call imprecise_dst_spill_join_sub;"
	2802	+"1:"
	2803	+ "r0 = 0;"
	2804	+ "exit;"
	2805	+ :: __imm_addr(map),
	2806	+ __imm(bpf_map_lookup_elem)
	2807	+ : __clobber_all);
	2808	+}
	2809	+
	2810	+static __used __naked void imprecise_dst_spill_join_sub(void)
	2811	+{
	2812	+ asm volatile (
	2813	+ /* r6 = &caller_fp-24 (frame=0), r8 = map_value */
	2814	+ "r6 = r1;"
	2815	+ "r8 = r2;"
	2816	+ /* spill &fp1-16 to fp1-8: at_stack[0] = fp1-16 */
	2817	+ "(u64 )(r10 - 16) = 0;"
	2818	+ "r7 = r10;"
	2819	+ "r7 += -16;"
	2820	+ "(u64 )(r10 - 8) = r7;"
	2821	+ /* branch to create ARG_IMPRECISE pointer */
	2822	+ "call %[bpf_get_prandom_u32];"
	2823	+ /* path B: r1 = caller fp-24 (frame=0) */
	2824	+ "r1 = r6;"
	2825	+ "if r0 == 0 goto 1f;"
	2826	+ /* path A: r1 = callee fp-8 (frame=1) */
	2827	+ "r1 = r10;"
	2828	+ "r1 += -8;"
	2829	+"1:"
	2830	+ /* r1 = ARG_IMPRECISE{mask=BIT(0)\|BIT(1)}.
	2831	+ * Write map_value (non-FP) through r1. On path A this overwrites fp1-8.
	2832	+ * Should join at_stack[0] with none: fp1-16\|fp1+0.
	2833	+ */
	2834	+ "(u64 )(r1 + 0) = r8;"
	2835	+ /* read fp1-8: should be fp1-16\|fp1+0 (joined) */
	2836	+ "r0 = (u64 )(r10 - 8);"
	2837	+ "(u64 )(r0 + 0) = 42;"
	2838	+ "r0 = 0;"
	2839	+ "exit;"
	2840	+ :: __imm(bpf_get_prandom_u32)
	2841	+ : __clobber_all);
	2842	+}

tools/testing/selftests/bpf/sysctl_helpers.c+37−0

@@ -0,0 +1,37 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <stdio.h>
	3	+#include <errno.h>
	4	+#include <string.h>
	5	+
	6	+#include "sysctl_helpers.h"
	7	+#include "test_progs.h"
	8	+
	9	+int sysctl_set(const char sysctl_path, char old_val, const char *new_val)
	10	+{
	11	+ int ret = 0;
	12	+ FILE *fp;
	13	+
	14	+ fp = fopen(sysctl_path, "r+");
	15	+ if (!fp)
	16	+ return -errno;
	17	+ if (old_val && fscanf(fp, "%s", old_val) <= 0) {
	18	+ ret = -ENOENT;
	19	+ } else if (!old_val \|\| strcmp(old_val, new_val) != 0) {
	20	+ fseek(fp, 0, SEEK_SET);
	21	+ if (fprintf(fp, "%s", new_val) < 0)
	22	+ ret = -errno;
	23	+ }
	24	+ fclose(fp);
	25	+
	26	+ return ret;
	27	+}
	28	+
	29	+int sysctl_set_or_fail(const char sysctl_path, char old_val, const char *new_val)
	30	+{
	31	+ int err;
	32	+
	33	+ err = sysctl_set(sysctl_path, old_val, new_val);
	34	+ if (err)
	35	+ PRINT_FAIL("failed to set %s to %s: %s\n", sysctl_path, new_val, strerror(-err));
	36	+ return err;
	37	+}

tools/testing/selftests/bpf/sysctl_helpers.h+8−0

@@ -0,0 +1,8 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __SYSCTL_HELPERS_H
	3	+#define __SYSCTL_HELPERS_H
	4	+
	5	+int sysctl_set(const char sysctl_path, char old_val, const char *new_val);
	6	+int sysctl_set_or_fail(const char sysctl_path, char old_val, const char *new_val);
	7	+
	8	+#endif