History

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull bpf fixes from Alexei Starovoitov: "Most of the diff stat comes from Xu Kuohai's fix to emit ENDBR/BTI, since all JITs had to be touched to move constant blinding out and pass bpf_verifier_env in. - Fix use-after-free in arena_vm_close on fork (Alexei Starovoitov) - Dissociate struct_ops program with map if map_update fails (Amery Hung) - Fix out-of-range and off-by-one bugs in arm64 JIT (Daniel Borkmann) - Fix precedence bug in convert_bpf_ld_abs alignment check (Daniel Borkmann) - Fix arg tracking for imprecise/multi-offset in BPF_ST/STX insns (Eduard Zingerman) - Copy token from main to subprogs to fix missing kallsyms (Eduard Zingerman) - Prevent double close and leak of btf objects in libbpf (Jiri Olsa) - Fix af_unix null-ptr-deref in sockmap (Michal Luczaj) - Fix NULL deref in map_kptr_match_type for scalar regs (Mykyta Yatsenko) - Avoid unnecessary IPIs. Remove redundant bpf_flush_icache() in arm64 and riscv JITs (Puranjay Mohan) - Fix out of bounds access. Validate node_id in arena_alloc_pages() (Puranjay Mohan) - Reject BPF-to-BPF calls and callbacks in arm32 JIT (Puranjay Mohan) - Refactor all JITs to pass bpf_verifier_env to emit ENDBR/BTI for indirect jump targets on x86-64, arm64 JITs (Xu Kuohai) - Allow UTF-8 literals in bpf_bprintf_prepare() (Yihan Ding)" * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (32 commits) bpf, arm32: Reject BPF-to-BPF calls and callbacks in the JIT bpf: Dissociate struct_ops program with map if map_update fails bpf: Validate node_id in arena_alloc_pages() libbpf: Prevent double close and leak of btf objects selftests/bpf: cover UTF-8 trace_printk output bpf: allow UTF-8 literals in bpf_bprintf_prepare() selftests/bpf: Reject scalar store into kptr slot bpf: Fix NULL deref in map_kptr_match_type for scalar regs bpf: Fix precedence bug in convert_bpf_ld_abs alignment check bpf, arm64: Emit BTI for indirect jump target bpf, x86: Emit ENDBR for indirect jump targets bpf: Add helper to detect indirect jump targets bpf: Pass bpf_verifier_env to JIT bpf: Move constants blinding out of arch-specific JITs bpf, sockmap: Take state lock for af_unix iter bpf, sockmap: Fix af_unix null-ptr-deref in proto update selftests/bpf: Extend bpf_iter_unix to attempt deadlocking bpf, sockmap: Fix af_unix iter deadlock bpf, sockmap: Annotate af_unix sock:: Sk_state data-races selftests/bpf: verify kallsyms entries for token-loaded subprograms ...

Linus Torvalds committed 7 months ago

Commit eb0d6d92 parents 12bffae e1d486446 files · +1173 −707Browse files

Unified · Split

46 files
  1. arch/arc/net/bpf_jit_core.c +15 −26
  2. arch/arm/net/bpf_jit_32.c +14 −35
  3. arch/arm64/lib/insn.c +2 −0
  4. arch/arm64/net/bpf_jit_comp.c +31 −72
  5. arch/loongarch/net/bpf_jit.c +18 −43
  6. arch/mips/net/bpf_jit_comp.c +2 −20
  7. arch/parisc/net/bpf_jit_core.c +30 −45
  8. arch/powerpc/net/bpf_jit_comp.c +27 −47
  9. arch/riscv/net/bpf_jit.h +0 −6
  10. arch/riscv/net/bpf_jit_core.c +23 −47
  11. arch/s390/net/bpf_jit_comp.c +21 −40
  12. arch/sparc/net/bpf_jit_comp_64.c +22 −41
  13. arch/x86/net/bpf_jit_comp.c +23 −50
  14. arch/x86/net/bpf_jit_comp32.c +5 −30
  15. include/linux/bpf.h +2 −0
  16. include/linux/bpf_verifier.h +5 −4
  17. include/linux/filter.h +48 −2
  18. kernel/bpf/arena.c +20 −3
  19. kernel/bpf/bpf_struct_ops.c +4 −3
  20. kernel/bpf/core.c +99 −39
  21. kernel/bpf/fixups.c +138 −25
  22. kernel/bpf/helpers.c +16 −1
  23. kernel/bpf/liveness.c +62 −52
  24. kernel/bpf/syscall.c +0 −4
  25. kernel/bpf/verifier.c +19 −7
  26. net/core/filter.c +1 −1
  27. net/core/sock_map.c +2 −2
  28. net/unix/af_unix.c +5 −4
  29. net/unix/unix_bpf.c +3 −0
  30. tools/lib/bpf/libbpf.c +11 −10
  31. tools/testing/selftests/bpf/Makefile +1 −0
  32. tools/testing/selftests/bpf/prog_tests/snprintf.c +2 −1
  33. tools/testing/selftests/bpf/prog_tests/task_local_data.h +10 −3
  34. tools/testing/selftests/bpf/prog_tests/test_task_local_data.c +87 −9
  35. tools/testing/selftests/bpf/prog_tests/token.c +83 −3
  36. tools/testing/selftests/bpf/prog_tests/trace_printk.c +22 −6
  37. tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c +1 −20
  38. tools/testing/selftests/bpf/progs/bpf_iter_unix.c +10 −0
  39. tools/testing/selftests/bpf/progs/map_kptr_fail.c +15 −0
  40. tools/testing/selftests/bpf/progs/task_local_data.bpf.h +3 −2
  41. tools/testing/selftests/bpf/progs/timer_start_deadlock.c +4 −4
  42. tools/testing/selftests/bpf/progs/token_kallsyms.c +19 −0
  43. tools/testing/selftests/bpf/progs/trace_printk.c +10 −0
  44. tools/testing/selftests/bpf/progs/verifier_live_stack.c +193 −0
  45. tools/testing/selftests/bpf/sysctl_helpers.c +37 −0
  46. tools/testing/selftests/bpf/sysctl_helpers.h +8 −0
arch/arc/net/bpf_jit_core.c+15−26
@@ -79,7 +79,6 @@ struct arc_jit_data { · 78 unmodified lines
79 * The JIT pertinent context that is used by different functions.79 * The JIT pertinent context that is used by different functions.
80 *80 *
81 * prog: The current eBPF program being handled.81 * prog: The current eBPF program being handled.
82 * orig_prog: The original eBPF program before any possible change.
83 * jit: The JIT buffer and its length.82 * jit: The JIT buffer and its length.
84 * bpf_header: The JITed program header. "jit.buf" points inside it.83 * bpf_header: The JITed program header. "jit.buf" points inside it.
85 * emit: If set, opcodes are written to memory; else, a dry-run.84 * emit: If set, opcodes are written to memory; else, a dry-run.
@@ -94,12 +93,10 @@ struct arc_jit_data { · 8 unmodified lines
94 * need_extra_pass: A forecast if an "extra_pass" will occur.93 * need_extra_pass: A forecast if an "extra_pass" will occur.
95 * is_extra_pass: Indicates if the current pass is an extra pass.94 * is_extra_pass: Indicates if the current pass is an extra pass.
96 * user_bpf_prog: True, if VM opcodes come from a real program.95 * user_bpf_prog: True, if VM opcodes come from a real program.
97 * blinded: True if "constant blinding" step returned a new "prog".
98 * success: Indicates if the whole JIT went OK.96 * success: Indicates if the whole JIT went OK.
99 */97 */
100struct jit_context {98struct jit_context {
101 struct bpf_prog *prog;99 struct bpf_prog *prog;
102 struct bpf_prog *orig_prog;
103 struct jit_buffer jit;100 struct jit_buffer jit;
104 struct bpf_binary_header *bpf_header;101 struct bpf_binary_header *bpf_header;
105 bool emit;102 bool emit;
@@ -114,7 +111,6 @@ struct jit_context { · 8 unmodified lines
114 bool need_extra_pass;111 bool need_extra_pass;
115 bool is_extra_pass;112 bool is_extra_pass;
116 bool user_bpf_prog;113 bool user_bpf_prog;
117 bool blinded;
118 bool success;114 bool success;
119};115};
120 116
@@ -161,13 +157,7 @@ static int jit_ctx_init(struct jit_context *ctx, struct bpf_prog *prog) · 40 unmodified lines
161{157{
162 memset(ctx, 0, sizeof(*ctx));158 memset(ctx, 0, sizeof(*ctx));
163 159
164 ctx->orig_prog = prog;160 ctx->prog = prog;
165
166 /* If constant blinding was requested but failed, scram. */
167 ctx->prog = bpf_jit_blind_constants(prog);
168 if (IS_ERR(ctx->prog))
169 return PTR_ERR(ctx->prog);
170 ctx->blinded = (ctx->prog != ctx->orig_prog);
171 161
172 /* If the verifier doesn't zero-extend, then we have to do it. */162 /* If the verifier doesn't zero-extend, then we have to do it. */
173 ctx->do_zext = !ctx->prog->aux->verifier_zext;163 ctx->do_zext = !ctx->prog->aux->verifier_zext;
@@ -214,14 +204,6 @@ static inline void maybe_free(struct jit_context *ctx, void **mem) · 40 unmodified lines
214 */204 */
215static void jit_ctx_cleanup(struct jit_context *ctx)205static void jit_ctx_cleanup(struct jit_context *ctx)
216{206{
217 if (ctx->blinded) {
218 /* if all went well, release the orig_prog. */
219 if (ctx->success)
220 bpf_jit_prog_release_other(ctx->prog, ctx->orig_prog);
221 else
222 bpf_jit_prog_release_other(ctx->orig_prog, ctx->prog);
223 }
224
225 maybe_free(ctx, (void **)&ctx->bpf2insn);207 maybe_free(ctx, (void **)&ctx->bpf2insn);
226 maybe_free(ctx, (void **)&ctx->jit_data);208 maybe_free(ctx, (void **)&ctx->jit_data);
227 209
@@ -229,12 +211,19 @@ static void jit_ctx_cleanup(struct jit_context *ctx) · 1 unmodified line
229 ctx->bpf2insn_valid = false;211 ctx->bpf2insn_valid = false;
230 212
231 /* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */213 /* Freeing "bpf_header" is enough. "jit.buf" is a sub-array of it. */
232 if (!ctx->success && ctx->bpf_header) {214 if (!ctx->success) {
233 bpf_jit_binary_free(ctx->bpf_header);215 if (ctx->bpf_header) {
234 ctx->bpf_header = NULL;216 bpf_jit_binary_free(ctx->bpf_header);
235 ctx->jit.buf = NULL;217 ctx->bpf_header = NULL;
236 ctx->jit.index = 0;218 ctx->jit.buf = NULL;
237 ctx->jit.len = 0;219 ctx->jit.index = 0;
220 ctx->jit.len = 0;
221 }
222 if (ctx->is_extra_pass) {
223 ctx->prog->bpf_func = NULL;
224 ctx->prog->jited = 0;
225 ctx->prog->jited_len = 0;
226 }
238 }227 }
239 228
240 ctx->emit = false;229 ctx->emit = false;
@@ -1411,7 +1400,7 @@ static struct bpf_prog *do_extra_pass(struct bpf_prog *prog) · 1170 unmodified lines
1411 * (re)locations involved that their addresses are not known1400 * (re)locations involved that their addresses are not known
1412 * during the first run.1401 * during the first run.
1413 */1402 */
1414struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)1403struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
1415{1404{
1416 vm_dump(prog);1405 vm_dump(prog);
1417 1406
arch/arm/net/bpf_jit_32.c+14−35
@@ -1852,6 +1852,9 @@ exit: · 1851 unmodified lines
1852 {1852 {
1853 u64 val = (u32)imm | (u64)insn[1].imm << 32;1853 u64 val = (u32)imm | (u64)insn[1].imm << 32;
1854 1854
1855 if (insn->src_reg == BPF_PSEUDO_FUNC)
1856 goto notyet;
1857
1855 emit_a32_mov_i64(dst, val, ctx);1858 emit_a32_mov_i64(dst, val, ctx);
1856 1859
1857 return 1;1860 return 1;
@@ -2055,6 +2058,9 @@ go_jmp: · 197 unmodified lines
2055 const s8 *r5 = bpf2a32[BPF_REG_5];2058 const s8 *r5 = bpf2a32[BPF_REG_5];
2056 const u32 func = (u32)__bpf_call_base + (u32)imm;2059 const u32 func = (u32)__bpf_call_base + (u32)imm;
2057 2060
2061 if (insn->src_reg == BPF_PSEUDO_CALL)
2062 goto notyet;
2063
2058 emit_a32_mov_r64(true, r0, r1, ctx);2064 emit_a32_mov_r64(true, r0, r1, ctx);
2059 emit_a32_mov_r64(true, r1, r2, ctx);2065 emit_a32_mov_r64(true, r1, r2, ctx);
2060 emit_push_r64(r5, ctx);2066 emit_push_r64(r5, ctx);
@@ -2142,11 +2148,9 @@ bool bpf_jit_needs_zext(void) · 81 unmodified lines
2142 return true;2148 return true;
2143}2149}
2144 2150
2145struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)2151struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
2146{2152{
2147 struct bpf_prog *tmp, *orig_prog = prog;
2148 struct bpf_binary_header *header;2153 struct bpf_binary_header *header;
2149 bool tmp_blinded = false;
2150 struct jit_ctx ctx;2154 struct jit_ctx ctx;
2151 unsigned int tmp_idx;2155 unsigned int tmp_idx;
2152 unsigned int image_size;2156 unsigned int image_size;
@@ -2156,20 +2160,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 3 unmodified lines
2156 * the interpreter.2160 * the interpreter.
2157 */2161 */
2158 if (!prog->jit_requested)2162 if (!prog->jit_requested)
2159 return orig_prog;2163 return prog;
2160
2161 /* If constant blinding was enabled and we failed during blinding
2162 * then we must fall back to the interpreter. Otherwise, we save
2163 * the new JITed code.
2164 */
2165 tmp = bpf_jit_blind_constants(prog);
2166
2167 if (IS_ERR(tmp))
2168 return orig_prog;
2169 if (tmp != prog) {
2170 tmp_blinded = true;
2171 prog = tmp;
2172 }
2173 2164
2174 memset(&ctx, 0, sizeof(ctx));2165 memset(&ctx, 0, sizeof(ctx));
2175 ctx.prog = prog;2166 ctx.prog = prog;
@@ -2179,10 +2170,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 3 unmodified lines
2179 * we must fall back to the interpreter2170 * we must fall back to the interpreter
2180 */2171 */
2181 ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);2172 ctx.offsets = kcalloc(prog->len, sizeof(int), GFP_KERNEL);
2182 if (ctx.offsets == NULL) {2173 if (ctx.offsets == NULL)
2183 prog = orig_prog;2174 return prog;
2184 goto out;
2185 }
2186 2175
2187 /* 1) fake pass to find in the length of the JITed code,2176 /* 1) fake pass to find in the length of the JITed code,
2188 * to compute ctx->offsets and other context variables2177 * to compute ctx->offsets and other context variables
@@ -2194,10 +2183,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 5 unmodified lines
2194 * being successful in the second pass, so just fall back2183 * being successful in the second pass, so just fall back
2195 * to the interpreter.2184 * to the interpreter.
2196 */2185 */
2197 if (build_body(&ctx)) {2186 if (build_body(&ctx))
2198 prog = orig_prog;
2199 goto out_off;2187 goto out_off;
2200 }
2201 2188
2202 tmp_idx = ctx.idx;2189 tmp_idx = ctx.idx;
2203 build_prologue(&ctx);2190 build_prologue(&ctx);
@@ -2213,10 +2200,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 9 unmodified lines
2213 ctx.idx += ctx.imm_count;2200 ctx.idx += ctx.imm_count;
2214 if (ctx.imm_count) {2201 if (ctx.imm_count) {
2215 ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);2202 ctx.imms = kcalloc(ctx.imm_count, sizeof(u32), GFP_KERNEL);
2216 if (ctx.imms == NULL) {2203 if (ctx.imms == NULL)
2217 prog = orig_prog;
2218 goto out_off;2204 goto out_off;
2219 }
2220 }2205 }
2221#else2206#else
2222 /* there's nothing about the epilogue on ARMv7 */2207 /* there's nothing about the epilogue on ARMv7 */
@@ -2238,10 +2223,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 15 unmodified lines
2238 /* Not able to allocate memory for the structure then2223 /* Not able to allocate memory for the structure then
2239 * we must fall back to the interpretation2224 * we must fall back to the interpretation
2240 */2225 */
2241 if (header == NULL) {2226 if (header == NULL)
2242 prog = orig_prog;
2243 goto out_imms;2227 goto out_imms;
2244 }
2245 2228
2246 /* 2.) Actual pass to generate final JIT code */2229 /* 2.) Actual pass to generate final JIT code */
2247 ctx.target = (u32 *) image_ptr;2230 ctx.target = (u32 *) image_ptr;
@@ -2278,16 +2261,12 @@ out_imms: · 30 unmodified lines
2278#endif2261#endif
2279out_off:2262out_off:
2280 kfree(ctx.offsets);2263 kfree(ctx.offsets);
2281out:2264
2282 if (tmp_blinded)
2283 bpf_jit_prog_release_other(prog, prog == orig_prog ?
2284 tmp : orig_prog);
2285 return prog;2265 return prog;
2286 2266
2287out_free:2267out_free:
2288 image_ptr = NULL;2268 image_ptr = NULL;
2289 bpf_jit_binary_free(header);2269 bpf_jit_binary_free(header);
2290 prog = orig_prog;
2291 goto out_imms;2270 goto out_imms;
2292}2271}
2293 2272
arch/arm64/lib/insn.c+2−0
@@ -338,6 +338,8 @@ u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr, · 337 unmodified lines
338 long offset;338 long offset;
339 339
340 offset = label_imm_common(pc, addr, SZ_1M);340 offset = label_imm_common(pc, addr, SZ_1M);
341 if (offset >= SZ_1M)
342 return AARCH64_BREAK_FAULT;
341 343
342 insn = aarch64_insn_get_bcond_value();344 insn = aarch64_insn_get_bcond_value();
343 345
arch/arm64/net/bpf_jit_comp.c+31−72
@@ -18,7 +18,6 @@ · 17 unmodified lines
18 18
19#include <asm/asm-extable.h>19#include <asm/asm-extable.h>
20#include <asm/byteorder.h>20#include <asm/byteorder.h>
21#include <asm/cacheflush.h>
22#include <asm/cpufeature.h>21#include <asm/cpufeature.h>
23#include <asm/debug-monitors.h>22#include <asm/debug-monitors.h>
24#include <asm/insn.h>23#include <asm/insn.h>
@@ -35,8 +34,8 @@ · 10 unmodified lines
35#define ARENA_VM_START (MAX_BPF_JIT_REG + 5)34#define ARENA_VM_START (MAX_BPF_JIT_REG + 5)
36 35
37#define check_imm(bits, imm) do { \36#define check_imm(bits, imm) do { \
38 if ((((imm) > 0) && ((imm) >> (bits))) || \37 if ((((imm) > 0) && ((imm) >> ((bits) - 1))) || \
39 (((imm) < 0) && (~(imm) >> (bits)))) { \38 (((imm) < 0) && (~(imm) >> ((bits) - 1)))) { \
40 pr_info("[%2d] imm=%d(0x%x) out of range\n", \39 pr_info("[%2d] imm=%d(0x%x) out of range\n", \
41 i, imm, imm); \40 i, imm, imm); \
42 return -EINVAL; \41 return -EINVAL; \
@@ -1198,8 +1197,8 @@ static int add_exception_handler(const struct bpf_insn *insn, · 1155 unmodified lines
1198 * >0 - successfully JITed a 16-byte eBPF instruction.1197 * >0 - successfully JITed a 16-byte eBPF instruction.
1199 * <0 - failed to JIT.1198 * <0 - failed to JIT.
1200 */1199 */
1201static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,1200static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn *insn,
1202 bool extra_pass)1201 struct jit_ctx *ctx, bool extra_pass)
1203{1202{
1204 const u8 code = insn->code;1203 const u8 code = insn->code;
1205 u8 dst = bpf2a64[insn->dst_reg];1204 u8 dst = bpf2a64[insn->dst_reg];
@@ -1224,6 +1223,9 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, · 18 unmodified lines
1224 int ret;1223 int ret;
1225 bool sign_extend;1224 bool sign_extend;
1226 1225
1226 if (bpf_insn_is_indirect_target(env, ctx->prog, i))
1227 emit_bti(A64_BTI_J, ctx);
1228
1227 switch (code) {1229 switch (code) {
1228 /* dst = src */1230 /* dst = src */
1229 case BPF_ALU | BPF_MOV | BPF_X:1231 case BPF_ALU | BPF_MOV | BPF_X:
@@ -1899,7 +1901,7 @@ emit_cond_jmp: · 669 unmodified lines
1899 return 0;1901 return 0;
1900}1902}
1901 1903
1902static int build_body(struct jit_ctx *ctx, bool extra_pass)1904static int build_body(struct bpf_verifier_env *env, struct jit_ctx *ctx, bool extra_pass)
1903{1905{
1904 const struct bpf_prog *prog = ctx->prog;1906 const struct bpf_prog *prog = ctx->prog;
1905 int i;1907 int i;
@@ -1918,7 +1920,7 @@ static int build_body(struct jit_ctx *ctx, bool extra_pass) · 12 unmodified lines
1918 int ret;1920 int ret;
1919 1921
1920 ctx->offset[i] = ctx->idx;1922 ctx->offset[i] = ctx->idx;
1921 ret = build_insn(insn, ctx, extra_pass);1923 ret = build_insn(env, insn, ctx, extra_pass);
1922 if (ret > 0) {1924 if (ret > 0) {
1923 i++;1925 i++;
1924 ctx->offset[i] = ctx->idx;1926 ctx->offset[i] = ctx->idx;
@@ -1961,11 +1963,6 @@ static int validate_ctx(struct jit_ctx *ctx) · 36 unmodified lines
1961 return 0;1963 return 0;
1962}1964}
1963 1965
1964static inline void bpf_flush_icache(void *start, void *end)
1965{
1966 flush_icache_range((unsigned long)start, (unsigned long)end);
1967}
1968
1969static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)1966static void priv_stack_init_guard(void __percpu *priv_stack_ptr, int alloc_size)
1970{1967{
1971 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;1968 int cpu, underflow_idx = (alloc_size - PRIV_STACK_GUARD_SZ) >> 3;
@@ -2006,17 +2003,15 @@ struct arm64_jit_data { · 34 unmodified lines
2006 struct jit_ctx ctx;2003 struct jit_ctx ctx;
2007};2004};
2008 2005
2009struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)2006struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
2010{2007{
2011 int image_size, prog_size, extable_size, extable_align, extable_offset;2008 int image_size, prog_size, extable_size, extable_align, extable_offset;
2012 struct bpf_prog *tmp, *orig_prog = prog;
2013 struct bpf_binary_header *header;2009 struct bpf_binary_header *header;
2014 struct bpf_binary_header *ro_header = NULL;2010 struct bpf_binary_header *ro_header = NULL;
2015 struct arm64_jit_data *jit_data;2011 struct arm64_jit_data *jit_data;
2016 void __percpu *priv_stack_ptr = NULL;2012 void __percpu *priv_stack_ptr = NULL;
2017 bool was_classic = bpf_prog_was_classic(prog);2013 bool was_classic = bpf_prog_was_classic(prog);
2018 int priv_stack_alloc_sz;2014 int priv_stack_alloc_sz;
2019 bool tmp_blinded = false;
2020 bool extra_pass = false;2015 bool extra_pass = false;
2021 struct jit_ctx ctx;2016 struct jit_ctx ctx;
2022 u8 *image_ptr;2017 u8 *image_ptr;
@@ -2025,26 +2020,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 2 unmodified lines
2025 int exentry_idx;2020 int exentry_idx;
2026 2021
2027 if (!prog->jit_requested)2022 if (!prog->jit_requested)
2028 return orig_prog;2023 return prog;
2029
2030 tmp = bpf_jit_blind_constants(prog);
2031 /* If blinding was requested and we failed during blinding,
2032 * we must fall back to the interpreter.
2033 */
2034 if (IS_ERR(tmp))
2035 return orig_prog;
2036 if (tmp != prog) {
2037 tmp_blinded = true;
2038 prog = tmp;
2039 }
2040 2024
2041 jit_data = prog->aux->jit_data;2025 jit_data = prog->aux->jit_data;
2042 if (!jit_data) {2026 if (!jit_data) {
2043 jit_data = kzalloc_obj(*jit_data);2027 jit_data = kzalloc_obj(*jit_data);
2044 if (!jit_data) {2028 if (!jit_data)
2045 prog = orig_prog;2029 return prog;
2046 goto out;
2047 }
2048 prog->aux->jit_data = jit_data;2030 prog->aux->jit_data = jit_data;
2049 }2031 }
2050 priv_stack_ptr = prog->aux->priv_stack_ptr;2032 priv_stack_ptr = prog->aux->priv_stack_ptr;
@@ -2056,10 +2038,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 5 unmodified lines
2056 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +2038 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 16) +
2057 2 * PRIV_STACK_GUARD_SZ;2039 2 * PRIV_STACK_GUARD_SZ;
2058 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL);2040 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 16, GFP_KERNEL);
2059 if (!priv_stack_ptr) {2041 if (!priv_stack_ptr)
2060 prog = orig_prog;
2061 goto out_priv_stack;2042 goto out_priv_stack;
2062 }
2063 2043
2064 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);2044 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
2065 prog->aux->priv_stack_ptr = priv_stack_ptr;2045 prog->aux->priv_stack_ptr = priv_stack_ptr;
@@ -2079,10 +2059,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 13 unmodified lines
2079 ctx.prog = prog;2059 ctx.prog = prog;
2080 2060
2081 ctx.offset = kvzalloc_objs(int, prog->len + 1);2061 ctx.offset = kvzalloc_objs(int, prog->len + 1);
2082 if (ctx.offset == NULL) {2062 if (ctx.offset == NULL)
2083 prog = orig_prog;
2084 goto out_off;2063 goto out_off;
2085 }
2086 2064
2087 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);2065 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
2088 ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);2066 ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
@@ -2095,15 +2073,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 6 unmodified lines
2095 * BPF line info needs ctx->offset[i] to be the offset of2073 * BPF line info needs ctx->offset[i] to be the offset of
2096 * instruction[i] in jited image, so build prologue first.2074 * instruction[i] in jited image, so build prologue first.
2097 */2075 */
2098 if (build_prologue(&ctx, was_classic)) {2076 if (build_prologue(&ctx, was_classic))
2099 prog = orig_prog;
2100 goto out_off;2077 goto out_off;
2101 }
2102 2078
2103 if (build_body(&ctx, extra_pass)) {2079 if (build_body(env, &ctx, extra_pass))
2104 prog = orig_prog;
2105 goto out_off;2080 goto out_off;
2106 }
2107 2081
2108 ctx.epilogue_offset = ctx.idx;2082 ctx.epilogue_offset = ctx.idx;
2109 build_epilogue(&ctx, was_classic);2083 build_epilogue(&ctx, was_classic);
@@ -2121,10 +2095,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 11 unmodified lines
2121 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,2095 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,
2122 sizeof(u64), &header, &image_ptr,2096 sizeof(u64), &header, &image_ptr,
2123 jit_fill_hole);2097 jit_fill_hole);
2124 if (!ro_header) {2098 if (!ro_header)
2125 prog = orig_prog;
2126 goto out_off;2099 goto out_off;
2127 }
2128 2100
2129 /* Pass 2: Determine jited position and result for each instruction */2101 /* Pass 2: Determine jited position and result for each instruction */
2130 2102
@@ -2152,10 +2124,8 @@ skip_init_ctx: · 21 unmodified lines
2152 /* Dont write body instructions to memory for now */2124 /* Dont write body instructions to memory for now */
2153 ctx.write = false;2125 ctx.write = false;
2154 2126
2155 if (build_body(&ctx, extra_pass)) {2127 if (build_body(env, &ctx, extra_pass))
2156 prog = orig_prog;
2157 goto out_free_hdr;2128 goto out_free_hdr;
2158 }
2159 2129
2160 ctx.epilogue_offset = ctx.idx;2130 ctx.epilogue_offset = ctx.idx;
2161 ctx.exentry_idx = exentry_idx;2131 ctx.exentry_idx = exentry_idx;
@@ -2163,20 +2133,16 @@ skip_init_ctx: · 1 unmodified line
2163 ctx.write = true;2133 ctx.write = true;
2164 2134
2165 /* Pass 3: Adjust jump offset and write final image */2135 /* Pass 3: Adjust jump offset and write final image */
2166 if (build_body(&ctx, extra_pass) ||2136 if (build_body(env, &ctx, extra_pass) ||
2167 WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset)) {2137 WARN_ON_ONCE(ctx.idx != ctx.epilogue_offset))
2168 prog = orig_prog;
2169 goto out_free_hdr;2138 goto out_free_hdr;
2170 }
2171 2139
2172 build_epilogue(&ctx, was_classic);2140 build_epilogue(&ctx, was_classic);
2173 build_plt(&ctx);2141 build_plt(&ctx);
2174 2142
2175 /* Extra pass to validate JITed code. */2143 /* Extra pass to validate JITed code. */
2176 if (validate_ctx(&ctx)) {2144 if (validate_ctx(&ctx))
2177 prog = orig_prog;
2178 goto out_free_hdr;2145 goto out_free_hdr;
2179 }
2180 2146
2181 /* update the real prog size */2147 /* update the real prog size */
2182 prog_size = sizeof(u32) * ctx.idx;2148 prog_size = sizeof(u32) * ctx.idx;
@@ -2193,23 +2159,14 @@ skip_init_ctx: · 10 unmodified lines
2193 if (extra_pass && ctx.idx > jit_data->ctx.idx) {2159 if (extra_pass && ctx.idx > jit_data->ctx.idx) {
2194 pr_err_once("multi-func JIT bug %d > %d\n",2160 pr_err_once("multi-func JIT bug %d > %d\n",
2195 ctx.idx, jit_data->ctx.idx);2161 ctx.idx, jit_data->ctx.idx);
2196 prog->bpf_func = NULL;
2197 prog->jited = 0;
2198 prog->jited_len = 0;
2199 goto out_free_hdr;2162 goto out_free_hdr;
2200 }2163 }
2201 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {2164 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
2202 /* ro_header has been freed */2165 /* ro_header and header has been freed */
2203 ro_header = NULL;2166 ro_header = NULL;
2204 prog = orig_prog;2167 header = NULL;
2205 goto out_off;2168 goto out_free_hdr;
2206 }2169 }
2207 /*
2208 * The instructions have now been copied to the ROX region from
2209 * where they will execute. Now the data cache has to be cleaned to
2210 * the PoU and the I-cache has to be invalidated for the VAs.
2211 */
2212 bpf_flush_icache(ro_header, ctx.ro_image + ctx.idx);
2213 } else {2170 } else {
2214 jit_data->ctx = ctx;2171 jit_data->ctx = ctx;
2215 jit_data->ro_image = ro_image_ptr;2172 jit_data->ro_image = ro_image_ptr;
@@ -2245,13 +2202,15 @@ out_priv_stack: · 29 unmodified lines
2245 kfree(jit_data);2202 kfree(jit_data);
2246 prog->aux->jit_data = NULL;2203 prog->aux->jit_data = NULL;
2247 }2204 }
2248out:2205
2249 if (tmp_blinded)
2250 bpf_jit_prog_release_other(prog, prog == orig_prog ?
2251 tmp : orig_prog);
2252 return prog;2206 return prog;
2253 2207
2254out_free_hdr:2208out_free_hdr:
2209 if (extra_pass) {
2210 prog->bpf_func = NULL;
2211 prog->jited = 0;
2212 prog->jited_len = 0;
2213 }
2255 if (header) {2214 if (header) {
2256 bpf_arch_text_copy(&ro_header->size, &header->size,2215 bpf_arch_text_copy(&ro_header->size, &header->size,
2257 sizeof(header->size));2216 sizeof(header->size));
arch/loongarch/net/bpf_jit.c+18−43
@@ -1920,45 +1920,28 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 1919 unmodified lines
1920 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;1920 return ret < 0 ? ret : ret * LOONGARCH_INSN_SIZE;
1921}1921}
1922 1922
1923struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)1923struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
1924{1924{
1925 bool tmp_blinded = false, extra_pass = false;1925 bool extra_pass = false;
1926 u8 *image_ptr, *ro_image_ptr;1926 u8 *image_ptr, *ro_image_ptr;
1927 int image_size, prog_size, extable_size;1927 int image_size, prog_size, extable_size;
1928 struct jit_ctx ctx;1928 struct jit_ctx ctx;
1929 struct jit_data *jit_data;1929 struct jit_data *jit_data;
1930 struct bpf_binary_header *header;1930 struct bpf_binary_header *header;
1931 struct bpf_binary_header *ro_header;1931 struct bpf_binary_header *ro_header;
1932 struct bpf_prog *tmp, *orig_prog = prog;
1933 1932
1934 /*1933 /*
1935 * If BPF JIT was not enabled then we must fall back to1934 * If BPF JIT was not enabled then we must fall back to
1936 * the interpreter.1935 * the interpreter.
1937 */1936 */
1938 if (!prog->jit_requested)1937 if (!prog->jit_requested)
1939 return orig_prog;1938 return prog;
1940
1941 tmp = bpf_jit_blind_constants(prog);
1942 /*
1943 * If blinding was requested and we failed during blinding,
1944 * we must fall back to the interpreter. Otherwise, we save
1945 * the new JITed code.
1946 */
1947 if (IS_ERR(tmp))
1948 return orig_prog;
1949
1950 if (tmp != prog) {
1951 tmp_blinded = true;
1952 prog = tmp;
1953 }
1954 1939
1955 jit_data = prog->aux->jit_data;1940 jit_data = prog->aux->jit_data;
1956 if (!jit_data) {1941 if (!jit_data) {
1957 jit_data = kzalloc_obj(*jit_data);1942 jit_data = kzalloc_obj(*jit_data);
1958 if (!jit_data) {1943 if (!jit_data)
1959 prog = orig_prog;1944 return prog;
1960 goto out;
1961 }
1962 prog->aux->jit_data = jit_data;1945 prog->aux->jit_data = jit_data;
1963 }1946 }
1964 if (jit_data->ctx.offset) {1947 if (jit_data->ctx.offset) {
@@ -1978,17 +1961,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 13 unmodified lines
1978 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);1961 ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
1979 1962
1980 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);1963 ctx.offset = kvcalloc(prog->len + 1, sizeof(u32), GFP_KERNEL);
1981 if (ctx.offset == NULL) {1964 if (ctx.offset == NULL)
1982 prog = orig_prog;
1983 goto out_offset;1965 goto out_offset;
1984 }
1985 1966
1986 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */1967 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
1987 build_prologue(&ctx);1968 build_prologue(&ctx);
1988 if (build_body(&ctx, extra_pass)) {1969 if (build_body(&ctx, extra_pass))
1989 prog = orig_prog;
1990 goto out_offset;1970 goto out_offset;
1991 }
1992 ctx.epilogue_offset = ctx.idx;1971 ctx.epilogue_offset = ctx.idx;
1993 build_epilogue(&ctx);1972 build_epilogue(&ctx);
1994 1973
@@ -2004,10 +1983,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 9 unmodified lines
2004 /* Now we know the size of the structure to make */1983 /* Now we know the size of the structure to make */
2005 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),1984 ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
2006 &header, &image_ptr, jit_fill_hole);1985 &header, &image_ptr, jit_fill_hole);
2007 if (!ro_header) {1986 if (!ro_header)
2008 prog = orig_prog;
2009 goto out_offset;1987 goto out_offset;
2010 }
2011 1988
2012 /* 2. Now, the actual pass to generate final JIT code */1989 /* 2. Now, the actual pass to generate final JIT code */
2013 /*1990 /*
@@ -2027,17 +2004,13 @@ skip_init_ctx: · 13 unmodified lines
2027 ctx.num_exentries = 0;2004 ctx.num_exentries = 0;
2028 2005
2029 build_prologue(&ctx);2006 build_prologue(&ctx);
2030 if (build_body(&ctx, extra_pass)) {2007 if (build_body(&ctx, extra_pass))
2031 prog = orig_prog;
2032 goto out_free;2008 goto out_free;
2033 }
2034 build_epilogue(&ctx);2009 build_epilogue(&ctx);
2035 2010
2036 /* 3. Extra pass to validate JITed code */2011 /* 3. Extra pass to validate JITed code */
2037 if (validate_ctx(&ctx)) {2012 if (validate_ctx(&ctx))
2038 prog = orig_prog;
2039 goto out_free;2013 goto out_free;
2040 }
2041 2014
2042 /* And we're done */2015 /* And we're done */
2043 if (bpf_jit_enable > 1)2016 if (bpf_jit_enable > 1)
@@ -2050,9 +2023,9 @@ skip_init_ctx: · 6 unmodified lines
2050 goto out_free;2023 goto out_free;
2051 }2024 }
2052 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {2025 if (WARN_ON(bpf_jit_binary_pack_finalize(ro_header, header))) {
2053 /* ro_header has been freed */2026 /* ro_header and header have been freed */
2054 ro_header = NULL;2027 ro_header = NULL;
2055 prog = orig_prog;2028 header = NULL;
2056 goto out_free;2029 goto out_free;
2057 }2030 }
2058 /*2031 /*
@@ -2084,13 +2057,15 @@ out_offset: · 25 unmodified lines
2084 prog->aux->jit_data = NULL;2057 prog->aux->jit_data = NULL;
2085 }2058 }
2086 2059
2087out:
2088 if (tmp_blinded)
2089 bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog);
2090
2091 return prog;2060 return prog;
2092 2061
2093out_free:2062out_free:
2063 if (extra_pass) {
2064 prog->bpf_func = NULL;
2065 prog->jited = 0;
2066 prog->jited_len = 0;
2067 }
2068
2094 if (header) {2069 if (header) {
2095 bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size));2070 bpf_arch_text_copy(&ro_header->size, &header->size, sizeof(header->size));
2096 bpf_jit_binary_pack_free(ro_header, header);2071 bpf_jit_binary_pack_free(ro_header, header);
arch/mips/net/bpf_jit_comp.c+2−20
@@ -909,12 +909,10 @@ bool bpf_jit_needs_zext(void) · 908 unmodified lines
909 return true;909 return true;
910}910}
911 911
912struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)912struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
913{913{
914 struct bpf_prog *tmp, *orig_prog = prog;
915 struct bpf_binary_header *header = NULL;914 struct bpf_binary_header *header = NULL;
916 struct jit_context ctx;915 struct jit_context ctx;
917 bool tmp_blinded = false;
918 unsigned int tmp_idx;916 unsigned int tmp_idx;
919 unsigned int image_size;917 unsigned int image_size;
920 u8 *image_ptr;918 u8 *image_ptr;
@@ -925,19 +923,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 4 unmodified lines
925 * the interpreter.923 * the interpreter.
926 */924 */
927 if (!prog->jit_requested)925 if (!prog->jit_requested)
928 return orig_prog;926 return prog;
929 /*
930 * If constant blinding was enabled and we failed during blinding
931 * then we must fall back to the interpreter. Otherwise, we save
932 * the new JITed code.
933 */
934 tmp = bpf_jit_blind_constants(prog);
935 if (IS_ERR(tmp))
936 return orig_prog;
937 if (tmp != prog) {
938 tmp_blinded = true;
939 prog = tmp;
940 }
941 927
942 memset(&ctx, 0, sizeof(ctx));928 memset(&ctx, 0, sizeof(ctx));
943 ctx.program = prog;929 ctx.program = prog;
@@ -1025,14 +1011,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 81 unmodified lines
1025 prog->jited_len = image_size;1011 prog->jited_len = image_size;
1026 1012
1027out:1013out:
1028 if (tmp_blinded)
1029 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1030 tmp : orig_prog);
1031 kfree(ctx.descriptors);1014 kfree(ctx.descriptors);
1032 return prog;1015 return prog;
1033 1016
1034out_err:1017out_err:
1035 prog = orig_prog;
1036 if (header)1018 if (header)
1037 bpf_jit_binary_free(header);1019 bpf_jit_binary_free(header);
1038 goto out;1020 goto out;
arch/parisc/net/bpf_jit_core.c+30−45
@@ -41,33 +41,22 @@ bool bpf_jit_needs_zext(void) · 40 unmodified lines
41 return true;41 return true;
42}42}
43 43
44struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)44struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
45{45{
46 unsigned int prog_size = 0, extable_size = 0;46 unsigned int prog_size = 0, extable_size = 0;
47 bool tmp_blinded = false, extra_pass = false;47 bool extra_pass = false;
48 struct bpf_prog *tmp, *orig_prog = prog;
49 int pass = 0, prev_ninsns = 0, prologue_len, i;48 int pass = 0, prev_ninsns = 0, prologue_len, i;
50 struct hppa_jit_data *jit_data;49 struct hppa_jit_data *jit_data;
51 struct hppa_jit_context *ctx;50 struct hppa_jit_context *ctx;
52 51
53 if (!prog->jit_requested)52 if (!prog->jit_requested)
54 return orig_prog;53 return prog;
55
56 tmp = bpf_jit_blind_constants(prog);
57 if (IS_ERR(tmp))
58 return orig_prog;
59 if (tmp != prog) {
60 tmp_blinded = true;
61 prog = tmp;
62 }
63 54
64 jit_data = prog->aux->jit_data;55 jit_data = prog->aux->jit_data;
65 if (!jit_data) {56 if (!jit_data) {
66 jit_data = kzalloc_obj(*jit_data);57 jit_data = kzalloc_obj(*jit_data);
67 if (!jit_data) {58 if (!jit_data)
68 prog = orig_prog;59 return prog;
69 goto out;
70 }
71 prog->aux->jit_data = jit_data;60 prog->aux->jit_data = jit_data;
72 }61 }
73 62
@@ -81,10 +70,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 7 unmodified lines
81 70
82 ctx->prog = prog;71 ctx->prog = prog;
83 ctx->offset = kzalloc_objs(int, prog->len);72 ctx->offset = kzalloc_objs(int, prog->len);
84 if (!ctx->offset) {73 if (!ctx->offset)
85 prog = orig_prog;74 goto out_err;
86 goto out_offset;
87 }
88 for (i = 0; i < prog->len; i++) {75 for (i = 0; i < prog->len; i++) {
89 prev_ninsns += 20;76 prev_ninsns += 20;
90 ctx->offset[i] = prev_ninsns;77 ctx->offset[i] = prev_ninsns;
@@ -93,10 +80,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 2 unmodified lines
93 for (i = 0; i < NR_JIT_ITERATIONS; i++) {80 for (i = 0; i < NR_JIT_ITERATIONS; i++) {
94 pass++;81 pass++;
95 ctx->ninsns = 0;82 ctx->ninsns = 0;
96 if (build_body(ctx, extra_pass, ctx->offset)) {83 if (build_body(ctx, extra_pass, ctx->offset))
97 prog = orig_prog;84 goto out_err;
98 goto out_offset;
99 }
100 ctx->body_len = ctx->ninsns;85 ctx->body_len = ctx->ninsns;
101 bpf_jit_build_prologue(ctx);86 bpf_jit_build_prologue(ctx);
102 ctx->prologue_len = ctx->ninsns - ctx->body_len;87 ctx->prologue_len = ctx->ninsns - ctx->body_len;
@@ -116,10 +101,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 13 unmodified lines
116 &jit_data->image,101 &jit_data->image,
117 sizeof(long),102 sizeof(long),
118 bpf_fill_ill_insns);103 bpf_fill_ill_insns);
119 if (!jit_data->header) {104 if (!jit_data->header)
120 prog = orig_prog;105 goto out_err;
121 goto out_offset;
122 }
123 106
124 ctx->insns = (u32 *)jit_data->image;107 ctx->insns = (u32 *)jit_data->image;
125 /*108 /*
@@ -134,8 +117,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 8 unmodified lines
134 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);117 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
135 if (jit_data->header)118 if (jit_data->header)
136 bpf_jit_binary_free(jit_data->header);119 bpf_jit_binary_free(jit_data->header);
137 prog = orig_prog;120 goto out_err;
138 goto out_offset;
139 }121 }
140 122
141 if (extable_size)123 if (extable_size)
@@ -148,8 +130,7 @@ skip_init_ctx: · 6 unmodified lines
148 bpf_jit_build_prologue(ctx);130 bpf_jit_build_prologue(ctx);
149 if (build_body(ctx, extra_pass, NULL)) {131 if (build_body(ctx, extra_pass, NULL)) {
150 bpf_jit_binary_free(jit_data->header);132 bpf_jit_binary_free(jit_data->header);
151 prog = orig_prog;133 goto out_err;
152 goto out_offset;
153 }134 }
154 bpf_jit_build_epilogue(ctx);135 bpf_jit_build_epilogue(ctx);
155 136
@@ -160,20 +141,19 @@ skip_init_ctx: · 4 unmodified lines
160 { extern int machine_restart(char *); machine_restart(""); }141 { extern int machine_restart(char *); machine_restart(""); }
161 }142 }
162 143
144 if (!prog->is_func || extra_pass) {
145 if (bpf_jit_binary_lock_ro(jit_data->header)) {
146 bpf_jit_binary_free(jit_data->header);
147 goto out_err;
148 }
149 bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
150 }
151
163 prog->bpf_func = (void *)ctx->insns;152 prog->bpf_func = (void *)ctx->insns;
164 prog->jited = 1;153 prog->jited = 1;
165 prog->jited_len = prog_size;154 prog->jited_len = prog_size;
166 155
167 bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
168
169 if (!prog->is_func || extra_pass) {156 if (!prog->is_func || extra_pass) {
170 if (bpf_jit_binary_lock_ro(jit_data->header)) {
171 bpf_jit_binary_free(jit_data->header);
172 prog->bpf_func = NULL;
173 prog->jited = 0;
174 prog->jited_len = 0;
175 goto out_offset;
176 }
177 prologue_len = ctx->epilogue_offset - ctx->body_len;157 prologue_len = ctx->epilogue_offset - ctx->body_len;
178 for (i = 0; i < prog->len; i++)158 for (i = 0; i < prog->len; i++)
179 ctx->offset[i] += prologue_len;159 ctx->offset[i] += prologue_len;
@@ -183,14 +163,19 @@ out_offset: · 3 unmodified lines
183 kfree(jit_data);163 kfree(jit_data);
184 prog->aux->jit_data = NULL;164 prog->aux->jit_data = NULL;
185 }165 }
186out:166
187 if (HPPA_JIT_REBOOT)167 if (HPPA_JIT_REBOOT)
188 { extern int machine_restart(char *); machine_restart(""); }168 { extern int machine_restart(char *); machine_restart(""); }
189 169
190 if (tmp_blinded)
191 bpf_jit_prog_release_other(prog, prog == orig_prog ?
192 tmp : orig_prog);
193 return prog;170 return prog;
171
172out_err:
173 if (extra_pass) {
174 prog->bpf_func = NULL;
175 prog->jited = 0;
176 prog->jited_len = 0;
177 }
178 goto out_offset;
194}179}
195 180
196u64 hppa_div64(u64 div, u64 divisor)181u64 hppa_div64(u64 div, u64 divisor)
arch/powerpc/net/bpf_jit_comp.c+27−47
@@ -162,7 +162,7 @@ static void priv_stack_check_guard(void __percpu *priv_stack_ptr, int alloc_size · 161 unmodified lines
162 }162 }
163}163}
164 164
165struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)165struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
166{166{
167 u32 proglen;167 u32 proglen;
168 u32 alloclen;168 u32 alloclen;
@@ -177,9 +177,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 8 unmodified lines
177 void __percpu *priv_stack_ptr = NULL;177 void __percpu *priv_stack_ptr = NULL;
178 struct bpf_binary_header *fhdr = NULL;178 struct bpf_binary_header *fhdr = NULL;
179 struct bpf_binary_header *hdr = NULL;179 struct bpf_binary_header *hdr = NULL;
180 struct bpf_prog *org_fp = fp;
181 struct bpf_prog *tmp_fp = NULL;
182 bool bpf_blinded = false;
183 bool extra_pass = false;180 bool extra_pass = false;
184 u8 *fimage = NULL;181 u8 *fimage = NULL;
185 u32 *fcode_base = NULL;182 u32 *fcode_base = NULL;
@@ -187,24 +184,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 1 unmodified line
187 u32 fixup_len;184 u32 fixup_len;
188 185
189 if (!fp->jit_requested)186 if (!fp->jit_requested)
190 return org_fp;187 return fp;
191
192 tmp_fp = bpf_jit_blind_constants(org_fp);
193 if (IS_ERR(tmp_fp))
194 return org_fp;
195
196 if (tmp_fp != org_fp) {
197 bpf_blinded = true;
198 fp = tmp_fp;
199 }
200 188
201 jit_data = fp->aux->jit_data;189 jit_data = fp->aux->jit_data;
202 if (!jit_data) {190 if (!jit_data) {
203 jit_data = kzalloc_obj(*jit_data);191 jit_data = kzalloc_obj(*jit_data);
204 if (!jit_data) {192 if (!jit_data)
205 fp = org_fp;193 return fp;
206 goto out;
207 }
208 fp->aux->jit_data = jit_data;194 fp->aux->jit_data = jit_data;
209 }195 }
210 196
@@ -219,10 +205,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 8 unmodified lines
219 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) +205 priv_stack_alloc_size = round_up(fp->aux->stack_depth, 16) +
220 2 * PRIV_STACK_GUARD_SZ;206 2 * PRIV_STACK_GUARD_SZ;
221 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL);207 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_size, 16, GFP_KERNEL);
222 if (!priv_stack_ptr) {208 if (!priv_stack_ptr)
223 fp = org_fp;
224 goto out_priv_stack;209 goto out_priv_stack;
225 }
226 210
227 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size);211 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_size);
228 fp->aux->priv_stack_ptr = priv_stack_ptr;212 fp->aux->priv_stack_ptr = priv_stack_ptr;
@@ -249,10 +233,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 20 unmodified lines
249 }233 }
250 234
251 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);235 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL);
252 if (addrs == NULL) {236 if (addrs == NULL)
253 fp = org_fp;237 goto out_err;
254 goto out_addrs;
255 }
256 238
257 memset(&cgctx, 0, sizeof(struct codegen_context));239 memset(&cgctx, 0, sizeof(struct codegen_context));
258 bpf_jit_init_reg_mapping(&cgctx);240 bpf_jit_init_reg_mapping(&cgctx);
@@ -279,11 +261,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 20 unmodified lines
279 }261 }
280 262
281 /* Scouting faux-generate pass 0 */263 /* Scouting faux-generate pass 0 */
282 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {264 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
283 /* We hit something illegal or unsupported. */265 /* We hit something illegal or unsupported. */
284 fp = org_fp;266 goto out_err;
285 goto out_addrs;
286 }
287 267
288 /*268 /*
289 * If we have seen a tail call, we need a second pass.269 * If we have seen a tail call, we need a second pass.
@@ -294,10 +274,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 4 unmodified lines
294 */274 */
295 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {275 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) {
296 cgctx.idx = 0;276 cgctx.idx = 0;
297 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) {277 if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false))
298 fp = org_fp;278 goto out_err;
299 goto out_addrs;
300 }
301 }279 }
302 280
303 bpf_jit_realloc_regs(&cgctx);281 bpf_jit_realloc_regs(&cgctx);
@@ -318,10 +296,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 14 unmodified lines
318 296
319 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,297 fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
320 bpf_jit_fill_ill_insns);298 bpf_jit_fill_ill_insns);
321 if (!fhdr) {299 if (!fhdr)
322 fp = org_fp;300 goto out_err;
323 goto out_addrs;
324 }
325 301
326 if (extable_len)302 if (extable_len)
327 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;303 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len;
@@ -340,8 +316,7 @@ skip_init_ctx: · 12 unmodified lines
340 extra_pass)) {316 extra_pass)) {
341 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));317 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size));
342 bpf_jit_binary_pack_free(fhdr, hdr);318 bpf_jit_binary_pack_free(fhdr, hdr);
343 fp = org_fp;319 goto out_err;
344 goto out_addrs;
345 }320 }
346 bpf_jit_build_epilogue(code_base, &cgctx);321 bpf_jit_build_epilogue(code_base, &cgctx);
347 322
@@ -363,15 +338,16 @@ skip_init_ctx: · 15 unmodified lines
363 ((u64 *)image)[1] = local_paca->kernel_toc;338 ((u64 *)image)[1] = local_paca->kernel_toc;
364#endif339#endif
365 340
341 if (!fp->is_func || extra_pass) {
342 if (bpf_jit_binary_pack_finalize(fhdr, hdr))
343 goto out_err;
344 }
345
366 fp->bpf_func = (void *)fimage;346 fp->bpf_func = (void *)fimage;
367 fp->jited = 1;347 fp->jited = 1;
368 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE;348 fp->jited_len = cgctx.idx * 4 + FUNCTION_DESCR_SIZE;
369 349
370 if (!fp->is_func || extra_pass) {350 if (!fp->is_func || extra_pass) {
371 if (bpf_jit_binary_pack_finalize(fhdr, hdr)) {
372 fp = org_fp;
373 goto out_addrs;
374 }
375 bpf_prog_fill_jited_linfo(fp, addrs);351 bpf_prog_fill_jited_linfo(fp, addrs);
376 /*352 /*
377 * On ABI V1, executable code starts after the function353 * On ABI V1, executable code starts after the function
@@ -398,11 +374,15 @@ out_priv_stack: · 20 unmodified lines
398 jit_data->hdr = hdr;374 jit_data->hdr = hdr;
399 }375 }
400 376
401out:
402 if (bpf_blinded)
403 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp);
404
405 return fp;377 return fp;
378
379out_err:
380 if (extra_pass) {
381 fp->bpf_func = NULL;
382 fp->jited = 0;
383 fp->jited_len = 0;
384 }
385 goto out_addrs;
406}386}
407 387
408/*388/*
arch/riscv/net/bpf_jit.h+0−6
@@ -11,7 +11,6 @@ · 10 unmodified lines
11 11
12#include <linux/bpf.h>12#include <linux/bpf.h>
13#include <linux/filter.h>13#include <linux/filter.h>
14#include <asm/cacheflush.h>
15 14
16/* verify runtime detection extension status */15/* verify runtime detection extension status */
17#define rv_ext_enabled(ext) \16#define rv_ext_enabled(ext) \
@@ -105,11 +104,6 @@ static inline void bpf_fill_ill_insns(void *area, unsigned int size) · 87 unmodified lines
105 memset(area, 0, size);104 memset(area, 0, size);
106}105}
107 106
108static inline void bpf_flush_icache(void *start, void *end)
109{
110 flush_icache_range((unsigned long)start, (unsigned long)end);
111}
112
113/* Emit a 4-byte riscv instruction. */107/* Emit a 4-byte riscv instruction. */
114static inline void emit(const u32 insn, struct rv_jit_context *ctx)108static inline void emit(const u32 insn, struct rv_jit_context *ctx)
115{109{
arch/riscv/net/bpf_jit_core.c+23−47
@@ -41,32 +41,22 @@ bool bpf_jit_needs_zext(void) · 40 unmodified lines
41 return true;41 return true;
42}42}
43 43
44struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)44struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
45{45{
46 unsigned int prog_size = 0, extable_size = 0;46 unsigned int prog_size = 0, extable_size = 0;
47 bool tmp_blinded = false, extra_pass = false;47 bool extra_pass = false;
48 struct bpf_prog *tmp, *orig_prog = prog;
49 int pass = 0, prev_ninsns = 0, i;48 int pass = 0, prev_ninsns = 0, i;
50 struct rv_jit_data *jit_data;49 struct rv_jit_data *jit_data;
51 struct rv_jit_context *ctx;50 struct rv_jit_context *ctx;
52 51
53 if (!prog->jit_requested)52 if (!prog->jit_requested)
54 return orig_prog;53 return prog;
55
56 tmp = bpf_jit_blind_constants(prog);
57 if (IS_ERR(tmp))
58 return orig_prog;
59 if (tmp != prog) {
60 tmp_blinded = true;
61 prog = tmp;
62 }
63 54
64 jit_data = prog->aux->jit_data;55 jit_data = prog->aux->jit_data;
65 if (!jit_data) {56 if (!jit_data) {
66 jit_data = kzalloc_obj(*jit_data);57 jit_data = kzalloc_obj(*jit_data);
67 if (!jit_data) {58 if (!jit_data) {
68 prog = orig_prog;59 return prog;
69 goto out;
70 }60 }
71 prog->aux->jit_data = jit_data;61 prog->aux->jit_data = jit_data;
72 }62 }
@@ -83,15 +73,11 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 10 unmodified lines
83 ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);73 ctx->user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
84 ctx->prog = prog;74 ctx->prog = prog;
85 ctx->offset = kzalloc_objs(int, prog->len);75 ctx->offset = kzalloc_objs(int, prog->len);
86 if (!ctx->offset) {76 if (!ctx->offset)
87 prog = orig_prog;
88 goto out_offset;77 goto out_offset;
89 }
90 78
91 if (build_body(ctx, extra_pass, NULL)) {79 if (build_body(ctx, extra_pass, NULL))
92 prog = orig_prog;
93 goto out_offset;80 goto out_offset;
94 }
95 81
96 for (i = 0; i < prog->len; i++) {82 for (i = 0; i < prog->len; i++) {
97 prev_ninsns += 32;83 prev_ninsns += 32;
@@ -105,10 +91,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 7 unmodified lines
105 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));91 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
106 ctx->prologue_len = ctx->ninsns;92 ctx->prologue_len = ctx->ninsns;
107 93
108 if (build_body(ctx, extra_pass, ctx->offset)) {94 if (build_body(ctx, extra_pass, ctx->offset))
109 prog = orig_prog;
110 goto out_offset;95 goto out_offset;
111 }
112 96
113 ctx->epilogue_offset = ctx->ninsns;97 ctx->epilogue_offset = ctx->ninsns;
114 bpf_jit_build_epilogue(ctx);98 bpf_jit_build_epilogue(ctx);
@@ -126,10 +110,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 11 unmodified lines
126 &jit_data->ro_image, sizeof(u32),110 &jit_data->ro_image, sizeof(u32),
127 &jit_data->header, &jit_data->image,111 &jit_data->header, &jit_data->image,
128 bpf_fill_ill_insns);112 bpf_fill_ill_insns);
129 if (!jit_data->ro_header) {113 if (!jit_data->ro_header)
130 prog = orig_prog;
131 goto out_offset;114 goto out_offset;
132 }
133 115
134 /*116 /*
135 * Use the image(RW) for writing the JITed instructions. But also save117 * Use the image(RW) for writing the JITed instructions. But also save
@@ -150,7 +132,6 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 14 unmodified lines
150 132
151 if (i == NR_JIT_ITERATIONS) {133 if (i == NR_JIT_ITERATIONS) {
152 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);134 pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
153 prog = orig_prog;
154 goto out_free_hdr;135 goto out_free_hdr;
155 }136 }
156 137
@@ -163,33 +144,27 @@ skip_init_ctx: · 6 unmodified lines
163 ctx->nexentries = 0;144 ctx->nexentries = 0;
164 145
165 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));146 bpf_jit_build_prologue(ctx, bpf_is_subprog(prog));
166 if (build_body(ctx, extra_pass, NULL)) {147 if (build_body(ctx, extra_pass, NULL))
167 prog = orig_prog;
168 goto out_free_hdr;148 goto out_free_hdr;
169 }
170 bpf_jit_build_epilogue(ctx);149 bpf_jit_build_epilogue(ctx);
171 150
172 if (bpf_jit_enable > 1)151 if (bpf_jit_enable > 1)
173 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);152 bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
174 153
175 prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset();
176 prog->jited = 1;
177 prog->jited_len = prog_size - cfi_get_offset();
178
179 if (!prog->is_func || extra_pass) {154 if (!prog->is_func || extra_pass) {
180 if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) {155 if (WARN_ON(bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header))) {
181 /* ro_header has been freed */156 /* ro_header has been freed */
182 jit_data->ro_header = NULL;157 jit_data->ro_header = NULL;
183 prog = orig_prog;158 jit_data->header = NULL;
184 goto out_offset;159 goto out_free_hdr;
185 }160 }
186 /*161 }
187 * The instructions have now been copied to the ROX region from162
188 * where they will execute.163 prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset();
189 * Write any modified data cache blocks out to memory and164 prog->jited = 1;
190 * invalidate the corresponding blocks in the instruction cache.165 prog->jited_len = prog_size - cfi_get_offset();
191 */166
192 bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);167 if (!prog->is_func || extra_pass) {
193 for (i = 0; i < prog->len; i++)168 for (i = 0; i < prog->len; i++)
194 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);169 ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
195 bpf_prog_fill_jited_linfo(prog, ctx->offset);170 bpf_prog_fill_jited_linfo(prog, ctx->offset);
@@ -198,14 +173,15 @@ out_offset: · 2 unmodified lines
198 kfree(jit_data);173 kfree(jit_data);
199 prog->aux->jit_data = NULL;174 prog->aux->jit_data = NULL;
200 }175 }
201out:
202 176
203 if (tmp_blinded)
204 bpf_jit_prog_release_other(prog, prog == orig_prog ?
205 tmp : orig_prog);
206 return prog;177 return prog;
207 178
208out_free_hdr:179out_free_hdr:
180 if (extra_pass) {
181 prog->bpf_func = NULL;
182 prog->jited = 0;
183 prog->jited_len = 0;
184 }
209 if (jit_data->header) {185 if (jit_data->header) {
210 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,186 bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
211 sizeof(jit_data->header->size));187 sizeof(jit_data->header->size));
arch/s390/net/bpf_jit_comp.c+21−40
@@ -2312,38 +2312,22 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit, · 2311 unmodified lines
2312/*2312/*
2313 * Compile eBPF program "fp"2313 * Compile eBPF program "fp"
2314 */2314 */
2315struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)2315struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *fp)
2316{2316{
2317 struct bpf_prog *tmp, *orig_fp = fp;
2318 struct bpf_binary_header *header;2317 struct bpf_binary_header *header;
2319 struct s390_jit_data *jit_data;2318 struct s390_jit_data *jit_data;
2320 bool tmp_blinded = false;
2321 bool extra_pass = false;2319 bool extra_pass = false;
2322 struct bpf_jit jit;2320 struct bpf_jit jit;
2323 int pass;2321 int pass;
2324 2322
2325 if (!fp->jit_requested)2323 if (!fp->jit_requested)
2326 return orig_fp;2324 return fp;
2327
2328 tmp = bpf_jit_blind_constants(fp);
2329 /*
2330 * If blinding was requested and we failed during blinding,
2331 * we must fall back to the interpreter.
2332 */
2333 if (IS_ERR(tmp))
2334 return orig_fp;
2335 if (tmp != fp) {
2336 tmp_blinded = true;
2337 fp = tmp;
2338 }
2339 2325
2340 jit_data = fp->aux->jit_data;2326 jit_data = fp->aux->jit_data;
2341 if (!jit_data) {2327 if (!jit_data) {
2342 jit_data = kzalloc_obj(*jit_data);2328 jit_data = kzalloc_obj(*jit_data);
2343 if (!jit_data) {2329 if (!jit_data)
2344 fp = orig_fp;2330 return fp;
2345 goto out;
2346 }
2347 fp->aux->jit_data = jit_data;2331 fp->aux->jit_data = jit_data;
2348 }2332 }
2349 if (jit_data->ctx.addrs) {2333 if (jit_data->ctx.addrs) {
@@ -2356,34 +2340,27 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) · 6 unmodified lines
2356 2340
2357 memset(&jit, 0, sizeof(jit));2341 memset(&jit, 0, sizeof(jit));
2358 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);2342 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
2359 if (jit.addrs == NULL) {2343 if (jit.addrs == NULL)
2360 fp = orig_fp;2344 goto out_err;
2361 goto free_addrs;
2362 }
2363 /*2345 /*
2364 * Three initial passes:2346 * Three initial passes:
2365 * - 1/2: Determine clobbered registers2347 * - 1/2: Determine clobbered registers
2366 * - 3: Calculate program size and addrs array2348 * - 3: Calculate program size and addrs array
2367 */2349 */
2368 for (pass = 1; pass <= 3; pass++) {2350 for (pass = 1; pass <= 3; pass++) {
2369 if (bpf_jit_prog(&jit, fp, extra_pass)) {2351 if (bpf_jit_prog(&jit, fp, extra_pass))
2370 fp = orig_fp;2352 goto out_err;
2371 goto free_addrs;
2372 }
2373 }2353 }
2374 /*2354 /*
2375 * Final pass: Allocate and generate program2355 * Final pass: Allocate and generate program
2376 */2356 */
2377 header = bpf_jit_alloc(&jit, fp);2357 header = bpf_jit_alloc(&jit, fp);
2378 if (!header) {2358 if (!header)
2379 fp = orig_fp;2359 goto out_err;
2380 goto free_addrs;
2381 }
2382skip_init_ctx:2360skip_init_ctx:
2383 if (bpf_jit_prog(&jit, fp, extra_pass)) {2361 if (bpf_jit_prog(&jit, fp, extra_pass)) {
2384 bpf_jit_binary_free(header);2362 bpf_jit_binary_free(header);
2385 fp = orig_fp;2363 goto out_err;
2386 goto free_addrs;
2387 }2364 }
2388 if (bpf_jit_enable > 1) {2365 if (bpf_jit_enable > 1) {
2389 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);2366 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
@@ -2392,8 +2369,7 @@ skip_init_ctx: · 2 unmodified lines
2392 if (!fp->is_func || extra_pass) {2369 if (!fp->is_func || extra_pass) {
2393 if (bpf_jit_binary_lock_ro(header)) {2370 if (bpf_jit_binary_lock_ro(header)) {
2394 bpf_jit_binary_free(header);2371 bpf_jit_binary_free(header);
2395 fp = orig_fp;2372 goto out_err;
2396 goto free_addrs;
2397 }2373 }
2398 } else {2374 } else {
2399 jit_data->header = header;2375 jit_data->header = header;
@@ -2411,11 +2387,16 @@ free_addrs: · 11 unmodified lines
2411 kfree(jit_data);2387 kfree(jit_data);
2412 fp->aux->jit_data = NULL;2388 fp->aux->jit_data = NULL;
2413 }2389 }
2414out:2390
2415 if (tmp_blinded)
2416 bpf_jit_prog_release_other(fp, fp == orig_fp ?
2417 tmp : orig_fp);
2418 return fp;2391 return fp;
2392
2393out_err:
2394 if (extra_pass) {
2395 fp->bpf_func = NULL;
2396 fp->jited = 0;
2397 fp->jited_len = 0;
2398 }
2399 goto free_addrs;
2419}2400}
2420 2401
2421bool bpf_jit_supports_kfunc_call(void)2402bool bpf_jit_supports_kfunc_call(void)
arch/sparc/net/bpf_jit_comp_64.c+22−41
@@ -1477,39 +1477,24 @@ struct sparc64_jit_data { · 1476 unmodified lines
1477 struct jit_ctx ctx;1477 struct jit_ctx ctx;
1478};1478};
1479 1479
1480struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)1480struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
1481{1481{
1482 struct bpf_prog *tmp, *orig_prog = prog;
1483 struct sparc64_jit_data *jit_data;1482 struct sparc64_jit_data *jit_data;
1484 struct bpf_binary_header *header;1483 struct bpf_binary_header *header;
1485 u32 prev_image_size, image_size;1484 u32 prev_image_size, image_size;
1486 bool tmp_blinded = false;
1487 bool extra_pass = false;1485 bool extra_pass = false;
1488 struct jit_ctx ctx;1486 struct jit_ctx ctx;
1489 u8 *image_ptr;1487 u8 *image_ptr;
1490 int pass, i;1488 int pass, i;
1491 1489
1492 if (!prog->jit_requested)1490 if (!prog->jit_requested)
1493 return orig_prog;1491 return prog;
1494
1495 tmp = bpf_jit_blind_constants(prog);
1496 /* If blinding was requested and we failed during blinding,
1497 * we must fall back to the interpreter.
1498 */
1499 if (IS_ERR(tmp))
1500 return orig_prog;
1501 if (tmp != prog) {
1502 tmp_blinded = true;
1503 prog = tmp;
1504 }
1505 1492
1506 jit_data = prog->aux->jit_data;1493 jit_data = prog->aux->jit_data;
1507 if (!jit_data) {1494 if (!jit_data) {
1508 jit_data = kzalloc_obj(*jit_data);1495 jit_data = kzalloc_obj(*jit_data);
1509 if (!jit_data) {1496 if (!jit_data)
1510 prog = orig_prog;1497 return prog;
1511 goto out;
1512 }
1513 prog->aux->jit_data = jit_data;1498 prog->aux->jit_data = jit_data;
1514 }1499 }
1515 if (jit_data->ctx.offset) {1500 if (jit_data->ctx.offset) {
@@ -1527,10 +1512,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 11 unmodified lines
1527 ctx.prog = prog;1512 ctx.prog = prog;
1528 1513
1529 ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);1514 ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL);
1530 if (ctx.offset == NULL) {1515 if (ctx.offset == NULL)
1531 prog = orig_prog;1516 goto out_err;
1532 goto out_off;
1533 }
1534 1517
1535 /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook1518 /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook
1536 * the offset array so that we converge faster.1519 * the offset array so that we converge faster.
@@ -1543,10 +1526,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 6 unmodified lines
1543 ctx.idx = 0;1526 ctx.idx = 0;
1544 1527
1545 build_prologue(&ctx);1528 build_prologue(&ctx);
1546 if (build_body(&ctx)) {1529 if (build_body(&ctx))
1547 prog = orig_prog;1530 goto out_err;
1548 goto out_off;
1549 }
1550 build_epilogue(&ctx);1531 build_epilogue(&ctx);
1551 1532
1552 if (bpf_jit_enable > 1)1533 if (bpf_jit_enable > 1)
@@ -1569,10 +1550,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 16 unmodified lines
1569 image_size = sizeof(u32) * ctx.idx;1550 image_size = sizeof(u32) * ctx.idx;
1570 header = bpf_jit_binary_alloc(image_size, &image_ptr,1551 header = bpf_jit_binary_alloc(image_size, &image_ptr,
1571 sizeof(u32), jit_fill_hole);1552 sizeof(u32), jit_fill_hole);
1572 if (header == NULL) {1553 if (header == NULL)
1573 prog = orig_prog;1554 goto out_err;
1574 goto out_off;
1575 }
1576 1555
1577 ctx.image = (u32 *)image_ptr;1556 ctx.image = (u32 *)image_ptr;
1578skip_init_ctx:1557skip_init_ctx:
@@ -1582,8 +1561,7 @@ skip_init_ctx: · 3 unmodified lines
1582 1561
1583 if (build_body(&ctx)) {1562 if (build_body(&ctx)) {
1584 bpf_jit_binary_free(header);1563 bpf_jit_binary_free(header);
1585 prog = orig_prog;1564 goto out_err;
1586 goto out_off;
1587 }1565 }
1588 1566
1589 build_epilogue(&ctx);1567 build_epilogue(&ctx);
@@ -1592,8 +1570,7 @@ skip_init_ctx: · 2 unmodified lines
1592 pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",1570 pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n",
1593 prev_image_size, ctx.idx * 4);1571 prev_image_size, ctx.idx * 4);
1594 bpf_jit_binary_free(header);1572 bpf_jit_binary_free(header);
1595 prog = orig_prog;1573 goto out_err;
1596 goto out_off;
1597 }1574 }
1598 1575
1599 if (bpf_jit_enable > 1)1576 if (bpf_jit_enable > 1)
@@ -1604,8 +1581,7 @@ skip_init_ctx: · 4 unmodified lines
1604 if (!prog->is_func || extra_pass) {1581 if (!prog->is_func || extra_pass) {
1605 if (bpf_jit_binary_lock_ro(header)) {1582 if (bpf_jit_binary_lock_ro(header)) {
1606 bpf_jit_binary_free(header);1583 bpf_jit_binary_free(header);
1607 prog = orig_prog;1584 goto out_err;
1608 goto out_off;
1609 }1585 }
1610 } else {1586 } else {
1611 jit_data->ctx = ctx;1587 jit_data->ctx = ctx;
@@ -1624,9 +1600,14 @@ out_off: · 12 unmodified lines
1624 kfree(jit_data);1600 kfree(jit_data);
1625 prog->aux->jit_data = NULL;1601 prog->aux->jit_data = NULL;
1626 }1602 }
1627out:1603
1628 if (tmp_blinded)
1629 bpf_jit_prog_release_other(prog, prog == orig_prog ?
1630 tmp : orig_prog);
1631 return prog;1604 return prog;
1605
1606out_err:
1607 if (extra_pass) {
1608 prog->bpf_func = NULL;
1609 prog->jited = 0;
1610 prog->jited_len = 0;
1611 }
1612 goto out_off;
1632}1613}
arch/x86/net/bpf_jit_comp.c+23−50
@@ -58,8 +58,8 @@ static u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len) · 57 unmodified lines
58#define EMIT_ENDBR() EMIT(gen_endbr(), 4)58#define EMIT_ENDBR() EMIT(gen_endbr(), 4)
59#define EMIT_ENDBR_POISON() EMIT(gen_endbr_poison(), 4)59#define EMIT_ENDBR_POISON() EMIT(gen_endbr_poison(), 4)
60#else60#else
61#define EMIT_ENDBR()61#define EMIT_ENDBR() do { } while (0)
62#define EMIT_ENDBR_POISON()62#define EMIT_ENDBR_POISON() do { } while (0)
63#endif63#endif
64 64
65static bool is_imm8(int value)65static bool is_imm8(int value)
@@ -1649,8 +1649,8 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, · 1583 unmodified lines
1649 return 0;1649 return 0;
1650}1650}
1651 1651
1652static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,1652static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *addrs, u8 *image,
1653 int oldproglen, struct jit_context *ctx, bool jmp_padding)1653 u8 *rw_image, int oldproglen, struct jit_context *ctx, bool jmp_padding)
1654{1654{
1655 bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;1655 bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
1656 struct bpf_insn *insn = bpf_prog->insnsi;1656 struct bpf_insn *insn = bpf_prog->insnsi;
@@ -1663,7 +1663,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 6 unmodified lines
1663 void __percpu *priv_stack_ptr;1663 void __percpu *priv_stack_ptr;
1664 int i, excnt = 0;1664 int i, excnt = 0;
1665 int ilen, proglen = 0;1665 int ilen, proglen = 0;
1666 u8 *prog = temp;1666 u8 *ip, *prog = temp;
1667 u32 stack_depth;1667 u32 stack_depth;
1668 int err;1668 int err;
1669 1669
@@ -1734,6 +1734,11 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image · 64 unmodified lines
1734 dst_reg = X86_REG_R9;1734 dst_reg = X86_REG_R9;
1735 }1735 }
1736 1736
1737 if (bpf_insn_is_indirect_target(env, bpf_prog, i - 1))
1738 EMIT_ENDBR();
1739
1740 ip = image + addrs[i - 1] + (prog - temp);
1741
1737 switch (insn->code) {1742 switch (insn->code) {
1738 /* ALU */1743 /* ALU */
1739 case BPF_ALU | BPF_ADD | BPF_X:1744 case BPF_ALU | BPF_ADD | BPF_X:
@@ -2440,8 +2445,6 @@ populate_extable: · 700 unmodified lines
2440 2445
2441 /* call */2446 /* call */
2442 case BPF_JMP | BPF_CALL: {2447 case BPF_JMP | BPF_CALL: {
2443 u8 *ip = image + addrs[i - 1];
2444
2445 func = (u8 *) __bpf_call_base + imm32;2448 func = (u8 *) __bpf_call_base + imm32;
2446 if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {2449 if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) {
2447 LOAD_TAIL_CALL_CNT_PTR(stack_depth);2450 LOAD_TAIL_CALL_CNT_PTR(stack_depth);
@@ -2465,7 +2468,8 @@ populate_extable: · 17 unmodified lines
2465 if (imm32)2468 if (imm32)
2466 emit_bpf_tail_call_direct(bpf_prog,2469 emit_bpf_tail_call_direct(bpf_prog,
2467 &bpf_prog->aux->poke_tab[imm32 - 1],2470 &bpf_prog->aux->poke_tab[imm32 - 1],
2468 &prog, image + addrs[i - 1],2471 &prog,
2472 ip,
2469 callee_regs_used,2473 callee_regs_used,
2470 stack_depth,2474 stack_depth,
2471 ctx);2475 ctx);
@@ -2474,7 +2478,7 @@ populate_extable: · 2 unmodified lines
2474 &prog,2478 &prog,
2475 callee_regs_used,2479 callee_regs_used,
2476 stack_depth,2480 stack_depth,
2477 image + addrs[i - 1],2481 ip,
2478 ctx);2482 ctx);
2479 break;2483 break;
2480 2484
@@ -2639,7 +2643,7 @@ emit_cond_jmp: /* Convert BPF opcode to x86 */ · 158 unmodified lines
2639 break;2643 break;
2640 2644
2641 case BPF_JMP | BPF_JA | BPF_X:2645 case BPF_JMP | BPF_JA | BPF_X:
2642 emit_indirect_jump(&prog, insn->dst_reg, image + addrs[i - 1]);2646 emit_indirect_jump(&prog, insn->dst_reg, ip);
2643 break;2647 break;
2644 case BPF_JMP | BPF_JA:2648 case BPF_JMP | BPF_JA:
2645 case BPF_JMP32 | BPF_JA:2649 case BPF_JMP32 | BPF_JA:
@@ -2729,8 +2733,6 @@ emit_jmp: · 83 unmodified lines
2729 ctx->cleanup_addr = proglen;2733 ctx->cleanup_addr = proglen;
2730 if (bpf_prog_was_classic(bpf_prog) &&2734 if (bpf_prog_was_classic(bpf_prog) &&
2731 !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) {2735 !ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) {
2732 u8 *ip = image + addrs[i - 1];
2733
2734 if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog))2736 if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog))
2735 return -EINVAL;2737 return -EINVAL;
2736 }2738 }
@@ -3713,17 +3715,15 @@ struct x64_jit_data { · 976 unmodified lines
3713#define MAX_PASSES 203715#define MAX_PASSES 20
3714#define PADDING_PASSES (MAX_PASSES - 5)3716#define PADDING_PASSES (MAX_PASSES - 5)
3715 3717
3716struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)3718struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
3717{3719{
3718 struct bpf_binary_header *rw_header = NULL;3720 struct bpf_binary_header *rw_header = NULL;
3719 struct bpf_binary_header *header = NULL;3721 struct bpf_binary_header *header = NULL;
3720 struct bpf_prog *tmp, *orig_prog = prog;
3721 void __percpu *priv_stack_ptr = NULL;3722 void __percpu *priv_stack_ptr = NULL;
3722 struct x64_jit_data *jit_data;3723 struct x64_jit_data *jit_data;
3723 int priv_stack_alloc_sz;3724 int priv_stack_alloc_sz;
3724 int proglen, oldproglen = 0;3725 int proglen, oldproglen = 0;
3725 struct jit_context ctx = {};3726 struct jit_context ctx = {};
3726 bool tmp_blinded = false;
3727 bool extra_pass = false;3727 bool extra_pass = false;
3728 bool padding = false;3728 bool padding = false;
3729 u8 *rw_image = NULL;3729 u8 *rw_image = NULL;
@@ -3733,27 +3733,13 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 3 unmodified lines
3733 int i;3733 int i;
3734 3734
3735 if (!prog->jit_requested)3735 if (!prog->jit_requested)
3736 return orig_prog;3736 return prog;
3737
3738 tmp = bpf_jit_blind_constants(prog);
3739 /*
3740 * If blinding was requested and we failed during blinding,
3741 * we must fall back to the interpreter.
3742 */
3743 if (IS_ERR(tmp))
3744 return orig_prog;
3745 if (tmp != prog) {
3746 tmp_blinded = true;
3747 prog = tmp;
3748 }
3749 3737
3750 jit_data = prog->aux->jit_data;3738 jit_data = prog->aux->jit_data;
3751 if (!jit_data) {3739 if (!jit_data) {
3752 jit_data = kzalloc_obj(*jit_data);3740 jit_data = kzalloc_obj(*jit_data);
3753 if (!jit_data) {3741 if (!jit_data)
3754 prog = orig_prog;3742 return prog;
3755 goto out;
3756 }
3757 prog->aux->jit_data = jit_data;3743 prog->aux->jit_data = jit_data;
3758 }3744 }
3759 priv_stack_ptr = prog->aux->priv_stack_ptr;3745 priv_stack_ptr = prog->aux->priv_stack_ptr;
@@ -3765,10 +3751,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 5 unmodified lines
3765 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +3751 priv_stack_alloc_sz = round_up(prog->aux->stack_depth, 8) +
3766 2 * PRIV_STACK_GUARD_SZ;3752 2 * PRIV_STACK_GUARD_SZ;
3767 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL);3753 priv_stack_ptr = __alloc_percpu_gfp(priv_stack_alloc_sz, 8, GFP_KERNEL);
3768 if (!priv_stack_ptr) {3754 if (!priv_stack_ptr)
3769 prog = orig_prog;
3770 goto out_priv_stack;3755 goto out_priv_stack;
3771 }
3772 3756
3773 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);3757 priv_stack_init_guard(priv_stack_ptr, priv_stack_alloc_sz);
3774 prog->aux->priv_stack_ptr = priv_stack_ptr;3758 prog->aux->priv_stack_ptr = priv_stack_ptr;
@@ -3786,10 +3770,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) · 11 unmodified lines
3786 goto skip_init_addrs;3770 goto skip_init_addrs;
3787 }3771 }
3788 addrs = kvmalloc_objs(*addrs, prog->len + 1);3772 addrs = kvmalloc_objs(*addrs, prog->len + 1);
3789 if (!addrs) {3773 if (!addrs)
3790 prog = orig_prog;
3791 goto out_addrs;3774 goto out_addrs;
3792 }
3793 3775
3794 /*3776 /*
3795 * Before first pass, make a rough estimation of addrs[]3777 * Before first pass, make a rough estimation of addrs[]
@@ -3811,7 +3793,7 @@ skip_init_addrs: · 15 unmodified lines
3811 for (pass = 0; pass < MAX_PASSES || image; pass++) {3793 for (pass = 0; pass < MAX_PASSES || image; pass++) {
3812 if (!padding && pass >= PADDING_PASSES)3794 if (!padding && pass >= PADDING_PASSES)
3813 padding = true;3795 padding = true;
3814 proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding);3796 proglen = do_jit(env, prog, addrs, image, rw_image, oldproglen, &ctx, padding);
3815 if (proglen <= 0) {3797 if (proglen <= 0) {
3816out_image:3798out_image:
3817 image = NULL;3799 image = NULL;
@@ -3820,8 +3802,6 @@ out_image: · 2 unmodified lines
3820 sizeof(rw_header->size));3802 sizeof(rw_header->size));
3821 bpf_jit_binary_pack_free(header, rw_header);3803 bpf_jit_binary_pack_free(header, rw_header);
3822 }3804 }
3823 /* Fall back to interpreter mode */
3824 prog = orig_prog;
3825 if (extra_pass) {3805 if (extra_pass) {
3826 prog->bpf_func = NULL;3806 prog->bpf_func = NULL;
3827 prog->jited = 0;3807 prog->jited = 0;
@@ -3852,10 +3832,8 @@ out_image: · 24 unmodified lines
3852 header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,3832 header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
3853 &image, align, &rw_header, &rw_image,3833 &image, align, &rw_header, &rw_image,
3854 jit_fill_hole);3834 jit_fill_hole);
3855 if (!header) {3835 if (!header)
3856 prog = orig_prog;
3857 goto out_addrs;3836 goto out_addrs;
3858 }
3859 prog->aux->extable = (void *) image + roundup(proglen, align);3837 prog->aux->extable = (void *) image + roundup(proglen, align);
3860 }3838 }
3861 oldproglen = proglen;3839 oldproglen = proglen;
@@ -3908,8 +3886,6 @@ out_image: · 46 unmodified lines
3908 prog->bpf_func = (void *)image + cfi_get_offset();3886 prog->bpf_func = (void *)image + cfi_get_offset();
3909 prog->jited = 1;3887 prog->jited = 1;
3910 prog->jited_len = proglen - cfi_get_offset();3888 prog->jited_len = proglen - cfi_get_offset();
3911 } else {
3912 prog = orig_prog;
3913 }3889 }
3914 3890
3915 if (!image || !prog->is_func || extra_pass) {3891 if (!image || !prog->is_func || extra_pass) {
@@ -3925,10 +3901,7 @@ out_priv_stack: · 9 unmodified lines
3925 kfree(jit_data);3901 kfree(jit_data);
3926 prog->aux->jit_data = NULL;3902 prog->aux->jit_data = NULL;
3927 }3903 }
3928out:3904
3929 if (tmp_blinded)
3930 bpf_jit_prog_release_other(prog, prog == orig_prog ?
3931 tmp : orig_prog);
3932 return prog;3905 return prog;
3933}3906}
3934 3907
arch/x86/net/bpf_jit_comp32.c+5−30
@@ -2518,38 +2518,22 @@ bool bpf_jit_needs_zext(void) · 2517 unmodified lines
2518 return true;2518 return true;
2519}2519}
2520 2520
2521struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)2521struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
2522{2522{
2523 struct bpf_binary_header *header = NULL;2523 struct bpf_binary_header *header = NULL;
2524 struct bpf_prog *tmp, *orig_prog = prog;
2525 int proglen, oldproglen = 0;2524 int proglen, oldproglen = 0;
2526 struct jit_context ctx = {};2525 struct jit_context ctx = {};
2527 bool tmp_blinded = false;
2528 u8 *image = NULL;2526 u8 *image = NULL;
2529 int *addrs;2527 int *addrs;
2530 int pass;2528 int pass;
2531 int i;2529 int i;
2532 2530
2533 if (!prog->jit_requested)2531 if (!prog->jit_requested)
2534 return orig_prog;2532 return prog;
2535
2536 tmp = bpf_jit_blind_constants(prog);
2537 /*
2538 * If blinding was requested and we failed during blinding,
2539 * we must fall back to the interpreter.
2540 */
2541 if (IS_ERR(tmp))
2542 return orig_prog;
2543 if (tmp != prog) {
2544 tmp_blinded = true;
2545 prog = tmp;
2546 }
2547 2533
2548 addrs = kmalloc_objs(*addrs, prog->len);2534 addrs = kmalloc_objs(*addrs, prog->len);
2549 if (!addrs) {2535 if (!addrs)
2550 prog = orig_prog;2536 return prog;
2551 goto out;
2552 }
2553 2537
2554 /*2538 /*
2555 * Before first pass, make a rough estimation of addrs[]2539 * Before first pass, make a rough estimation of addrs[]
@@ -2574,7 +2558,6 @@ out_image: · 18 unmodified lines
2574 image = NULL;2558 image = NULL;
2575 if (header)2559 if (header)
2576 bpf_jit_binary_free(header);2560 bpf_jit_binary_free(header);
2577 prog = orig_prog;
2578 goto out_addrs;2561 goto out_addrs;
2579 }2562 }
2580 if (image) {2563 if (image) {
@@ -2588,10 +2571,8 @@ out_image: · 7 unmodified lines
2588 if (proglen == oldproglen) {2571 if (proglen == oldproglen) {
2589 header = bpf_jit_binary_alloc(proglen, &image,2572 header = bpf_jit_binary_alloc(proglen, &image,
2590 1, jit_fill_hole);2573 1, jit_fill_hole);
2591 if (!header) {2574 if (!header)
2592 prog = orig_prog;
2593 goto out_addrs;2575 goto out_addrs;
2594 }
2595 }2576 }
2596 oldproglen = proglen;2577 oldproglen = proglen;
2597 cond_resched();2578 cond_resched();
@@ -2604,16 +2585,10 @@ out_image: · 6 unmodified lines
2604 prog->bpf_func = (void *)image;2585 prog->bpf_func = (void *)image;
2605 prog->jited = 1;2586 prog->jited = 1;
2606 prog->jited_len = proglen;2587 prog->jited_len = proglen;
2607 } else {
2608 prog = orig_prog;
2609 }2588 }
2610 2589
2611out_addrs:2590out_addrs:
2612 kfree(addrs);2591 kfree(addrs);
2613out:
2614 if (tmp_blinded)
2615 bpf_jit_prog_release_other(prog, prog == orig_prog ?
2616 tmp : orig_prog);
2617 return prog;2592 return prog;
2618}2593}
2619 2594
include/linux/bpf.h+2−0
@@ -1541,6 +1541,8 @@ bool bpf_has_frame_pointer(unsigned long ip); · 1540 unmodified lines
1541int bpf_jit_charge_modmem(u32 size);1541int bpf_jit_charge_modmem(u32 size);
1542void bpf_jit_uncharge_modmem(u32 size);1542void bpf_jit_uncharge_modmem(u32 size);
1543bool bpf_prog_has_trampoline(const struct bpf_prog *prog);1543bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
1544bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
1545 int insn_idx);
1544#else1546#else
1545static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,1547static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
1546 struct bpf_trampoline *tr,1548 struct bpf_trampoline *tr,
include/linux/bpf_verifier.h+5−4
@@ -630,16 +630,17 @@ struct bpf_insn_aux_data { · 629 unmodified lines
630 630
631 /* below fields are initialized once */631 /* below fields are initialized once */
632 unsigned int orig_idx; /* original instruction index */632 unsigned int orig_idx; /* original instruction index */
633 bool jmp_point;633 u32 jmp_point:1;
634 bool prune_point;634 u32 prune_point:1;
635 /* ensure we check state equivalence and save state checkpoint and635 /* ensure we check state equivalence and save state checkpoint and
636 * this instruction, regardless of any heuristics636 * this instruction, regardless of any heuristics
637 */637 */
638 bool force_checkpoint;638 u32 force_checkpoint:1;
639 /* true if instruction is a call to a helper function that639 /* true if instruction is a call to a helper function that
640 * accepts callback function as a parameter.640 * accepts callback function as a parameter.
641 */641 */
642 bool calls_callback;642 u32 calls_callback:1;
643 u32 indirect_target:1; /* if it is an indirect jump target */
643 /*644 /*
644 * CFG strongly connected component this instruction belongs to,645 * CFG strongly connected component this instruction belongs to,
645 * zero if it is a singleton SCC.646 * zero if it is a singleton SCC.
include/linux/filter.h+48−2
@@ -1108,6 +1108,8 @@ sk_filter_reason(struct sock *sk, struct sk_buff *skb) · 1107 unmodified lines
1108 return sk_filter_trim_cap(sk, skb, 1);1108 return sk_filter_trim_cap(sk, skb, 1);
1109}1109}
1110 1110
1111struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
1112 int *err);
1111struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);1113struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
1112void bpf_prog_free(struct bpf_prog *fp);1114void bpf_prog_free(struct bpf_prog *fp);
1113 1115
@@ -1153,7 +1155,7 @@ u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); · 39 unmodified lines
1153 ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \1155 ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
1154 (void *)__bpf_call_base)1156 (void *)__bpf_call_base)
1155 1157
1156struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);1158struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
1157void bpf_jit_compile(struct bpf_prog *prog);1159void bpf_jit_compile(struct bpf_prog *prog);
1158bool bpf_jit_needs_zext(void);1160bool bpf_jit_needs_zext(void);
1159bool bpf_jit_inlines_helper_call(s32 imm);1161bool bpf_jit_inlines_helper_call(s32 imm);
@@ -1184,6 +1186,31 @@ static inline bool bpf_dump_raw_ok(const struct cred *cred) · 24 unmodified lines
1184 1186
1185struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,1187struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
1186 const struct bpf_insn *patch, u32 len);1188 const struct bpf_insn *patch, u32 len);
1189
1190#ifdef CONFIG_BPF_SYSCALL
1191struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
1192 const struct bpf_insn *patch, u32 len);
1193struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env);
1194void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
1195 struct bpf_insn_aux_data *orig_insn_aux);
1196#else
1197static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
1198 const struct bpf_insn *patch, u32 len)
1199{
1200 return ERR_PTR(-ENOTSUPP);
1201}
1202
1203static inline struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
1204{
1205 return NULL;
1206}
1207
1208static inline void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
1209 struct bpf_insn_aux_data *orig_insn_aux)
1210{
1211}
1212#endif /* CONFIG_BPF_SYSCALL */
1213
1187int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);1214int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt);
1188 1215
1189static inline bool xdp_return_frame_no_direct(void)1216static inline bool xdp_return_frame_no_direct(void)
@@ -1310,9 +1337,14 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog, · 120 unmodified lines
1310 1337
1311const char *bpf_jit_get_prog_name(struct bpf_prog *prog);1338const char *bpf_jit_get_prog_name(struct bpf_prog *prog);
1312 1339
1313struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);1340struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog);
1314void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);1341void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
1315 1342
1343static inline bool bpf_prog_need_blind(const struct bpf_prog *prog)
1344{
1345 return prog->blinding_requested && !prog->blinded;
1346}
1347
1316static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,1348static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
1317 u32 pass, void *image)1349 u32 pass, void *image)
1318{1350{
@@ -1451,6 +1483,20 @@ static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp) · 132 unmodified lines
1451{1483{
1452}1484}
1453 1485
1486static inline bool bpf_prog_need_blind(const struct bpf_prog *prog)
1487{
1488 return false;
1489}
1490
1491static inline
1492struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
1493{
1494 return prog;
1495}
1496
1497static inline void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other)
1498{
1499}
1454#endif /* CONFIG_BPF_JIT */1500#endif /* CONFIG_BPF_JIT */
1455 1501
1456void bpf_prog_kallsyms_del_all(struct bpf_prog *fp);1502void bpf_prog_kallsyms_del_all(struct bpf_prog *fp);
kernel/bpf/arena.c+20−3
@@ -341,6 +341,16 @@ static void arena_vm_open(struct vm_area_struct *vma) · 340 unmodified lines
341 refcount_inc(&vml->mmap_count);341 refcount_inc(&vml->mmap_count);
342}342}
343 343
344static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr)
345{
346 return -EINVAL;
347}
348
349static int arena_vm_mremap(struct vm_area_struct *vma)
350{
351 return -EINVAL;
352}
353
344static void arena_vm_close(struct vm_area_struct *vma)354static void arena_vm_close(struct vm_area_struct *vma)
345{355{
346 struct bpf_map *map = vma->vm_file->private_data;356 struct bpf_map *map = vma->vm_file->private_data;
@@ -417,6 +427,8 @@ out_unlock_sigsegv: · 70 unmodified lines
417 427
418static const struct vm_operations_struct arena_vm_ops = {428static const struct vm_operations_struct arena_vm_ops = {
419 .open = arena_vm_open,429 .open = arena_vm_open,
430 .may_split = arena_vm_may_split,
431 .mremap = arena_vm_mremap,
420 .close = arena_vm_close,432 .close = arena_vm_close,
421 .fault = arena_vm_fault,433 .fault = arena_vm_fault,
422};434};
@@ -486,10 +498,11 @@ static int arena_map_mmap(struct bpf_map *map, struct vm_area_struct *vma) · 63 unmodified lines
486 arena->user_vm_end = vma->vm_end;498 arena->user_vm_end = vma->vm_end;
487 /*499 /*
488 * bpf_map_mmap() checks that it's being mmaped as VM_SHARED and500 * bpf_map_mmap() checks that it's being mmaped as VM_SHARED and
489 * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid501 * clears VM_MAYEXEC. Set VM_DONTEXPAND to avoid potential change
490 * potential change of user_vm_start.502 * of user_vm_start. Set VM_DONTCOPY to prevent arena VMA from
503 * being copied into the child process on fork.
491 */504 */
492 vm_flags_set(vma, VM_DONTEXPAND);505 vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY);
493 vma->vm_ops = &arena_vm_ops;506 vma->vm_ops = &arena_vm_ops;
494 return 0;507 return 0;
495}508}
@@ -549,6 +562,10 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt · 53 unmodified lines
549 u32 uaddr32;562 u32 uaddr32;
550 int ret, i;563 int ret, i;
551 564
565 if (node_id != NUMA_NO_NODE &&
566 ((unsigned int)node_id >= nr_node_ids || !node_online(node_id)))
567 return 0;
568
552 if (page_cnt > page_cnt_max)569 if (page_cnt > page_cnt_max)
553 return 0;570 return 0;
554 571
kernel/bpf/bpf_struct_ops.c+4−3
@@ -811,9 +811,6 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 810 unmodified lines
811 goto reset_unlock;811 goto reset_unlock;
812 }812 }
813 813
814 /* Poison pointer on error instead of return for backward compatibility */
815 bpf_prog_assoc_struct_ops(prog, &st_map->map);
816
817 link = kzalloc_obj(*link, GFP_USER);814 link = kzalloc_obj(*link, GFP_USER);
818 if (!link) {815 if (!link) {
819 bpf_prog_put(prog);816 bpf_prog_put(prog);
@@ -824,6 +821,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 4 unmodified lines
824 &bpf_struct_ops_link_lops, prog, prog->expected_attach_type);821 &bpf_struct_ops_link_lops, prog, prog->expected_attach_type);
825 *plink++ = &link->link;822 *plink++ = &link->link;
826 823
824 /* Poison pointer on error instead of return for backward compatibility */
825 bpf_prog_assoc_struct_ops(prog, &st_map->map);
826
827 ksym = kzalloc_obj(*ksym, GFP_USER);827 ksym = kzalloc_obj(*ksym, GFP_USER);
828 if (!ksym) {828 if (!ksym) {
829 err = -ENOMEM;829 err = -ENOMEM;
@@ -906,6 +906,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map *map, void *key, · 76 unmodified lines
906reset_unlock:906reset_unlock:
907 bpf_struct_ops_map_free_ksyms(st_map);907 bpf_struct_ops_map_free_ksyms(st_map);
908 bpf_struct_ops_map_free_image(st_map);908 bpf_struct_ops_map_free_image(st_map);
909 bpf_struct_ops_map_dissoc_progs(st_map);
909 bpf_struct_ops_map_put_progs(st_map);910 bpf_struct_ops_map_put_progs(st_map);
910 memset(uvalue, 0, map->value_size);911 memset(uvalue, 0, map->value_size);
911 memset(kvalue, 0, map->value_size);912 memset(kvalue, 0, map->value_size);
kernel/bpf/core.c+99−39
@@ -1491,24 +1491,11 @@ void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) · 1490 unmodified lines
1491 bpf_prog_clone_free(fp_other);1491 bpf_prog_clone_free(fp_other);
1492}1492}
1493 1493
1494static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)1494/*
1495{1495 * Now this function is used only to blind the main prog and must be invoked only when
1496#ifdef CONFIG_BPF_SYSCALL1496 * bpf_prog_need_blind() returns true.
1497 struct bpf_map *map;1497 */
1498 int i;1498struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog)
1499
1500 if (len <= 1)
1501 return;
1502
1503 for (i = 0; i < prog->aux->used_map_cnt; i++) {
1504 map = prog->aux->used_maps[i];
1505 if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
1506 bpf_insn_array_adjust(map, off, len);
1507 }
1508#endif
1509}
1510
1511struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
1512{1499{
1513 struct bpf_insn insn_buff[16], aux[2];1500 struct bpf_insn insn_buff[16], aux[2];
1514 struct bpf_prog *clone, *tmp;1501 struct bpf_prog *clone, *tmp;
@@ -1516,13 +1503,17 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) · 1 unmodified line
1516 struct bpf_insn *insn;1503 struct bpf_insn *insn;
1517 int i, rewritten;1504 int i, rewritten;
1518 1505
1519 if (!prog->blinding_requested || prog->blinded)1506 if (WARN_ON_ONCE(env && env->prog != prog))
1520 return prog;1507 return ERR_PTR(-EINVAL);
1521 1508
1522 clone = bpf_prog_clone_create(prog, GFP_USER);1509 clone = bpf_prog_clone_create(prog, GFP_USER);
1523 if (!clone)1510 if (!clone)
1524 return ERR_PTR(-ENOMEM);1511 return ERR_PTR(-ENOMEM);
1525 1512
1513 /* make sure bpf_patch_insn_data() patches the correct prog */
1514 if (env)
1515 env->prog = clone;
1516
1526 insn_cnt = clone->len;1517 insn_cnt = clone->len;
1527 insn = clone->insnsi;1518 insn = clone->insnsi;
1528 1519
@@ -1550,21 +1541,28 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) · 21 unmodified lines
1550 if (!rewritten)1541 if (!rewritten)
1551 continue;1542 continue;
1552 1543
1553 tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);1544 if (env)
1554 if (IS_ERR(tmp)) {1545 tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
1546 else
1547 tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
1548
1549 if (IS_ERR_OR_NULL(tmp)) {
1550 if (env)
1551 /* restore the original prog */
1552 env->prog = prog;
1555 /* Patching may have repointed aux->prog during1553 /* Patching may have repointed aux->prog during
1556 * realloc from the original one, so we need to1554 * realloc from the original one, so we need to
1557 * fix it up here on error.1555 * fix it up here on error.
1558 */1556 */
1559 bpf_jit_prog_release_other(prog, clone);1557 bpf_jit_prog_release_other(prog, clone);
1560 return tmp;1558 return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM);
1561 }1559 }
1562 1560
1563 clone = tmp;1561 clone = tmp;
1564 insn_delta = rewritten - 1;1562 insn_delta = rewritten - 1;
1565 1563
1566 /* Instructions arrays must be updated using absolute xlated offsets */1564 if (env)
1567 adjust_insn_arrays(clone, prog->aux->subprog_start + i, rewritten);1565 env->prog = clone;
1568 1566
1569 /* Walk new program and skip insns we just inserted. */1567 /* Walk new program and skip insns we just inserted. */
1570 insn = clone->insnsi + i + insn_delta;1568 insn = clone->insnsi + i + insn_delta;
@@ -1575,6 +1573,15 @@ struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) · 4 unmodified lines
1575 clone->blinded = 1;1573 clone->blinded = 1;
1576 return clone;1574 return clone;
1577}1575}
1576
1577bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
1578 int insn_idx)
1579{
1580 if (!env)
1581 return false;
1582 insn_idx += prog->aux->subprog_start;
1583 return env->insn_aux_data[insn_idx].indirect_target;
1584}
1578#endif /* CONFIG_BPF_JIT */1585#endif /* CONFIG_BPF_JIT */
1579 1586
1580/* Base function for offset calculation. Needs to go into .text section,1587/* Base function for offset calculation. Needs to go into .text section,
@@ -2533,18 +2540,55 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) · 952 unmodified lines
2533 return select_interpreter;2540 return select_interpreter;
2534}2541}
2535 2542
2536/**2543static struct bpf_prog *bpf_prog_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
2537 * bpf_prog_select_runtime - select exec runtime for BPF program2544{
2538 * @fp: bpf_prog populated with BPF program2545#ifdef CONFIG_BPF_JIT
2539 * @err: pointer to error variable2546 struct bpf_prog *orig_prog;
2540 *2547 struct bpf_insn_aux_data *orig_insn_aux;
2541 * Try to JIT eBPF program, if JIT is not available, use interpreter.2548
2542 * The BPF program will be executed via bpf_prog_run() function.2549 if (!bpf_prog_need_blind(prog))
2543 *2550 return bpf_int_jit_compile(env, prog);
2544 * Return: the &fp argument along with &err set to 0 for success or2551
2545 * a negative errno code on failure2552 if (env) {
2546 */2553 /*
2547struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)2554 * If env is not NULL, we are called from the end of bpf_check(), at this
2555 * point, only insn_aux_data is used after failure, so it should be restored
2556 * on failure.
2557 */
2558 orig_insn_aux = bpf_dup_insn_aux_data(env);
2559 if (!orig_insn_aux)
2560 return prog;
2561 }
2562
2563 orig_prog = prog;
2564 prog = bpf_jit_blind_constants(env, prog);
2565 /*
2566 * If blinding was requested and we failed during blinding, we must fall
2567 * back to the interpreter.
2568 */
2569 if (IS_ERR(prog))
2570 goto out_restore;
2571
2572 prog = bpf_int_jit_compile(env, prog);
2573 if (prog->jited) {
2574 bpf_jit_prog_release_other(prog, orig_prog);
2575 if (env)
2576 vfree(orig_insn_aux);
2577 return prog;
2578 }
2579
2580 bpf_jit_prog_release_other(orig_prog, prog);
2581
2582out_restore:
2583 prog = orig_prog;
2584 if (env)
2585 bpf_restore_insn_aux_data(env, orig_insn_aux);
2586#endif
2587 return prog;
2588}
2589
2590struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp,
2591 int *err)
2548{2592{
2549 /* In case of BPF to BPF calls, verifier did all the prep2593 /* In case of BPF to BPF calls, verifier did all the prep
2550 * work with regards to JITing, etc.2594 * work with regards to JITing, etc.
@@ -2572,7 +2616,7 @@ struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) · 21 unmodified lines
2572 if (*err)2616 if (*err)
2573 return fp;2617 return fp;
2574 2618
2575 fp = bpf_int_jit_compile(fp);2619 fp = bpf_prog_jit_compile(env, fp);
2576 bpf_prog_jit_attempt_done(fp);2620 bpf_prog_jit_attempt_done(fp);
2577 if (!fp->jited && jit_needed) {2621 if (!fp->jited && jit_needed) {
2578 *err = -ENOTSUPP;2622 *err = -ENOTSUPP;
@@ -2598,6 +2642,22 @@ finalize: · 19 unmodified lines
2598 2642
2599 return fp;2643 return fp;
2600}2644}
2645
2646/**
2647 * bpf_prog_select_runtime - select exec runtime for BPF program
2648 * @fp: bpf_prog populated with BPF program
2649 * @err: pointer to error variable
2650 *
2651 * Try to JIT eBPF program, if JIT is not available, use interpreter.
2652 * The BPF program will be executed via bpf_prog_run() function.
2653 *
2654 * Return: the &fp argument along with &err set to 0 for success or
2655 * a negative errno code on failure
2656 */
2657struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err)
2658{
2659 return __bpf_prog_select_runtime(NULL, fp, err);
2660}
2601EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);2661EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
2602 2662
2603static unsigned int __bpf_prog_ret1(const void *ctx,2663static unsigned int __bpf_prog_ret1(const void *ctx,
@@ -3085,7 +3145,7 @@ const struct bpf_func_proto bpf_tail_call_proto = { · 481 unmodified lines
3085 * It is encouraged to implement bpf_int_jit_compile() instead, so that3145 * It is encouraged to implement bpf_int_jit_compile() instead, so that
3086 * eBPF and implicitly also cBPF can get JITed!3146 * eBPF and implicitly also cBPF can get JITed!
3087 */3147 */
3088struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)3148struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog)
3089{3149{
3090 return prog;3150 return prog;
3091}3151}
kernel/bpf/fixups.c+138−25
@@ -183,6 +183,18 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, · 182 unmodified lines
183 data[i].seen = old_seen;183 data[i].seen = old_seen;
184 data[i].zext_dst = insn_has_def32(insn + i);184 data[i].zext_dst = insn_has_def32(insn + i);
185 }185 }
186
187 /*
188 * The indirect_target flag of the original instruction was moved to the last of the
189 * new instructions by the above memmove and memset, but the indirect jump target is
190 * actually the first instruction, so move it back. This also matches with the behavior
191 * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new
192 * instruction.
193 */
194 if (data[off + cnt - 1].indirect_target) {
195 data[off].indirect_target = 1;
196 data[off + cnt - 1].indirect_target = 0;
197 }
186}198}
187 199
188static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)200static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
@@ -232,8 +244,8 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) · 43 unmodified lines
232 }244 }
233}245}
234 246
235static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,247struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
236 const struct bpf_insn *patch, u32 len)248 const struct bpf_insn *patch, u32 len)
237{249{
238 struct bpf_prog *new_prog;250 struct bpf_prog *new_prog;
239 struct bpf_insn_aux_data *new_data = NULL;251 struct bpf_insn_aux_data *new_data = NULL;
@@ -973,7 +985,47 @@ patch_insn_buf: · 733 unmodified lines
973 return 0;985 return 0;
974}986}
975 987
976int bpf_jit_subprogs(struct bpf_verifier_env *env)988static u32 *bpf_dup_subprog_starts(struct bpf_verifier_env *env)
989{
990 u32 *starts = NULL;
991
992 starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT);
993 if (starts) {
994 for (int i = 0; i < env->subprog_cnt; i++)
995 starts[i] = env->subprog_info[i].start;
996 }
997 return starts;
998}
999
1000static void bpf_restore_subprog_starts(struct bpf_verifier_env *env, u32 *orig_starts)
1001{
1002 for (int i = 0; i < env->subprog_cnt; i++)
1003 env->subprog_info[i].start = orig_starts[i];
1004 /* restore the start of fake 'exit' subprog as well */
1005 env->subprog_info[env->subprog_cnt].start = env->prog->len;
1006}
1007
1008struct bpf_insn_aux_data *bpf_dup_insn_aux_data(struct bpf_verifier_env *env)
1009{
1010 size_t size;
1011 void *new_aux;
1012
1013 size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);
1014 new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT);
1015 if (new_aux)
1016 memcpy(new_aux, env->insn_aux_data, size);
1017 return new_aux;
1018}
1019
1020void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
1021 struct bpf_insn_aux_data *orig_insn_aux)
1022{
1023 /* the expanded elements are zero-filled, so no special handling is required */
1024 vfree(env->insn_aux_data);
1025 env->insn_aux_data = orig_insn_aux;
1026}
1027
1028static int jit_subprogs(struct bpf_verifier_env *env)
977{1029{
978 struct bpf_prog *prog = env->prog, **func, *tmp;1030 struct bpf_prog *prog = env->prog, **func, *tmp;
979 int i, j, subprog_start, subprog_end = 0, len, subprog;1031 int i, j, subprog_start, subprog_end = 0, len, subprog;
@@ -981,10 +1033,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 1 unmodified line
981 struct bpf_insn *insn;1033 struct bpf_insn *insn;
982 void *old_bpf_func;1034 void *old_bpf_func;
983 int err, num_exentries;1035 int err, num_exentries;
984 int old_len, subprog_start_adjustment = 0;
985
986 if (env->subprog_cnt <= 1)
987 return 0;
988 1036
989 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {1037 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
990 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))1038 if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
@@ -1053,10 +1101,11 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 62 unmodified lines
1053 goto out_free;1101 goto out_free;
1054 func[i]->is_func = 1;1102 func[i]->is_func = 1;
1055 func[i]->sleepable = prog->sleepable;1103 func[i]->sleepable = prog->sleepable;
1104 func[i]->blinded = prog->blinded;
1056 func[i]->aux->func_idx = i;1105 func[i]->aux->func_idx = i;
1057 /* Below members will be freed only at prog->aux */1106 /* Below members will be freed only at prog->aux */
1058 func[i]->aux->btf = prog->aux->btf;1107 func[i]->aux->btf = prog->aux->btf;
1059 func[i]->aux->subprog_start = subprog_start + subprog_start_adjustment;1108 func[i]->aux->subprog_start = subprog_start;
1060 func[i]->aux->func_info = prog->aux->func_info;1109 func[i]->aux->func_info = prog->aux->func_info;
1061 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;1110 func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
1062 func[i]->aux->poke_tab = prog->aux->poke_tab;1111 func[i]->aux->poke_tab = prog->aux->poke_tab;
@@ -1110,17 +1159,10 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 47 unmodified lines
1110 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;1159 func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
1111 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;1160 func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
1112 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;1161 func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
1162 func[i]->aux->token = prog->aux->token;
1113 if (!i)1163 if (!i)
1114 func[i]->aux->exception_boundary = env->seen_exception;1164 func[i]->aux->exception_boundary = env->seen_exception;
1115 1165 func[i] = bpf_int_jit_compile(env, func[i]);
1116 /*
1117 * To properly pass the absolute subprog start to jit
1118 * all instruction adjustments should be accumulated
1119 */
1120 old_len = func[i]->len;
1121 func[i] = bpf_int_jit_compile(func[i]);
1122 subprog_start_adjustment += func[i]->len - old_len;
1123
1124 if (!func[i]->jited) {1166 if (!func[i]->jited) {
1125 err = -ENOTSUPP;1167 err = -ENOTSUPP;
1126 goto out_free;1168 goto out_free;
@@ -1164,7 +1206,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 37 unmodified lines
1164 }1206 }
1165 for (i = 0; i < env->subprog_cnt; i++) {1207 for (i = 0; i < env->subprog_cnt; i++) {
1166 old_bpf_func = func[i]->bpf_func;1208 old_bpf_func = func[i]->bpf_func;
1167 tmp = bpf_int_jit_compile(func[i]);1209 tmp = bpf_int_jit_compile(env, func[i]);
1168 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {1210 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
1169 verbose(env, "JIT doesn't support bpf-to-bpf calls\n");1211 verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
1170 err = -ENOTSUPP;1212 err = -ENOTSUPP;
@@ -1246,16 +1288,87 @@ out_free: · 75 unmodified lines
1246 }1288 }
1247 kfree(func);1289 kfree(func);
1248out_undo_insn:1290out_undo_insn:
1291 bpf_prog_jit_attempt_done(prog);
1292 return err;
1293}
1294
1295int bpf_jit_subprogs(struct bpf_verifier_env *env)
1296{
1297 int err, i;
1298 bool blinded = false;
1299 struct bpf_insn *insn;
1300 struct bpf_prog *prog, *orig_prog;
1301 struct bpf_insn_aux_data *orig_insn_aux;
1302 u32 *orig_subprog_starts;
1303
1304 if (env->subprog_cnt <= 1)
1305 return 0;
1306
1307 prog = orig_prog = env->prog;
1308 if (bpf_prog_need_blind(prog)) {
1309 orig_insn_aux = bpf_dup_insn_aux_data(env);
1310 if (!orig_insn_aux) {
1311 err = -ENOMEM;
1312 goto out_cleanup;
1313 }
1314 orig_subprog_starts = bpf_dup_subprog_starts(env);
1315 if (!orig_subprog_starts) {
1316 vfree(orig_insn_aux);
1317 err = -ENOMEM;
1318 goto out_cleanup;
1319 }
1320 prog = bpf_jit_blind_constants(env, prog);
1321 if (IS_ERR(prog)) {
1322 err = -ENOMEM;
1323 prog = orig_prog;
1324 goto out_restore;
1325 }
1326 blinded = true;
1327 }
1328
1329 err = jit_subprogs(env);
1330 if (err)
1331 goto out_jit_err;
1332
1333 if (blinded) {
1334 bpf_jit_prog_release_other(prog, orig_prog);
1335 kvfree(orig_subprog_starts);
1336 vfree(orig_insn_aux);
1337 }
1338
1339 return 0;
1340
1341out_jit_err:
1342 if (blinded) {
1343 bpf_jit_prog_release_other(orig_prog, prog);
1344 /* roll back to the clean original prog */
1345 prog = env->prog = orig_prog;
1346 goto out_restore;
1347 } else {
1348 if (err != -EFAULT) {
1349 /*
1350 * We will fall back to interpreter mode when err is not -EFAULT, before
1351 * that, insn->off and insn->imm should be restored to their original
1352 * values since they were modified by jit_subprogs.
1353 */
1354 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
1355 if (!bpf_pseudo_call(insn))
1356 continue;
1357 insn->off = 0;
1358 insn->imm = env->insn_aux_data[i].call_imm;
1359 }
1360 }
1361 goto out_cleanup;
1362 }
1363
1364out_restore:
1365 bpf_restore_subprog_starts(env, orig_subprog_starts);
1366 bpf_restore_insn_aux_data(env, orig_insn_aux);
1367 kvfree(orig_subprog_starts);
1368out_cleanup:
1249 /* cleanup main prog to be interpreted */1369 /* cleanup main prog to be interpreted */
1250 prog->jit_requested = 0;1370 prog->jit_requested = 0;
1251 prog->blinding_requested = 0;1371 prog->blinding_requested = 0;
1252 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
1253 if (!bpf_pseudo_call(insn))
1254 continue;
1255 insn->off = 0;
1256 insn->imm = env->insn_aux_data[i].call_imm;
1257 }
1258 bpf_prog_jit_attempt_done(prog);
1259 return err;1372 return err;
1260}1373}
1261 1374
kernel/bpf/helpers.c+16−1
@@ -845,7 +845,13 @@ int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args, · 844 unmodified lines
845 data->buf = buffers->buf;845 data->buf = buffers->buf;
846 846
847 for (i = 0; i < fmt_size; i++) {847 for (i = 0; i < fmt_size; i++) {
848 if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) {848 unsigned char c = fmt[i];
849
850 /*
851 * Permit bytes >= 0x80 in plain text so UTF-8 literals can pass
852 * through unchanged, while still rejecting ASCII control bytes.
853 */
854 if (isascii(c) && !isprint(c) && !isspace(c)) {
849 err = -EINVAL;855 err = -EINVAL;
850 goto out;856 goto out;
851 }857 }
@@ -867,6 +873,15 @@ int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args, · 15 unmodified lines
867 * always access fmt[i + 1], in the worst case it will be a 0873 * always access fmt[i + 1], in the worst case it will be a 0
868 */874 */
869 i++;875 i++;
876 c = fmt[i];
877 /*
878 * The format parser below only understands ASCII conversion
879 * specifiers and modifiers, so reject non-ASCII after '%'.
880 */
881 if (!isascii(c)) {
882 err = -EINVAL;
883 goto out;
884 }
870 885
871 /* skip optional "[0 +-][num]" width formatting field */886 /* skip optional "[0 +-][num]" width formatting field */
872 while (fmt[i] == '0' || fmt[i] == '+' || fmt[i] == '-' ||887 while (fmt[i] == '0' || fmt[i] == '+' || fmt[i] == '-' ||
kernel/bpf/liveness.c+62−52
@@ -574,7 +574,7 @@ static int print_instances(struct bpf_verifier_env *env) · 573 unmodified lines
574 *574 *
575 * precise {frame=N, off=V} -- known absolute frame index and byte offset575 * precise {frame=N, off=V} -- known absolute frame index and byte offset
576 * |576 * |
577 * offset-imprecise {frame=N, off=OFF_IMPRECISE}577 * offset-imprecise {frame=N, cnt=0}
578 * | -- known frame identity, unknown offset578 * | -- known frame identity, unknown offset
579 * fully-imprecise {frame=ARG_IMPRECISE, mask=bitmask}579 * fully-imprecise {frame=ARG_IMPRECISE, mask=bitmask}
580 * -- unknown frame identity; .mask is a580 * -- unknown frame identity; .mask is a
@@ -607,8 +607,6 @@ enum arg_track_state { · 26 unmodified lines
607 ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */607 ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */
608};608};
609 609
610#define OFF_IMPRECISE S16_MIN /* arg identity known but offset unknown */
611
612/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */610/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */
613#define MAX_ARG_SPILL_SLOTS 64611#define MAX_ARG_SPILL_SLOTS 64
614 612
@@ -622,28 +620,6 @@ static bool arg_is_fp(const struct arg_track *at) · 7 unmodified lines
622 return at->frame >= 0 || at->frame == ARG_IMPRECISE;620 return at->frame >= 0 || at->frame == ARG_IMPRECISE;
623}621}
624 622
625/*
626 * Clear all tracked callee stack slots overlapping the byte range
627 * [off, off+sz-1] where off is a negative FP-relative offset.
628 */
629static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz)
630{
631 struct arg_track none = { .frame = ARG_NONE };
632
633 if (off == OFF_IMPRECISE) {
634 for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
635 at_stack[i] = none;
636 return;
637 }
638 for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
639 int slot_start = -((i + 1) * 8);
640 int slot_end = slot_start + 8;
641
642 if (slot_start < off + (int)sz && slot_end > off)
643 at_stack[i] = none;
644 }
645}
646
647static void verbose_arg_track(struct bpf_verifier_env *env, struct arg_track *at)623static void verbose_arg_track(struct bpf_verifier_env *env, struct arg_track *at)
648{624{
649 int i;625 int i;
@@ -863,16 +839,13 @@ static void arg_track_alu64(struct arg_track *dst, const struct arg_track *src) · 213 unmodified lines
863 *dst = arg_join_imprecise(*dst, *src);839 *dst = arg_join_imprecise(*dst, *src);
864}840}
865 841
866static s16 arg_add(s16 off, s64 delta)842static bool arg_add(s16 off, s64 delta, s16 *out)
867{843{
868 s64 res;844 s16 d = delta;
869 845
870 if (off == OFF_IMPRECISE)846 if (d != delta)
871 return OFF_IMPRECISE;847 return true;
872 res = (s64)off + delta;848 return check_add_overflow(off, d, out);
873 if (res < S16_MIN + 1 || res > S16_MAX)
874 return OFF_IMPRECISE;
875 return res;
876}849}
877 850
878static void arg_padd(struct arg_track *at, s64 delta)851static void arg_padd(struct arg_track *at, s64 delta)
@@ -882,9 +855,9 @@ static void arg_padd(struct arg_track *at, s64 delta) · 3 unmodified lines
882 if (at->off_cnt == 0)855 if (at->off_cnt == 0)
883 return;856 return;
884 for (i = 0; i < at->off_cnt; i++) {857 for (i = 0; i < at->off_cnt; i++) {
885 s16 new_off = arg_add(at->off[i], delta);858 s16 new_off;
886 859
887 if (new_off == OFF_IMPRECISE) {860 if (arg_add(at->off[i], delta, &new_off)) {
888 at->off_cnt = 0;861 at->off_cnt = 0;
889 return;862 return;
890 }863 }
@@ -899,8 +872,6 @@ static void arg_padd(struct arg_track *at, s64 delta) · 8 unmodified lines
899 */872 */
900static int fp_off_to_slot(s16 off)873static int fp_off_to_slot(s16 off)
901{874{
902 if (off == OFF_IMPRECISE)
903 return -1;
904 if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8))875 if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8))
905 return -1;876 return -1;
906 if (off % 8)877 if (off % 8)
@@ -930,9 +901,11 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, · 23 unmodified lines
930 return imp;901 return imp;
931 902
932 for (i = 0; i < cnt; i++) {903 for (i = 0; i < cnt; i++) {
933 s16 fp_off = arg_add(at_out[reg].off[i], insn->off);904 s16 fp_off, slot;
934 int slot = fp_off_to_slot(fp_off);
935 905
906 if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
907 return imp;
908 slot = fp_off_to_slot(fp_off);
936 if (slot < 0)909 if (slot < 0)
937 return imp;910 return imp;
938 result = __arg_track_join(result, at_stack_out[slot]);911 result = __arg_track_join(result, at_stack_out[slot]);
@@ -968,9 +941,12 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, · 29 unmodified lines
968 return;941 return;
969 }942 }
970 for (i = 0; i < cnt; i++) {943 for (i = 0; i < cnt; i++) {
971 s16 fp_off = arg_add(at_out[reg].off[i], insn->off);944 s16 fp_off;
972 int slot = fp_off_to_slot(fp_off);945 int slot;
973 946
947 if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
948 continue;
949 slot = fp_off_to_slot(fp_off);
974 if (slot < 0)950 if (slot < 0)
975 continue;951 continue;
976 if (cnt == 1)952 if (cnt == 1)
@@ -980,6 +956,32 @@ static void spill_to_stack(struct bpf_insn *insn, struct arg_track *at_out, · 3 unmodified lines
980 }956 }
981}957}
982 958
959/*
960 * Clear all tracked callee stack slots overlapping the byte range
961 * [off, off+sz-1] where off is a negative FP-relative offset.
962 */
963static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt)
964{
965 struct arg_track none = { .frame = ARG_NONE };
966
967 if (cnt == 0) {
968 for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
969 at_stack[i] = __arg_track_join(at_stack[i], none);
970 return;
971 }
972 for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
973 int slot_start = -((i + 1) * 8);
974 int slot_end = slot_start + 8;
975
976 if (slot_start < off + (int)sz && slot_end > off) {
977 if (cnt == 1)
978 at_stack[i] = none;
979 else
980 at_stack[i] = __arg_track_join(at_stack[i], none);
981 }
982 }
983}
984
983/*985/*
984 * Clear stack slots overlapping all possible FP offsets in @reg.986 * Clear stack slots overlapping all possible FP offsets in @reg.
985 */987 */
@@ -990,18 +992,22 @@ static void clear_stack_for_all_offs(struct bpf_insn *insn, · 4 unmodified lines
990 int cnt, i;992 int cnt, i;
991 993
992 if (reg == BPF_REG_FP) {994 if (reg == BPF_REG_FP) {
993 clear_overlapping_stack_slots(at_stack_out, insn->off, sz);995 clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1);
994 return;996 return;
995 }997 }
996 cnt = at_out[reg].off_cnt;998 cnt = at_out[reg].off_cnt;
997 if (cnt == 0) {999 if (cnt == 0) {
998 clear_overlapping_stack_slots(at_stack_out, OFF_IMPRECISE, sz);1000 clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt);
999 return;1001 return;
1000 }1002 }
1001 for (i = 0; i < cnt; i++) {1003 for (i = 0; i < cnt; i++) {
1002 s16 fp_off = arg_add(at_out[reg].off[i], insn->off);1004 s16 fp_off;
1003 1005
1004 clear_overlapping_stack_slots(at_stack_out, fp_off, sz);1006 if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) {
1007 clear_overlapping_stack_slots(at_stack_out, 0, sz, 0);
1008 break;
1009 }
1010 clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt);
1005 }1011 }
1006}1012}
1007 1013
@@ -1042,6 +1048,12 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i · 34 unmodified lines
1042 verbose(env, "\n");1048 verbose(env, "\n");
1043}1049}
1044 1050
1051static bool can_be_local_fp(int depth, int regno, struct arg_track *at)
1052{
1053 return regno == BPF_REG_FP || at->frame == depth ||
1054 (at->frame == ARG_IMPRECISE && (at->mask & BIT(depth)));
1055}
1056
1045/*1057/*
1046 * Pure dataflow transfer function for arg_track state.1058 * Pure dataflow transfer function for arg_track state.
1047 * Updates at_out[] based on how the instruction modifies registers.1059 * Updates at_out[] based on how the instruction modifies registers.
@@ -1111,8 +1123,7 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, · 63 unmodified lines
1111 at_out[r] = none;1123 at_out[r] = none;
1112 } else if (class == BPF_LDX) {1124 } else if (class == BPF_LDX) {
1113 u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));1125 u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1114 bool src_is_local_fp = insn->src_reg == BPF_REG_FP || src->frame == depth ||1126 bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src);
1115 (src->frame == ARG_IMPRECISE && (src->mask & BIT(depth)));
1116 1127
1117 /*1128 /*
1118 * Reload from callee stack: if src is current-frame FP-derived1129 * Reload from callee stack: if src is current-frame FP-derived
@@ -1147,7 +1158,7 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, · 28 unmodified lines
1147 bool dst_is_local_fp;1158 bool dst_is_local_fp;
1148 1159
1149 /* Track spills to current-frame FP-derived callee stack */1160 /* Track spills to current-frame FP-derived callee stack */
1150 dst_is_local_fp = insn->dst_reg == BPF_REG_FP || dst->frame == depth;1161 dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
1151 if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM)1162 if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM)
1152 spill_to_stack(insn, at_out, insn->dst_reg,1163 spill_to_stack(insn, at_out, insn->dst_reg,
1153 at_stack_out, src, sz);1164 at_stack_out, src, sz);
@@ -1166,7 +1177,7 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn, · 12 unmodified lines
1166 }1177 }
1167 } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) {1178 } else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) {
1168 u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));1179 u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1169 bool dst_is_local_fp = insn->dst_reg == BPF_REG_FP || dst->frame == depth;1180 bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
1170 1181
1171 /* BPF_ST to FP-derived dst: clear overlapping stack slots */1182 /* BPF_ST to FP-derived dst: clear overlapping stack slots */
1172 if (dst_is_local_fp)1183 if (dst_is_local_fp)
@@ -1316,8 +1327,7 @@ static int record_load_store_access(struct bpf_verifier_env *env, · 143 unmodified lines
1316 resolved.off_cnt = ptr->off_cnt;1327 resolved.off_cnt = ptr->off_cnt;
1317 resolved.frame = ptr->frame;1328 resolved.frame = ptr->frame;
1318 for (oi = 0; oi < ptr->off_cnt; oi++) {1329 for (oi = 0; oi < ptr->off_cnt; oi++) {
1319 resolved.off[oi] = arg_add(ptr->off[oi], insn->off);1330 if (arg_add(ptr->off[oi], insn->off, &resolved.off[oi])) {
1320 if (resolved.off[oi] == OFF_IMPRECISE) {
1321 resolved.off_cnt = 0;1331 resolved.off_cnt = 0;
1322 break;1332 break;
1323 }1333 }
kernel/bpf/syscall.c+0−4
@@ -3083,10 +3083,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) · 3082 unmodified lines
3083 if (err < 0)3083 if (err < 0)
3084 goto free_used_maps;3084 goto free_used_maps;
3085 3085
3086 prog = bpf_prog_select_runtime(prog, &err);
3087 if (err < 0)
3088 goto free_used_maps;
3089
3090 err = bpf_prog_mark_insn_arrays_ready(prog);3086 err = bpf_prog_mark_insn_arrays_ready(prog);
3091 if (err < 0)3087 if (err < 0)
3092 goto free_used_maps;3088 goto free_used_maps;
kernel/bpf/verifier.c+19−7
@@ -3497,6 +3497,11 @@ static int insn_stack_access_flags(int frameno, int spi) · 3496 unmodified lines
3497 return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;3497 return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
3498}3498}
3499 3499
3500static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
3501{
3502 env->insn_aux_data[idx].indirect_target = true;
3503}
3504
3500#define LR_FRAMENO_BITS 33505#define LR_FRAMENO_BITS 3
3501#define LR_SPI_BITS 63506#define LR_SPI_BITS 6
3502#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)3507#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)
@@ -4544,6 +4549,9 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, · 1041 unmodified lines
4544 int perm_flags;4549 int perm_flags;
4545 const char *reg_name = "";4550 const char *reg_name = "";
4546 4551
4552 if (base_type(reg->type) != PTR_TO_BTF_ID)
4553 goto bad_type;
4554
4547 if (btf_is_kernel(reg->btf)) {4555 if (btf_is_kernel(reg->btf)) {
4548 perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;4556 perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;
4549 4557
@@ -4556,7 +4564,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, · 6 unmodified lines
4556 perm_flags |= MEM_PERCPU;4564 perm_flags |= MEM_PERCPU;
4557 }4565 }
4558 4566
4559 if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags))4567 if (type_flag(reg->type) & ~perm_flags)
4560 goto bad_type;4568 goto bad_type;
4561 4569
4562 /* We need to verify reg->type and reg->btf, before accessing reg->btf */4570 /* We need to verify reg->type and reg->btf, before accessing reg->btf */
@@ -17545,12 +17553,14 @@ static int check_indirect_jump(struct bpf_verifier_env *env, struct bpf_insn *in · 12982 unmodified lines
17545 }17553 }
17546 17554
17547 for (i = 0; i < n - 1; i++) {17555 for (i = 0; i < n - 1; i++) {
17556 mark_indirect_target(env, env->gotox_tmp_buf->items[i]);
17548 other_branch = push_stack(env, env->gotox_tmp_buf->items[i],17557 other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
17549 env->insn_idx, env->cur_state->speculative);17558 env->insn_idx, env->cur_state->speculative);
17550 if (IS_ERR(other_branch))17559 if (IS_ERR(other_branch))
17551 return PTR_ERR(other_branch);17560 return PTR_ERR(other_branch);
17552 }17561 }
17553 env->insn_idx = env->gotox_tmp_buf->items[n-1];17562 env->insn_idx = env->gotox_tmp_buf->items[n-1];
17563 mark_indirect_target(env, env->insn_idx);
17554 return INSN_IDX_UPDATED;17564 return INSN_IDX_UPDATED;
17555}17565}
17556 17566
@@ -20155,6 +20165,14 @@ skip_full_check: · 2598 unmodified lines
20155 20165
20156 adjust_btf_func(env);20166 adjust_btf_func(env);
20157 20167
20168 /* extension progs temporarily inherit the attach_type of their targets
20169 for verification purposes, so set it back to zero before returning
20170 */
20171 if (env->prog->type == BPF_PROG_TYPE_EXT)
20172 env->prog->expected_attach_type = 0;
20173
20174 env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
20175
20158err_release_maps:20176err_release_maps:
20159 if (ret)20177 if (ret)
20160 release_insn_arrays(env);20178 release_insn_arrays(env);
@@ -20166,12 +20184,6 @@ err_release_maps: · 5 unmodified lines
20166 if (!env->prog->aux->used_btfs)20184 if (!env->prog->aux->used_btfs)
20167 release_btfs(env);20185 release_btfs(env);
20168 20186
20169 /* extension progs temporarily inherit the attach_type of their targets
20170 for verification purposes, so set it back to zero before returning
20171 */
20172 if (env->prog->type == BPF_PROG_TYPE_EXT)
20173 env->prog->expected_attach_type = 0;
20174
20175 *prog = env->prog;20187 *prog = env->prog;
20176 20188
20177 module_put(env->attach_btf_mod);20189 module_put(env->attach_btf_mod);
net/core/filter.c+1−1
@@ -503,7 +503,7 @@ static bool convert_bpf_ld_abs(struct sock_filter *fp, struct bpf_insn **insnp) · 502 unmodified lines
503 ((unaligned_ok && offset >= 0) ||503 ((unaligned_ok && offset >= 0) ||
504 (!unaligned_ok && offset >= 0 &&504 (!unaligned_ok && offset >= 0 &&
505 offset + ip_align >= 0 &&505 offset + ip_align >= 0 &&
506 offset + ip_align % size == 0))) {506 (offset + ip_align) % size == 0))) {
507 bool ldx_off_ok = offset <= S16_MAX;507 bool ldx_off_ok = offset <= S16_MAX;
508 508
509 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);509 *insn++ = BPF_MOV64_REG(BPF_REG_TMP, BPF_REG_H);
net/core/sock_map.c+2−2
@@ -530,7 +530,7 @@ static bool sock_map_redirect_allowed(const struct sock *sk) · 529 unmodified lines
530 if (sk_is_tcp(sk))530 if (sk_is_tcp(sk))
531 return sk->sk_state != TCP_LISTEN;531 return sk->sk_state != TCP_LISTEN;
532 else532 else
533 return sk->sk_state == TCP_ESTABLISHED;533 return READ_ONCE(sk->sk_state) == TCP_ESTABLISHED;
534}534}
535 535
536static bool sock_map_sk_is_suitable(const struct sock *sk)536static bool sock_map_sk_is_suitable(const struct sock *sk)
@@ -543,7 +543,7 @@ static bool sock_map_sk_state_allowed(const struct sock *sk) · 6 unmodified lines
543 if (sk_is_tcp(sk))543 if (sk_is_tcp(sk))
544 return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN);544 return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN);
545 if (sk_is_stream_unix(sk))545 if (sk_is_stream_unix(sk))
546 return (1 << sk->sk_state) & TCPF_ESTABLISHED;546 return (1 << READ_ONCE(sk->sk_state)) & TCPF_ESTABLISHED;
547 if (sk_is_vsock(sk) &&547 if (sk_is_vsock(sk) &&
548 (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET))548 (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET))
549 return (1 << sk->sk_state) & TCPF_ESTABLISHED;549 return (1 << sk->sk_state) & TCPF_ESTABLISHED;
net/unix/af_unix.c+5−4
@@ -3735,15 +3735,15 @@ static int bpf_iter_unix_seq_show(struct seq_file *seq, void *v) · 3734 unmodified lines
3735 struct bpf_prog *prog;3735 struct bpf_prog *prog;
3736 struct sock *sk = v;3736 struct sock *sk = v;
3737 uid_t uid;3737 uid_t uid;
3738 bool slow;
3739 int ret;3738 int ret;
3740 3739
3741 if (v == SEQ_START_TOKEN)3740 if (v == SEQ_START_TOKEN)
3742 return 0;3741 return 0;
3743 3742
3744 slow = lock_sock_fast(sk);3743 lock_sock(sk);
3744 unix_state_lock(sk);
3745 3745
3746 if (unlikely(sk_unhashed(sk))) {3746 if (unlikely(sock_flag(sk, SOCK_DEAD))) {
3747 ret = SEQ_SKIP;3747 ret = SEQ_SKIP;
3748 goto unlock;3748 goto unlock;
3749 }3749 }
@@ -3753,7 +3753,8 @@ static int bpf_iter_unix_seq_show(struct seq_file *seq, void *v) · 3 unmodified lines
3753 prog = bpf_iter_get_info(&meta, false);3753 prog = bpf_iter_get_info(&meta, false);
3754 ret = unix_prog_seq_show(prog, &meta, v, uid);3754 ret = unix_prog_seq_show(prog, &meta, v, uid);
3755unlock:3755unlock:
3756 unlock_sock_fast(sk, slow);3756 unix_state_unlock(sk);
3757 release_sock(sk);
3757 return ret;3758 return ret;
3758}3759}
3759 3760
net/unix/unix_bpf.c+3−0
@@ -185,6 +185,9 @@ int unix_stream_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool r · 184 unmodified lines
185 */185 */
186 if (!psock->sk_pair) {186 if (!psock->sk_pair) {
187 sk_pair = unix_peer(sk);187 sk_pair = unix_peer(sk);
188 if (unlikely(!sk_pair))
189 return -EINVAL;
190
188 sock_hold(sk_pair);191 sock_hold(sk_pair);
189 psock->sk_pair = sk_pair;192 psock->sk_pair = sk_pair;
190 }193 }
tools/lib/bpf/libbpf.c+11−10
@@ -5852,11 +5852,12 @@ static int load_module_btfs(struct bpf_object *obj) · 5851 unmodified lines
5852 info.name = ptr_to_u64(name);5852 info.name = ptr_to_u64(name);
5853 info.name_len = sizeof(name);5853 info.name_len = sizeof(name);
5854 5854
5855 btf = NULL;
5855 err = bpf_btf_get_info_by_fd(fd, &info, &len);5856 err = bpf_btf_get_info_by_fd(fd, &info, &len);
5856 if (err) {5857 if (err) {
5857 err = -errno;5858 err = -errno;
5858 pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err));5859 pr_warn("failed to get BTF object #%d info: %s\n", id, errstr(err));
5859 goto err_out;5860 break;
5860 }5861 }
5861 5862
5862 /* ignore non-module BTFs */5863 /* ignore non-module BTFs */
@@ -5870,15 +5871,15 @@ static int load_module_btfs(struct bpf_object *obj) · 7 unmodified lines
5870 if (err) {5871 if (err) {
5871 pr_warn("failed to load module [%s]'s BTF object #%d: %s\n",5872 pr_warn("failed to load module [%s]'s BTF object #%d: %s\n",
5872 name, id, errstr(err));5873 name, id, errstr(err));
5873 goto err_out;5874 break;
5874 }5875 }
5875 5876
5876 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,5877 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5877 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);5878 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5878 if (err)5879 if (err)
5879 goto err_out;5880 break;
5880 5881
5881 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];5882 mod_btf = &obj->btf_modules[obj->btf_module_cnt];
5882 5883
5883 mod_btf->btf = btf;5884 mod_btf->btf = btf;
5884 mod_btf->id = id;5885 mod_btf->id = id;
@@ -5886,16 +5887,16 @@ static int load_module_btfs(struct bpf_object *obj) · 1 unmodified line
5886 mod_btf->name = strdup(name);5887 mod_btf->name = strdup(name);
5887 if (!mod_btf->name) {5888 if (!mod_btf->name) {
5888 err = -ENOMEM;5889 err = -ENOMEM;
5889 goto err_out;5890 break;
5890 }5891 }
5891 continue;5892 obj->btf_module_cnt++;
5893 }
5892 5894
5893err_out:5895 if (err) {
5896 btf__free(btf);
5894 close(fd);5897 close(fd);
5895 return err;
5896 }5898 }
5897 5899 return err;
5898 return 0;
5899}5900}
5900 5901
5901static struct bpf_core_cand_list *5902static struct bpf_core_cand_list *
tools/testing/selftests/bpf/Makefile+1−0
@@ -751,6 +751,7 @@ TRUNNER_EXTRA_SOURCES := test_progs.c \ · 750 unmodified lines
751 btf_helpers.c \751 btf_helpers.c \
752 cap_helpers.c \752 cap_helpers.c \
753 unpriv_helpers.c \753 unpriv_helpers.c \
754 sysctl_helpers.c \
754 netlink_helpers.c \755 netlink_helpers.c \
755 jit_disasm_helpers.c \756 jit_disasm_helpers.c \
756 io_helpers.c \757 io_helpers.c \
tools/testing/selftests/bpf/prog_tests/snprintf.c+2−1
@@ -114,7 +114,8 @@ static void test_snprintf_negative(void) · 113 unmodified lines
114 ASSERT_ERR(load_single_snprintf("%--------"), "invalid specifier 5");114 ASSERT_ERR(load_single_snprintf("%--------"), "invalid specifier 5");
115 ASSERT_ERR(load_single_snprintf("%lc"), "invalid specifier 6");115 ASSERT_ERR(load_single_snprintf("%lc"), "invalid specifier 6");
116 ASSERT_ERR(load_single_snprintf("%llc"), "invalid specifier 7");116 ASSERT_ERR(load_single_snprintf("%llc"), "invalid specifier 7");
117 ASSERT_ERR(load_single_snprintf("\x80"), "non ascii character");117 ASSERT_OK(load_single_snprintf("\x80"), "non ascii plain text");
118 ASSERT_ERR(load_single_snprintf("%\x80"), "non ascii in specifier");
118 ASSERT_ERR(load_single_snprintf("\x1"), "non printable character");119 ASSERT_ERR(load_single_snprintf("\x1"), "non printable character");
119 ASSERT_ERR(load_single_snprintf("%p%"), "invalid specifier 8");120 ASSERT_ERR(load_single_snprintf("%p%"), "invalid specifier 8");
120 ASSERT_ERR(load_single_snprintf("%s%"), "invalid specifier 9");121 ASSERT_ERR(load_single_snprintf("%s%"), "invalid specifier 9");
tools/testing/selftests/bpf/prog_tests/task_local_data.h+10−3
@@ -99,14 +99,20 @@ struct tld_meta_u { · 98 unmodified lines
99 struct tld_metadata metadata[];99 struct tld_metadata metadata[];
100};100};
101 101
102/*
103 * The unused field ensures map_val.start > 0. On the BPF side, __tld_fetch_key()
104 * calculates off by summing map_val.start and tld_key_t.off and treats off == 0
105 * as key not cached.
106 */
102struct tld_data_u {107struct tld_data_u {
103 __u64 start; /* offset of tld_data_u->data in a page */108 __u64 unused;
104 char data[] __attribute__((aligned(8)));109 char data[] __attribute__((aligned(8)));
105};110};
106 111
107struct tld_map_value {112struct tld_map_value {
108 void *data;113 void *data;
109 struct tld_meta_u *meta;114 struct tld_meta_u *meta;
115 __u16 start; /* offset of tld_data_u->data in a page */
110};116};
111 117
112struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak));118struct tld_meta_u * _Atomic tld_meta_p __attribute__((weak));
@@ -182,7 +188,7 @@ static int __tld_init_data_p(int map_fd) · 69 unmodified lines
182 * is a page in BTF.188 * is a page in BTF.
183 */189 */
184 map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data);190 map_val.data = (void *)(TLD_PAGE_MASK & (intptr_t)data);
185 data->start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u);191 map_val.start = (~TLD_PAGE_MASK & (intptr_t)data) + sizeof(struct tld_data_u);
186 map_val.meta = tld_meta_p;192 map_val.meta = tld_meta_p;
187 193
188 err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0);194 err = bpf_map_update_elem(map_fd, &tid_fd, &map_val, 0);
@@ -241,7 +247,8 @@ retry: · 52 unmodified lines
241 * TLD_DYN_DATA_SIZE is allocated for tld_create_key()247 * TLD_DYN_DATA_SIZE is allocated for tld_create_key()
242 */248 */
243 if (dyn_data) {249 if (dyn_data) {
244 if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size)250 if (off + TLD_ROUND_UP(size, 8) > tld_meta_p->size ||
251 tld_meta_p->size > TLD_PAGE_SIZE - sizeof(struct tld_data_u))
245 return (tld_key_t){-E2BIG};252 return (tld_key_t){-E2BIG};
246 } else {253 } else {
247 if (off + TLD_ROUND_UP(size, 8) > TLD_PAGE_SIZE - sizeof(struct tld_data_u))254 if (off + TLD_ROUND_UP(size, 8) > TLD_PAGE_SIZE - sizeof(struct tld_data_u))
tools/testing/selftests/bpf/prog_tests/test_task_local_data.c+87−9
@@ -3,8 +3,14 @@ · 2 unmodified lines
3#include <bpf/btf.h>3#include <bpf/btf.h>
4#include <test_progs.h>4#include <test_progs.h>
5 5
6/*
7 * Only a page is pinned to kernel, so the maximum amount of dynamic data
8 * allowed is page_size - sizeof(struct tld_data_u) - static TLD fields.
9 */
10#define TLD_DYN_DATA_SIZE_MAX (getpagesize() - sizeof(struct tld_data_u) - 8)
11
6#define TLD_FREE_DATA_ON_THREAD_EXIT12#define TLD_FREE_DATA_ON_THREAD_EXIT
7#define TLD_DYN_DATA_SIZE (getpagesize() - 8)13#define TLD_DYN_DATA_SIZE TLD_DYN_DATA_SIZE_MAX
8#include "task_local_data.h"14#include "task_local_data.h"
9 15
10struct test_tld_struct {16struct test_tld_struct {
@@ -24,12 +30,12 @@ TLD_DEFINE_KEY(value0_key, "value0", sizeof(int)); · 13 unmodified lines
24 * sequentially. Users of task local data library should not touch30 * sequentially. Users of task local data library should not touch
25 * library internal.31 * library internal.
26 */32 */
27static void reset_tld(void)33static void reset_tld(__u16 dyn_data_size)
28{34{
29 if (tld_meta_p) {35 if (tld_meta_p) {
30 /* Remove TLDs created by tld_create_key() */36 /* Remove TLDs created by tld_create_key() */
31 tld_meta_p->cnt = 1;37 tld_meta_p->cnt = 1;
32 tld_meta_p->size = TLD_DYN_DATA_SIZE;38 tld_meta_p->size = dyn_data_size + 8;
33 memset(&tld_meta_p->metadata[1], 0,39 memset(&tld_meta_p->metadata[1], 0,
34 (TLD_MAX_DATA_CNT - 1) * sizeof(struct tld_metadata));40 (TLD_MAX_DATA_CNT - 1) * sizeof(struct tld_metadata));
35 }41 }
@@ -127,7 +133,7 @@ static void test_task_local_data_basic(void) · 91 unmodified lines
127 tld_key_t key;133 tld_key_t key;
128 int i, err;134 int i, err;
129 135
130 reset_tld();136 reset_tld(TLD_DYN_DATA_SIZE_MAX);
131 137
132 ASSERT_OK(pthread_mutex_init(&global_mutex, NULL), "pthread_mutex_init");138 ASSERT_OK(pthread_mutex_init(&global_mutex, NULL), "pthread_mutex_init");
133 139
@@ -147,11 +153,13 @@ static void test_task_local_data_basic(void) · 13 unmodified lines
147 153
148 /*154 /*
149 * Shouldn't be able to store data exceed a page. Create a TLD just big155 * Shouldn't be able to store data exceed a page. Create a TLD just big
150 * enough to exceed a page. TLDs already created are int value0, int156 * enough to exceed a page. Data already contains struct tld_data_u,
151 * value1, and struct test_tld_struct value2.157 * value0 and value1 of int type, and value 2 of struct test_tld_struct.
152 */158 */
153 key = tld_create_key("value_not_exist",159 key = tld_create_key("value_not_exist", TLD_PAGE_SIZE + 1 -
154 TLD_PAGE_SIZE - 2 * sizeof(int) - sizeof(struct test_tld_struct) + 1);160 sizeof(struct tld_data_u) -
161 TLD_ROUND_UP(sizeof(int), 8) * 2 -
162 TLD_ROUND_UP(sizeof(struct test_tld_struct), 8));
155 ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key");163 ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key");
156 164
157 key = tld_create_key("value2", sizeof(struct test_tld_struct));165 key = tld_create_key("value2", sizeof(struct test_tld_struct));
@@ -239,7 +247,7 @@ static void test_task_local_data_race(void) · 81 unmodified lines
239 tld_keys[0] = value0_key;247 tld_keys[0] = value0_key;
240 248
241 for (j = 0; j < 100; j++) {249 for (j = 0; j < 100; j++) {
242 reset_tld();250 reset_tld(TLD_DYN_DATA_SIZE_MAX);
243 251
244 for (i = 0; i < TEST_RACE_THREAD_NUM; i++) {252 for (i = 0; i < TEST_RACE_THREAD_NUM; i++) {
245 /*253 /*
@@ -288,10 +296,80 @@ out: · 42 unmodified lines
288 test_task_local_data__destroy(skel);296 test_task_local_data__destroy(skel);
289}297}
290 298
299static void test_task_local_data_dyn_size(__u16 dyn_data_size)
300{
301 LIBBPF_OPTS(bpf_test_run_opts, opts);
302 struct test_task_local_data *skel;
303 int max_keys, i, err, fd, *data;
304 char name[TLD_NAME_LEN];
305 tld_key_t key;
306
307 reset_tld(dyn_data_size);
308
309 skel = test_task_local_data__open_and_load();
310 if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
311 return;
312
313 tld_keys = calloc(TLD_MAX_DATA_CNT, sizeof(tld_key_t));
314 if (!ASSERT_OK_PTR(tld_keys, "calloc tld_keys"))
315 goto out;
316
317 fd = bpf_map__fd(skel->maps.tld_data_map);
318
319 /* Create as many int-sized TLDs as the dynamic data size allows */
320 max_keys = dyn_data_size / TLD_ROUND_UP(sizeof(int), 8);
321 for (i = 0; i < max_keys; i++) {
322 snprintf(name, TLD_NAME_LEN, "value_%d", i);
323 tld_keys[i] = tld_create_key(name, sizeof(int));
324 if (!ASSERT_FALSE(tld_key_is_err(tld_keys[i]), "tld_create_key"))
325 goto out;
326
327 data = tld_get_data(fd, tld_keys[i]);
328 if (!ASSERT_OK_PTR(data, "tld_get_data"))
329 goto out;
330 *data = i;
331 }
332
333 /* The next key should fail with E2BIG */
334 key = tld_create_key("overflow", sizeof(int));
335 ASSERT_EQ(tld_key_err_or_zero(key), -E2BIG, "tld_create_key overflow");
336
337 /* Verify data for value_i do not overlap */
338 for (i = 0; i < max_keys; i++) {
339 data = tld_get_data(fd, tld_keys[i]);
340 if (!ASSERT_OK_PTR(data, "tld_get_data"))
341 goto out;
342
343 ASSERT_EQ(*data, i, "tld_get_data value_i");
344 }
345
346 /* Verify BPF side can still read the static key */
347 data = tld_get_data(fd, value0_key);
348 if (!ASSERT_OK_PTR(data, "tld_get_data value0"))
349 goto out;
350 *data = 0xdeadbeef;
351
352 err = bpf_prog_test_run_opts(bpf_program__fd(skel->progs.task_main), &opts);
353 ASSERT_OK(err, "run task_main");
354 ASSERT_EQ(skel->bss->test_value0, 0xdeadbeef, "tld_get_data value0");
355
356out:
357 if (tld_keys) {
358 free(tld_keys);
359 tld_keys = NULL;
360 }
361 tld_free();
362 test_task_local_data__destroy(skel);
363}
364
291void test_task_local_data(void)365void test_task_local_data(void)
292{366{
293 if (test__start_subtest("task_local_data_basic"))367 if (test__start_subtest("task_local_data_basic"))
294 test_task_local_data_basic();368 test_task_local_data_basic();
295 if (test__start_subtest("task_local_data_race"))369 if (test__start_subtest("task_local_data_race"))
296 test_task_local_data_race();370 test_task_local_data_race();
371 if (test__start_subtest("task_local_data_dyn_size_small"))
372 test_task_local_data_dyn_size(64);
373 if (test__start_subtest("task_local_data_dyn_size_zero"))
374 test_task_local_data_dyn_size(0);
297}375}
tools/testing/selftests/bpf/prog_tests/token.c+83−3
@@ -1,9 +1,7 @@
1// SPDX-License-Identifier: GPL-2.01// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */2/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */
3#define _GNU_SOURCE3#define _GNU_SOURCE
4#include <test_progs.h>
5#include <bpf/btf.h>4#include <bpf/btf.h>
6#include "cap_helpers.h"
7#include <fcntl.h>5#include <fcntl.h>
8#include <sched.h>6#include <sched.h>
9#include <signal.h>7#include <signal.h>
@@ -15,9 +13,17 @@ · 5 unmodified lines
15#include <sys/stat.h>13#include <sys/stat.h>
16#include <sys/syscall.h>14#include <sys/syscall.h>
17#include <sys/un.h>15#include <sys/un.h>
16
17#include "bpf_util.h"
18#include "cap_helpers.h"
19#include "sysctl_helpers.h"
20#include "test_progs.h"
21#include "trace_helpers.h"
22
18#include "priv_map.skel.h"23#include "priv_map.skel.h"
19#include "priv_prog.skel.h"24#include "priv_prog.skel.h"
20#include "dummy_st_ops_success.skel.h"25#include "dummy_st_ops_success.skel.h"
26#include "token_kallsyms.skel.h"
21#include "token_lsm.skel.h"27#include "token_lsm.skel.h"
22#include "priv_freplace_prog.skel.h"28#include "priv_freplace_prog.skel.h"
23 29
@@ -1045,6 +1051,58 @@ err_out: · 1021 unmodified lines
1045 return -EINVAL;1051 return -EINVAL;
1046}1052}
1047 1053
1054static bool kallsyms_has_bpf_func(struct ksyms *ksyms, const char *func_name)
1055{
1056 char name[256];
1057 int i;
1058
1059 for (i = 0; i < ksyms->sym_cnt; i++) {
1060 if (sscanf(ksyms->syms[i].name, "bpf_prog_%*[^_]_%255s", name) == 1 &&
1061 strcmp(name, func_name) == 0)
1062 return true;
1063 }
1064 return false;
1065}
1066
1067static int userns_obj_priv_prog_kallsyms(int mnt_fd, struct token_lsm *lsm_skel)
1068{
1069 const char *func_names[] = { "xdp_main", "token_ksym_subprog" };
1070 LIBBPF_OPTS(bpf_object_open_opts, opts);
1071 struct token_kallsyms *skel;
1072 struct ksyms *ksyms = NULL;
1073 char buf[256];
1074 int i, err;
1075
1076 snprintf(buf, sizeof(buf), "/proc/self/fd/%d", mnt_fd);
1077 opts.bpf_token_path = buf;
1078 skel = token_kallsyms__open_opts(&opts);
1079 if (!ASSERT_OK_PTR(skel, "token_kallsyms__open_opts"))
1080 return -EINVAL;
1081
1082 err = token_kallsyms__load(skel);
1083 if (!ASSERT_OK(err, "token_kallsyms__load"))
1084 goto cleanup;
1085
1086 ksyms = load_kallsyms_local();
1087 if (!ASSERT_OK_PTR(ksyms, "load_kallsyms_local")) {
1088 err = -EINVAL;
1089 goto cleanup;
1090 }
1091
1092 for (i = 0; i < ARRAY_SIZE(func_names); i++) {
1093 if (!ASSERT_TRUE(kallsyms_has_bpf_func(ksyms, func_names[i]),
1094 func_names[i])) {
1095 err = -EINVAL;
1096 break;
1097 }
1098 }
1099
1100cleanup:
1101 free_kallsyms_local(ksyms);
1102 token_kallsyms__destroy(skel);
1103 return err;
1104}
1105
1048#define bit(n) (1ULL << (n))1106#define bit(n) (1ULL << (n))
1049 1107
1050static int userns_bpf_token_info(int mnt_fd, struct token_lsm *lsm_skel)1108static int userns_bpf_token_info(int mnt_fd, struct token_lsm *lsm_skel)
@@ -1082,7 +1140,7 @@ cleanup: · 31 unmodified lines
1082 return err;1140 return err;
1083}1141}
1084 1142
1085void test_token(void)1143void serial_test_token(void)
1086{1144{
1087 if (test__start_subtest("map_token")) {1145 if (test__start_subtest("map_token")) {
1088 struct bpffs_opts opts = {1146 struct bpffs_opts opts = {
@@ -1194,4 +1252,26 @@ void test_token(void) · 105 unmodified lines
1194 1252
1195 subtest_userns(&opts, userns_bpf_token_info);1253 subtest_userns(&opts, userns_bpf_token_info);
1196 }1254 }
1255 if (test__start_subtest("obj_priv_prog_kallsyms")) {
1256 char perf_paranoid_orig[32] = {};
1257 char kptr_restrict_orig[32] = {};
1258 struct bpffs_opts opts = {
1259 .cmds = bit(BPF_BTF_LOAD) | bit(BPF_PROG_LOAD),
1260 .progs = bit(BPF_PROG_TYPE_XDP),
1261 .attachs = ~0ULL,
1262 };
1263
1264 if (sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", perf_paranoid_orig, "0"))
1265 goto cleanup;
1266 if (sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", kptr_restrict_orig, "0"))
1267 goto cleanup;
1268
1269 subtest_userns(&opts, userns_obj_priv_prog_kallsyms);
1270
1271cleanup:
1272 if (perf_paranoid_orig[0])
1273 sysctl_set_or_fail("/proc/sys/kernel/perf_event_paranoid", NULL, perf_paranoid_orig);
1274 if (kptr_restrict_orig[0])
1275 sysctl_set_or_fail("/proc/sys/kernel/kptr_restrict", NULL, kptr_restrict_orig);
1276 }
1197}1277}
tools/testing/selftests/bpf/prog_tests/trace_printk.c+22−6
@@ -6,18 +6,21 @@ · 5 unmodified lines
6#include "trace_printk.lskel.h"6#include "trace_printk.lskel.h"
7 7
8#define SEARCHMSG "testing,testing"8#define SEARCHMSG "testing,testing"
9#define SEARCHMSG_UTF8 "中文,测试"
9 10
10static void trace_pipe_cb(const char *str, void *data)11static void trace_pipe_cb(const char *str, void *data)
11{12{
12 if (strstr(str, SEARCHMSG) != NULL)13 if (strstr(str, SEARCHMSG) != NULL)
13 (*(int *)data)++;14 ((int *)data)[0]++;
15 if (strstr(str, SEARCHMSG_UTF8))
16 ((int *)data)[1]++;
14}17}
15 18
16void serial_test_trace_printk(void)19void serial_test_trace_printk(void)
17{20{
18 struct trace_printk_lskel__bss *bss;21 struct trace_printk_lskel__bss *bss;
19 struct trace_printk_lskel *skel;22 struct trace_printk_lskel *skel;
20 int err = 0, found = 0;23 int err = 0, found[2] = {};
21 24
22 skel = trace_printk_lskel__open();25 skel = trace_printk_lskel__open();
23 if (!ASSERT_OK_PTR(skel, "trace_printk__open"))26 if (!ASSERT_OK_PTR(skel, "trace_printk__open"))
@@ -46,11 +49,24 @@ void serial_test_trace_printk(void) · 22 unmodified lines
46 if (!ASSERT_GT(bss->trace_printk_ret, 0, "bss->trace_printk_ret"))49 if (!ASSERT_GT(bss->trace_printk_ret, 0, "bss->trace_printk_ret"))
47 goto cleanup;50 goto cleanup;
48 51
49 /* verify our search string is in the trace buffer */52 if (!ASSERT_GT(bss->trace_printk_utf8_ran, 0, "bss->trace_printk_utf8_ran"))
50 ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, &found, 1000),53 goto cleanup;
51 "read_trace_pipe_iter");54
55 if (!ASSERT_GT(bss->trace_printk_utf8_ret, 0, "bss->trace_printk_utf8_ret"))
56 goto cleanup;
57
58 if (!ASSERT_LT(bss->trace_printk_invalid_spec_ret, 0,
59 "bss->trace_printk_invalid_spec_ret"))
60 goto cleanup;
61
62 /* verify our search strings are in the trace buffer */
63 ASSERT_OK(read_trace_pipe_iter(trace_pipe_cb, found, 1000),
64 "read_trace_pipe_iter");
65
66 if (!ASSERT_EQ(found[0], bss->trace_printk_ran, "found"))
67 goto cleanup;
52 68
53 if (!ASSERT_EQ(found, bss->trace_printk_ran, "found"))69 if (!ASSERT_EQ(found[1], bss->trace_printk_utf8_ran, "found_utf8"))
54 goto cleanup;70 goto cleanup;
55 71
56cleanup:72cleanup:
tools/testing/selftests/bpf/prog_tests/unpriv_bpf_disabled.c+1−20
@@ -8,6 +8,7 @@ · 7 unmodified lines
8 8
9#include "cap_helpers.h"9#include "cap_helpers.h"
10#include "bpf_util.h"10#include "bpf_util.h"
11#include "sysctl_helpers.h"
11 12
12/* Using CAP_LAST_CAP is risky here, since it can get pulled in from13/* Using CAP_LAST_CAP is risky here, since it can get pulled in from
13 * an old /usr/include/linux/capability.h and be < CAP_BPF; as a result14 * an old /usr/include/linux/capability.h and be < CAP_BPF; as a result
@@ -36,26 +37,6 @@ static void process_perfbuf(void *ctx, int cpu, void *data, __u32 len) · 22 unmodified lines
36 got_perfbuf_val = *(__u32 *)data;37 got_perfbuf_val = *(__u32 *)data;
37}38}
38 39
39static int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val)
40{
41 int ret = 0;
42 FILE *fp;
43
44 fp = fopen(sysctl_path, "r+");
45 if (!fp)
46 return -errno;
47 if (old_val && fscanf(fp, "%s", old_val) <= 0) {
48 ret = -ENOENT;
49 } else if (!old_val || strcmp(old_val, new_val) != 0) {
50 fseek(fp, 0, SEEK_SET);
51 if (fprintf(fp, "%s", new_val) < 0)
52 ret = -errno;
53 }
54 fclose(fp);
55
56 return ret;
57}
58
59static void test_unpriv_bpf_disabled_positive(struct test_unpriv_bpf_disabled *skel,40static void test_unpriv_bpf_disabled_positive(struct test_unpriv_bpf_disabled *skel,
60 __u32 prog_id, int prog_fd, int perf_fd,41 __u32 prog_id, int prog_fd, int perf_fd,
61 char **map_paths, int *map_fds)42 char **map_paths, int *map_fds)
tools/testing/selftests/bpf/progs/bpf_iter_unix.c+10−0
@@ -7,6 +7,13 @@ · 6 unmodified lines
7 7
8char _license[] SEC("license") = "GPL";8char _license[] SEC("license") = "GPL";
9 9
10SEC(".maps") struct {
11 __uint(type, BPF_MAP_TYPE_SOCKMAP);
12 __uint(max_entries, 1);
13 __type(key, __u32);
14 __type(value, __u64);
15} sockmap;
16
10static long sock_i_ino(const struct sock *sk)17static long sock_i_ino(const struct sock *sk)
11{18{
12 const struct socket *sk_socket = sk->sk_socket;19 const struct socket *sk_socket = sk->sk_socket;
@@ -76,5 +83,8 @@ int dump_unix(struct bpf_iter__unix *ctx) · 63 unmodified lines
76 83
77 BPF_SEQ_PRINTF(seq, "\n");84 BPF_SEQ_PRINTF(seq, "\n");
78 85
86 /* Test for deadlock. */
87 bpf_map_update_elem(&sockmap, &(int){0}, sk, 0);
88
79 return 0;89 return 0;
80}90}
tools/testing/selftests/bpf/progs/map_kptr_fail.c+15−0
@@ -385,4 +385,19 @@ int kptr_xchg_possibly_null(struct __sk_buff *ctx) · 384 unmodified lines
385 return 0;385 return 0;
386}386}
387 387
388SEC("?tc")
389__failure __msg("invalid kptr access, R")
390int reject_scalar_store_to_kptr(struct __sk_buff *ctx)
391{
392 struct map_value *v;
393 int key = 0;
394
395 v = bpf_map_lookup_elem(&array_map, &key);
396 if (!v)
397 return 0;
398
399 *(volatile u64 *)&v->unref_ptr = 0xBADC0DE;
400 return 0;
401}
402
388char _license[] SEC("license") = "GPL";403char _license[] SEC("license") = "GPL";
tools/testing/selftests/bpf/progs/task_local_data.bpf.h+3−2
@@ -86,13 +86,14 @@ struct tld_meta_u { · 85 unmodified lines
86};86};
87 87
88struct tld_data_u {88struct tld_data_u {
89 __u64 start; /* offset of tld_data_u->data in a page */89 __u64 unused;
90 char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8)));90 char data[__PAGE_SIZE - sizeof(__u64)] __attribute__((aligned(8)));
91};91};
92 92
93struct tld_map_value {93struct tld_map_value {
94 struct tld_data_u __uptr *data;94 struct tld_data_u __uptr *data;
95 struct tld_meta_u __uptr *meta;95 struct tld_meta_u __uptr *meta;
96 __u16 start; /* offset of tld_data_u->data in a page */
96};97};
97 98
98typedef struct tld_uptr_dummy {99typedef struct tld_uptr_dummy {
@@ -176,7 +177,7 @@ static int __tld_fetch_key(struct tld_object *tld_obj, const char *name, int i_s · 77 unmodified lines
176 if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta)177 if (!tld_obj->data_map || !tld_obj->data_map->data || !tld_obj->data_map->meta)
177 return 0;178 return 0;
178 179
179 start = tld_obj->data_map->data->start;180 start = tld_obj->data_map->start;
180 cnt = tld_obj->data_map->meta->cnt;181 cnt = tld_obj->data_map->meta->cnt;
181 metadata = tld_obj->data_map->meta->metadata;182 metadata = tld_obj->data_map->meta->metadata;
182 183
tools/testing/selftests/bpf/progs/timer_start_deadlock.c+4−4
@@ -27,13 +27,13 @@ static int timer_cb(void *map, int *key, struct elem *value) · 26 unmodified lines
27 return 0;27 return 0;
28}28}
29 29
30SEC("tp_btf/hrtimer_cancel")30SEC("tp_btf/hrtimer_start")
31int BPF_PROG(tp_hrtimer_cancel, struct hrtimer *hrtimer)31int BPF_PROG(tp_hrtimer_start, struct hrtimer *hrtimer, enum hrtimer_mode mode, bool was_armed)
32{32{
33 struct bpf_timer *timer;33 struct bpf_timer *timer;
34 int key = 0;34 int key = 0;
35 35
36 if (!in_timer_start)36 if (!in_timer_start || !was_armed)
37 return 0;37 return 0;
38 38
39 tp_called = 1;39 tp_called = 1;
@@ -60,7 +60,7 @@ int start_timer(void *ctx) · 20 unmodified lines
60 60
61 /*61 /*
62 * call hrtimer_start() twice, so that 2nd call does62 * call hrtimer_start() twice, so that 2nd call does
63 * remove_hrtimer() and trace_hrtimer_cancel() tracepoint.63 * trace_hrtimer_start(was_armed=1) tracepoint.
64 */64 */
65 in_timer_start = 1;65 in_timer_start = 1;
66 bpf_timer_start(timer, 1000000000, 0);66 bpf_timer_start(timer, 1000000000, 0);
tools/testing/selftests/bpf/progs/token_kallsyms.c+19−0
@@ -0,0 +1,19 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
3
4#include "vmlinux.h"
5#include <bpf/bpf_helpers.h>
6
7char _license[] SEC("license") = "GPL";
8
9__weak
10int token_ksym_subprog(void)
11{
12 return 0;
13}
14
15SEC("xdp")
16int xdp_main(struct xdp_md *xdp)
17{
18 return token_ksym_subprog();
19}
tools/testing/selftests/bpf/progs/trace_printk.c+10−0
@@ -10,13 +10,23 @@ char _license[] SEC("license") = "GPL"; · 9 unmodified lines
10 10
11int trace_printk_ret = 0;11int trace_printk_ret = 0;
12int trace_printk_ran = 0;12int trace_printk_ran = 0;
13int trace_printk_invalid_spec_ret = 0;
14int trace_printk_utf8_ret = 0;
15int trace_printk_utf8_ran = 0;
13 16
14const char fmt[] = "Testing,testing %d\n";17const char fmt[] = "Testing,testing %d\n";
18static const char utf8_fmt[] = "中文,测试 %d\n";
19/* Non-ASCII bytes after '%' must still be rejected. */
20static const char invalid_spec_fmt[] = "%\x80\n";
15 21
16SEC("fentry/" SYS_PREFIX "sys_nanosleep")22SEC("fentry/" SYS_PREFIX "sys_nanosleep")
17int sys_enter(void *ctx)23int sys_enter(void *ctx)
18{24{
19 trace_printk_ret = bpf_trace_printk(fmt, sizeof(fmt),25 trace_printk_ret = bpf_trace_printk(fmt, sizeof(fmt),
20 ++trace_printk_ran);26 ++trace_printk_ran);
27 trace_printk_utf8_ret = bpf_trace_printk(utf8_fmt, sizeof(utf8_fmt),
28 ++trace_printk_utf8_ran);
29 trace_printk_invalid_spec_ret = bpf_trace_printk(invalid_spec_fmt,
30 sizeof(invalid_spec_fmt));
21 return 0;31 return 0;
22}32}
tools/testing/selftests/bpf/progs/verifier_live_stack.c+193−0
@@ -2647,3 +2647,196 @@ __naked void spill_join_with_imprecise_off(void) · 2646 unmodified lines
2647 "exit;"2647 "exit;"
2648 ::: __clobber_all);2648 ::: __clobber_all);
2649}2649}
2650
2651/*
2652 * Same as spill_join_with_multi_off but the write is BPF_ST (store
2653 * immediate) instead of BPF_STX. BPF_ST goes through
2654 * clear_stack_for_all_offs() rather than spill_to_stack(), and that
2655 * path also needs to join instead of overwriting.
2656 *
2657 * fp-8 = &fp-24
2658 * fp-16 = &fp-32
2659 * r1 = fp-8 or fp-16 (two offsets from branch)
2660 * *(u64 *)(r1 + 0) = 0 -- BPF_ST with immediate
2661 * r0 = *(u64 *)(r10 - 16) -- fill from fp-16
2662 * r0 = *(u64 *)(r0 + 0) -- deref: should produce use
2663 */
2664SEC("socket")
2665__log_level(2)
2666__failure
2667__msg("15: (7a) *(u64 *)(r1 +0) = 0 fp-8: fp0-24 -> fp0-24|fp0+0 fp-16: fp0-32 -> fp0-32|fp0+0")
2668__msg("17: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-32")
2669__naked void st_imm_join_with_multi_off(void)
2670{
2671 asm volatile (
2672 "*(u64 *)(r10 - 24) = 0;"
2673 "*(u64 *)(r10 - 32) = 0;"
2674 "r1 = r10;"
2675 "r1 += -24;"
2676 "*(u64 *)(r10 - 8) = r1;"
2677 "r1 = r10;"
2678 "r1 += -32;"
2679 "*(u64 *)(r10 - 16) = r1;"
2680 /* create r1 with two candidate offsets: fp-8 or fp-16 */
2681 "call %[bpf_get_prandom_u32];"
2682 "if r0 == 0 goto 1f;"
2683 "r1 = r10;"
2684 "r1 += -8;"
2685 "goto 2f;"
2686"1:"
2687 "r1 = r10;"
2688 "r1 += -16;"
2689"2:"
2690 /* BPF_ST: store immediate through multi-offset r1 */
2691 "*(u64 *)(r1 + 0) = 0;"
2692 /* read back fp-16 and deref */
2693 "r0 = *(u64 *)(r10 - 16);"
2694 "r0 = *(u64 *)(r0 + 0);"
2695 "r0 = 0;"
2696 "exit;"
2697 :: __imm(bpf_get_prandom_u32)
2698 : __clobber_all);
2699}
2700
2701/*
2702 * Check that BPF_ST with a known offset fully overwrites stack slot
2703 * from the arg tracking point of view.
2704 */
2705SEC("socket")
2706__log_level(2)
2707__success
2708__msg("5: (7a) *(u64 *)(r1 +0) = 0 fp-8: fp0-16 -> _{{$}}")
2709__naked void st_imm_join_with_single_off(void)
2710{
2711 asm volatile (
2712 "r2 = r10;"
2713 "r2 += -16;"
2714 "*(u64 *)(r10 - 8) = r2;"
2715 "r1 = r10;"
2716 "r1 += -8;"
2717 "*(u64 *)(r1 + 0) = 0;"
2718 "r0 = 0;"
2719 "exit;"
2720 ::: __clobber_all);
2721}
2722
2723/*
2724 * Same as spill_join_with_imprecise_off but the write is BPF_ST.
2725 * Use "r2 = -8; r1 += r2" to make arg tracking lose offset
2726 * precision while the main verifier keeps r1 as fixed-offset.
2727 *
2728 * fp-8 = &fp-24
2729 * fp-16 = &fp-32
2730 * r1 = fp-8 (imprecise to arg tracking)
2731 * *(u64 *)(r1 + 0) = 0 -- BPF_ST with immediate
2732 * r0 = *(u64 *)(r10 - 16) -- fill from fp-16
2733 * r0 = *(u64 *)(r0 + 0) -- deref: should produce use
2734 */
2735SEC("socket")
2736__log_level(2)
2737__success
2738__msg("13: (79) r0 = *(u64 *)(r0 +0) ; use: fp0-32")
2739__naked void st_imm_join_with_imprecise_off(void)
2740{
2741 asm volatile (
2742 "*(u64 *)(r10 - 24) = 0;"
2743 "*(u64 *)(r10 - 32) = 0;"
2744 "r1 = r10;"
2745 "r1 += -24;"
2746 "*(u64 *)(r10 - 8) = r1;"
2747 "r1 = r10;"
2748 "r1 += -32;"
2749 "*(u64 *)(r10 - 16) = r1;"
2750 /* r1 = fp-8 but arg tracking sees off_cnt == 0 */
2751 "r1 = r10;"
2752 "r2 = -8;"
2753 "r1 += r2;"
2754 /* store immediate through imprecise r1 */
2755 "*(u64 *)(r1 + 0) = 0;"
2756 /* read back fp-16 */
2757 "r0 = *(u64 *)(r10 - 16);"
2758 /* deref: should produce use */
2759 "r0 = *(u64 *)(r0 + 0);"
2760 "r0 = 0;"
2761 "exit;"
2762 ::: __clobber_all);
2763}
2764
2765/*
2766 * Test that spilling through an ARG_IMPRECISE pointer joins with
2767 * existing at_stack values. Subprog receives r1 = fp0-24 and
2768 * r2 = map_value, creates an ARG_IMPRECISE pointer by joining caller
2769 * and callee FP on two branches.
2770 *
2771 * Setup: callee spills &fp1-16 to fp1-8 (precise, tracked).
2772 * Then writes map_value through ARG_IMPRECISE r1 — on path A
2773 * this hits fp1-8, on path B it hits caller stack.
2774 * Since spill_to_stack is skipped for ARG_IMPRECISE dst,
2775 * fp1-8 tracking isn't joined with none.
2776 *
2777 * Expected after the imprecise write:
2778 * - arg tracking should show fp1-8 = fp1-16|fp1+0 (joined with none)
2779 * - read from fp1-8 and deref should produce use for fp1-16
2780 * - write through it should NOT produce def for fp1-16
2781 */
2782SEC("socket")
2783__log_level(2)
2784__success
2785__msg("26: (79) r0 = *(u64 *)(r10 -8) // r1=IMP3 r6=fp0-24 r7=fp1-16 fp-8=fp1-16|fp1+0")
2786__naked void imprecise_dst_spill_join(void)
2787{
2788 asm volatile (
2789 "*(u64 *)(r10 - 24) = 0;"
2790 /* map lookup for a valid non-FP pointer */
2791 "*(u32 *)(r10 - 32) = 0;"
2792 "r1 = %[map] ll;"
2793 "r2 = r10;"
2794 "r2 += -32;"
2795 "call %[bpf_map_lookup_elem];"
2796 "if r0 == 0 goto 1f;"
2797 /* r1 = &caller_fp-24, r2 = map_value */
2798 "r1 = r10;"
2799 "r1 += -24;"
2800 "r2 = r0;"
2801 "call imprecise_dst_spill_join_sub;"
2802"1:"
2803 "r0 = 0;"
2804 "exit;"
2805 :: __imm_addr(map),
2806 __imm(bpf_map_lookup_elem)
2807 : __clobber_all);
2808}
2809
2810static __used __naked void imprecise_dst_spill_join_sub(void)
2811{
2812 asm volatile (
2813 /* r6 = &caller_fp-24 (frame=0), r8 = map_value */
2814 "r6 = r1;"
2815 "r8 = r2;"
2816 /* spill &fp1-16 to fp1-8: at_stack[0] = fp1-16 */
2817 "*(u64 *)(r10 - 16) = 0;"
2818 "r7 = r10;"
2819 "r7 += -16;"
2820 "*(u64 *)(r10 - 8) = r7;"
2821 /* branch to create ARG_IMPRECISE pointer */
2822 "call %[bpf_get_prandom_u32];"
2823 /* path B: r1 = caller fp-24 (frame=0) */
2824 "r1 = r6;"
2825 "if r0 == 0 goto 1f;"
2826 /* path A: r1 = callee fp-8 (frame=1) */
2827 "r1 = r10;"
2828 "r1 += -8;"
2829"1:"
2830 /* r1 = ARG_IMPRECISE{mask=BIT(0)|BIT(1)}.
2831 * Write map_value (non-FP) through r1. On path A this overwrites fp1-8.
2832 * Should join at_stack[0] with none: fp1-16|fp1+0.
2833 */
2834 "*(u64 *)(r1 + 0) = r8;"
2835 /* read fp1-8: should be fp1-16|fp1+0 (joined) */
2836 "r0 = *(u64 *)(r10 - 8);"
2837 "*(u64 *)(r0 + 0) = 42;"
2838 "r0 = 0;"
2839 "exit;"
2840 :: __imm(bpf_get_prandom_u32)
2841 : __clobber_all);
2842}
tools/testing/selftests/bpf/sysctl_helpers.c+37−0
@@ -0,0 +1,37 @@
1// SPDX-License-Identifier: GPL-2.0
2#include <stdio.h>
3#include <errno.h>
4#include <string.h>
5
6#include "sysctl_helpers.h"
7#include "test_progs.h"
8
9int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val)
10{
11 int ret = 0;
12 FILE *fp;
13
14 fp = fopen(sysctl_path, "r+");
15 if (!fp)
16 return -errno;
17 if (old_val && fscanf(fp, "%s", old_val) <= 0) {
18 ret = -ENOENT;
19 } else if (!old_val || strcmp(old_val, new_val) != 0) {
20 fseek(fp, 0, SEEK_SET);
21 if (fprintf(fp, "%s", new_val) < 0)
22 ret = -errno;
23 }
24 fclose(fp);
25
26 return ret;
27}
28
29int sysctl_set_or_fail(const char *sysctl_path, char *old_val, const char *new_val)
30{
31 int err;
32
33 err = sysctl_set(sysctl_path, old_val, new_val);
34 if (err)
35 PRINT_FAIL("failed to set %s to %s: %s\n", sysctl_path, new_val, strerror(-err));
36 return err;
37}
tools/testing/selftests/bpf/sysctl_helpers.h+8−0
@@ -0,0 +1,8 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __SYSCTL_HELPERS_H
3#define __SYSCTL_HELPERS_H
4
5int sysctl_set(const char *sysctl_path, char *old_val, const char *new_val);
6int sysctl_set_or_fail(const char *sysctl_path, char *old_val, const char *new_val);
7
8#endif