Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull bpf fixes from Alexei Starovoitov: "Most of the diff stat comes from Xu Kuohai's fix to emit ENDBR/BTI, since all JITs had to be touched to move constant blinding out and pass bpf_verifier_env in. - Fix use-after-free in arena_vm_close on fork (Alexei Starovoitov) - Dissociate struct_ops program with map if map_update fails (Amery Hung) - Fix out-of-range and off-by-one bugs in arm64 JIT (Daniel Borkmann) - Fix precedence bug in convert_bpf_ld_abs alignment check (Daniel Borkmann) - Fix arg tracking for imprecise/multi-offset in BPF_ST/STX insns (Eduard Zingerman) - Copy token from main to subprogs to fix missing kallsyms (Eduard Zingerman) - Prevent double close and leak of btf objects in libbpf (Jiri Olsa) - Fix af_unix null-ptr-deref in sockmap (Michal Luczaj) - Fix NULL deref in map_kptr_match_type for scalar regs (Mykyta Yatsenko) - Avoid unnecessary IPIs. Remove redundant bpf_flush_icache() in arm64 and riscv JITs (Puranjay Mohan) - Fix out of bounds access. Validate node_id in arena_alloc_pages() (Puranjay Mohan) - Reject BPF-to-BPF calls and callbacks in arm32 JIT (Puranjay Mohan) - Refactor all JITs to pass bpf_verifier_env to emit ENDBR/BTI for indirect jump targets on x86-64, arm64 JITs (Xu Kuohai) - Allow UTF-8 literals in bpf_bprintf_prepare() (Yihan Ding)" * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (32 commits) bpf, arm32: Reject BPF-to-BPF calls and callbacks in the JIT bpf: Dissociate struct_ops program with map if map_update fails bpf: Validate node_id in arena_alloc_pages() libbpf: Prevent double close and leak of btf objects selftests/bpf: cover UTF-8 trace_printk output bpf: allow UTF-8 literals in bpf_bprintf_prepare() selftests/bpf: Reject scalar store into kptr slot bpf: Fix NULL deref in map_kptr_match_type for scalar regs bpf: Fix precedence bug in convert_bpf_ld_abs alignment check bpf, arm64: Emit BTI for indirect jump target bpf, x86: Emit ENDBR for indirect jump targets bpf: Add helper to detect indirect jump targets bpf: Pass bpf_verifier_env to JIT bpf: Move constants blinding out of arch-specific JITs bpf, sockmap: Take state lock for af_unix iter bpf, sockmap: Fix af_unix null-ptr-deref in proto update selftests/bpf: Extend bpf_iter_unix to attempt deadlocking bpf, sockmap: Fix af_unix iter deadlock bpf, sockmap: Annotate af_unix sock:: Sk_state data-races selftests/bpf: verify kallsyms entries for token-loaded subprograms ...

Linus Torvalds committed 7 months ago

Commit eb0d6d92 parents 12bffae e1d486446 files · +1173 −707Browse files

Showing the changes to kernel (8 of 46 files). Show all

Unified · Split

8 files

kernel/bpf/arena.c +20 −3
kernel/bpf/bpf_struct_ops.c +4 −3
kernel/bpf/core.c +99 −39
kernel/bpf/fixups.c +138 −25
kernel/bpf/helpers.c +16 −1
kernel/bpf/liveness.c +62 −52
kernel/bpf/syscall.c +0 −4
kernel/bpf/verifier.c +19 −7

kernel/bpf/arena.c+20−3

@@ -341,6 +341,16 @@ static void arena_vm_open(struct vm_area_struct *vma) · 340 unmodified lines
341	341	refcount_inc(&vml->mmap_count);
342	342	}
343	343
	344	+static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr)
	345	+{
	346	+ return -EINVAL;
	347	+}
	348	+
	349	+static int arena_vm_mremap(struct vm_area_struct *vma)
	350	+{
	351	+ return -EINVAL;
	352	+}
	353	+
344	354	static void arena_vm_close(struct vm_area_struct *vma)
345	355	{
346	356	struct bpf_map *map = vma->vm_file->private_data;
@@ -417,6 +427,8 @@ out_unlock_sigsegv: · 70 unmodified lines
417	427
418	428	static const struct vm_operations_struct arena_vm_ops = {
419	429	.open = arena_vm_open,
	430	+ .may_split = arena_vm_may_split,
	431	+ .mremap = arena_vm_mremap,
420	432	.close = arena_vm_close,
421	433	.fault = arena_vm_fault,
422	434	};
@@ -486,10 +498,11 @@ static int arena_map_mmap(struct bpf_map map, struct vm_area_struct vma) · 63 unmodified lines
486	498	arena->user_vm_end = vma->vm_end;
487	499	/*
488	500	* bpf_map_mmap() checks that it's being mmaped as VM_SHARED and
489		− * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid
490		− * potential change of user_vm_start.
	501	+ * clears VM_MAYEXEC. Set VM_DONTEXPAND to avoid potential change
	502	+ * of user_vm_start. Set VM_DONTCOPY to prevent arena VMA from
	503	+ * being copied into the child process on fork.
491	504	*/
492		− vm_flags_set(vma, VM_DONTEXPAND);
	505	+ vm_flags_set(vma, VM_DONTEXPAND \| VM_DONTCOPY);
493	506	vma->vm_ops = &arena_vm_ops;
494	507	return 0;
495	508	}
@@ -549,6 +562,10 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt · 53 unmodified lines
549	562	u32 uaddr32;
550	563	int ret, i;
551	564
	565	+ if (node_id != NUMA_NO_NODE &&
	566	+ ((unsigned int)node_id >= nr_node_ids \|\| !node_online(node_id)))
	567	+ return 0;
	568	+
552	569	if (page_cnt > page_cnt_max)
553	570	return 0;
554	571

kernel/bpf/bpf_struct_ops.c+4−3

@@ -811,9 +811,6 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map map, void key, · 810 unmodified lines
811	811	goto reset_unlock;
812	812	}
813	813
814		− /* Poison pointer on error instead of return for backward compatibility */
815		− bpf_prog_assoc_struct_ops(prog, &st_map->map);
816		−
817	814	link = kzalloc_obj(*link, GFP_USER);
818	815	if (!link) {
819	816	bpf_prog_put(prog);
@@ -824,6 +821,9 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map map, void key, · 4 unmodified lines
824	821	&bpf_struct_ops_link_lops, prog, prog->expected_attach_type);
825	822	*plink++ = &link->link;
826	823
	824	+ /* Poison pointer on error instead of return for backward compatibility */
	825	+ bpf_prog_assoc_struct_ops(prog, &st_map->map);
	826	+
827	827	ksym = kzalloc_obj(*ksym, GFP_USER);
828	828	if (!ksym) {
829	829	err = -ENOMEM;
@@ -906,6 +906,7 @@ static long bpf_struct_ops_map_update_elem(struct bpf_map map, void key, · 76 unmodified lines
906	906	reset_unlock:
907	907	bpf_struct_ops_map_free_ksyms(st_map);
908	908	bpf_struct_ops_map_free_image(st_map);
	909	+ bpf_struct_ops_map_dissoc_progs(st_map);
909	910	bpf_struct_ops_map_put_progs(st_map);
910	911	memset(uvalue, 0, map->value_size);
911	912	memset(kvalue, 0, map->value_size);

kernel/bpf/core.c+99−39

@@ -1491,24 +1491,11 @@ void bpf_jit_prog_release_other(struct bpf_prog fp, struct bpf_prog fp_other) · 1490 unmodified lines
1491	1491	bpf_prog_clone_free(fp_other);
1492	1492	}
1493	1493
1494		−static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)
1495		−{
1496		−#ifdef CONFIG_BPF_SYSCALL
1497		− struct bpf_map *map;
1498		− int i;
1499		−
1500		− if (len <= 1)
1501		− return;
1502		−
1503		− for (i = 0; i < prog->aux->used_map_cnt; i++) {
1504		− map = prog->aux->used_maps[i];
1505		− if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
1506		− bpf_insn_array_adjust(map, off, len);
1507		− }
1508		−#endif
1509		−}
1510		−
1511		−struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog)
	1494	+/*
	1495	+ * Now this function is used only to blind the main prog and must be invoked only when
	1496	+ * bpf_prog_need_blind() returns true.
	1497	+ */
	1498	+struct bpf_prog bpf_jit_blind_constants(struct bpf_verifier_env env, struct bpf_prog *prog)
1512	1499	{
1513	1500	struct bpf_insn insn_buff[16], aux[2];
1514	1501	struct bpf_prog clone, tmp;
@@ -1516,13 +1503,17 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 1 unmodified line
1516	1503	struct bpf_insn *insn;
1517	1504	int i, rewritten;
1518	1505
1519		− if (!prog->blinding_requested \|\| prog->blinded)
1520		− return prog;
	1506	+ if (WARN_ON_ONCE(env && env->prog != prog))
	1507	+ return ERR_PTR(-EINVAL);
1521	1508
1522	1509	clone = bpf_prog_clone_create(prog, GFP_USER);
1523	1510	if (!clone)
1524	1511	return ERR_PTR(-ENOMEM);
1525	1512
	1513	+ /* make sure bpf_patch_insn_data() patches the correct prog */
	1514	+ if (env)
	1515	+ env->prog = clone;
	1516	+
1526	1517	insn_cnt = clone->len;
1527	1518	insn = clone->insnsi;
1528	1519
@@ -1550,21 +1541,28 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 21 unmodified lines
1550	1541	if (!rewritten)
1551	1542	continue;
1552	1543
1553		− tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
1554		− if (IS_ERR(tmp)) {
	1544	+ if (env)
	1545	+ tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
	1546	+ else
	1547	+ tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
	1548	+
	1549	+ if (IS_ERR_OR_NULL(tmp)) {
	1550	+ if (env)
	1551	+ /* restore the original prog */
	1552	+ env->prog = prog;
1555	1553	/* Patching may have repointed aux->prog during
1556	1554	* realloc from the original one, so we need to
1557	1555	* fix it up here on error.
1558	1556	*/
1559	1557	bpf_jit_prog_release_other(prog, clone);
1560		− return tmp;
	1558	+ return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM);
1561	1559	}
1562	1560
1563	1561	clone = tmp;
1564	1562	insn_delta = rewritten - 1;
1565	1563
1566		− /* Instructions arrays must be updated using absolute xlated offsets */
1567		− adjust_insn_arrays(clone, prog->aux->subprog_start + i, rewritten);
	1564	+ if (env)
	1565	+ env->prog = clone;
1568	1566
1569	1567	/* Walk new program and skip insns we just inserted. */
1570	1568	insn = clone->insnsi + i + insn_delta;
@@ -1575,6 +1573,15 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 4 unmodified lines
1575	1573	clone->blinded = 1;
1576	1574	return clone;
1577	1575	}
	1576	+
	1577	+bool bpf_insn_is_indirect_target(const struct bpf_verifier_env env, const struct bpf_prog prog,
	1578	+ int insn_idx)
	1579	+{
	1580	+ if (!env)
	1581	+ return false;
	1582	+ insn_idx += prog->aux->subprog_start;
	1583	+ return env->insn_aux_data[insn_idx].indirect_target;
	1584	+}
1578	1585	#endif /* CONFIG_BPF_JIT */
1579	1586
1580	1587	/* Base function for offset calculation. Needs to go into .text section,
@@ -2533,18 +2540,55 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) · 952 unmodified lines
2533	2540	return select_interpreter;
2534	2541	}
2535	2542
2536		−/**
2537		− * bpf_prog_select_runtime - select exec runtime for BPF program
2538		− * @fp: bpf_prog populated with BPF program
2539		− * @err: pointer to error variable
2540		− *
2541		− * Try to JIT eBPF program, if JIT is not available, use interpreter.
2542		− * The BPF program will be executed via bpf_prog_run() function.
2543		− *
2544		− * Return: the &fp argument along with &err set to 0 for success or
2545		− * a negative errno code on failure
2546		− */
2547		−struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err)
	2543	+static struct bpf_prog bpf_prog_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
	2544	+{
	2545	+#ifdef CONFIG_BPF_JIT
	2546	+ struct bpf_prog *orig_prog;
	2547	+ struct bpf_insn_aux_data *orig_insn_aux;
	2548	+
	2549	+ if (!bpf_prog_need_blind(prog))
	2550	+ return bpf_int_jit_compile(env, prog);
	2551	+
	2552	+ if (env) {
	2553	+ /*
	2554	+ * If env is not NULL, we are called from the end of bpf_check(), at this
	2555	+ * point, only insn_aux_data is used after failure, so it should be restored
	2556	+ * on failure.
	2557	+ */
	2558	+ orig_insn_aux = bpf_dup_insn_aux_data(env);
	2559	+ if (!orig_insn_aux)
	2560	+ return prog;
	2561	+ }
	2562	+
	2563	+ orig_prog = prog;
	2564	+ prog = bpf_jit_blind_constants(env, prog);
	2565	+ /*
	2566	+ * If blinding was requested and we failed during blinding, we must fall
	2567	+ * back to the interpreter.
	2568	+ */
	2569	+ if (IS_ERR(prog))
	2570	+ goto out_restore;
	2571	+
	2572	+ prog = bpf_int_jit_compile(env, prog);
	2573	+ if (prog->jited) {
	2574	+ bpf_jit_prog_release_other(prog, orig_prog);
	2575	+ if (env)
	2576	+ vfree(orig_insn_aux);
	2577	+ return prog;
	2578	+ }
	2579	+
	2580	+ bpf_jit_prog_release_other(orig_prog, prog);
	2581	+
	2582	+out_restore:
	2583	+ prog = orig_prog;
	2584	+ if (env)
	2585	+ bpf_restore_insn_aux_data(env, orig_insn_aux);
	2586	+#endif
	2587	+ return prog;
	2588	+}
	2589	+
	2590	+struct bpf_prog __bpf_prog_select_runtime(struct bpf_verifier_env env, struct bpf_prog *fp,
	2591	+ int *err)
2548	2592	{
2549	2593	/* In case of BPF to BPF calls, verifier did all the prep
2550	2594	* work with regards to JITing, etc.
@@ -2572,7 +2616,7 @@ struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err) · 21 unmodified lines
2572	2616	if (*err)
2573	2617	return fp;
2574	2618
2575		− fp = bpf_int_jit_compile(fp);
	2619	+ fp = bpf_prog_jit_compile(env, fp);
2576	2620	bpf_prog_jit_attempt_done(fp);
2577	2621	if (!fp->jited && jit_needed) {
2578	2622	*err = -ENOTSUPP;
@@ -2598,6 +2642,22 @@ finalize: · 19 unmodified lines
2598	2642
2599	2643	return fp;
2600	2644	}
	2645	+
	2646	+/**
	2647	+ * bpf_prog_select_runtime - select exec runtime for BPF program
	2648	+ * @fp: bpf_prog populated with BPF program
	2649	+ * @err: pointer to error variable
	2650	+ *
	2651	+ * Try to JIT eBPF program, if JIT is not available, use interpreter.
	2652	+ * The BPF program will be executed via bpf_prog_run() function.
	2653	+ *
	2654	+ * Return: the &fp argument along with &err set to 0 for success or
	2655	+ * a negative errno code on failure
	2656	+ */
	2657	+struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err)
	2658	+{
	2659	+ return __bpf_prog_select_runtime(NULL, fp, err);
	2660	+}
2601	2661	EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
2602	2662
2603	2663	static unsigned int __bpf_prog_ret1(const void *ctx,
@@ -3085,7 +3145,7 @@ const struct bpf_func_proto bpf_tail_call_proto = { · 481 unmodified lines
3085	3145	* It is encouraged to implement bpf_int_jit_compile() instead, so that
3086	3146	* eBPF and implicitly also cBPF can get JITed!
3087	3147	*/
3088		−struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)
	3148	+struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog prog)
3089	3149	{
3090	3150	return prog;
3091	3151	}

kernel/bpf/fixups.c+138−25

@@ -183,6 +183,18 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, · 182 unmodified lines
183	183	data[i].seen = old_seen;
184	184	data[i].zext_dst = insn_has_def32(insn + i);
185	185	}
	186	+
	187	+ /*
	188	+ * The indirect_target flag of the original instruction was moved to the last of the
	189	+ * new instructions by the above memmove and memset, but the indirect jump target is
	190	+ * actually the first instruction, so move it back. This also matches with the behavior
	191	+ * of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new
	192	+ * instruction.
	193	+ */
	194	+ if (data[off + cnt - 1].indirect_target) {
	195	+ data[off].indirect_target = 1;
	196	+ data[off + cnt - 1].indirect_target = 0;
	197	+ }
186	198	}
187	199
188	200	static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
@@ -232,8 +244,8 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) · 43 unmodified lines
232	244	}
233	245	}
234	246
235		−static struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,
236		− const struct bpf_insn *patch, u32 len)
	247	+struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,
	248	+ const struct bpf_insn *patch, u32 len)
237	249	{
238	250	struct bpf_prog *new_prog;
239	251	struct bpf_insn_aux_data *new_data = NULL;
@@ -973,7 +985,47 @@ patch_insn_buf: · 733 unmodified lines
973	985	return 0;
974	986	}
975	987
976		−int bpf_jit_subprogs(struct bpf_verifier_env *env)
	988	+static u32 bpf_dup_subprog_starts(struct bpf_verifier_env env)
	989	+{
	990	+ u32 *starts = NULL;
	991	+
	992	+ starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT);
	993	+ if (starts) {
	994	+ for (int i = 0; i < env->subprog_cnt; i++)
	995	+ starts[i] = env->subprog_info[i].start;
	996	+ }
	997	+ return starts;
	998	+}
	999	+
	1000	+static void bpf_restore_subprog_starts(struct bpf_verifier_env env, u32 orig_starts)
	1001	+{
	1002	+ for (int i = 0; i < env->subprog_cnt; i++)
	1003	+ env->subprog_info[i].start = orig_starts[i];
	1004	+ /* restore the start of fake 'exit' subprog as well */
	1005	+ env->subprog_info[env->subprog_cnt].start = env->prog->len;
	1006	+}
	1007	+
	1008	+struct bpf_insn_aux_data bpf_dup_insn_aux_data(struct bpf_verifier_env env)
	1009	+{
	1010	+ size_t size;
	1011	+ void *new_aux;
	1012	+
	1013	+ size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);
	1014	+ new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT);
	1015	+ if (new_aux)
	1016	+ memcpy(new_aux, env->insn_aux_data, size);
	1017	+ return new_aux;
	1018	+}
	1019	+
	1020	+void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
	1021	+ struct bpf_insn_aux_data *orig_insn_aux)
	1022	+{
	1023	+ /* the expanded elements are zero-filled, so no special handling is required */
	1024	+ vfree(env->insn_aux_data);
	1025	+ env->insn_aux_data = orig_insn_aux;
	1026	+}
	1027	+
	1028	+static int jit_subprogs(struct bpf_verifier_env *env)
977	1029	{
978	1030	struct bpf_prog prog = env->prog, func, tmp;
979	1031	int i, j, subprog_start, subprog_end = 0, len, subprog;
@@ -981,10 +1033,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 1 unmodified line
981	1033	struct bpf_insn *insn;
982	1034	void *old_bpf_func;
983	1035	int err, num_exentries;
984		− int old_len, subprog_start_adjustment = 0;
985		−
986		− if (env->subprog_cnt <= 1)
987		− return 0;
988	1036
989	1037	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
990	1038	if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
@@ -1053,10 +1101,11 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 62 unmodified lines
1053	1101	goto out_free;
1054	1102	func[i]->is_func = 1;
1055	1103	func[i]->sleepable = prog->sleepable;
	1104	+ func[i]->blinded = prog->blinded;
1056	1105	func[i]->aux->func_idx = i;
1057	1106	/* Below members will be freed only at prog->aux */
1058	1107	func[i]->aux->btf = prog->aux->btf;
1059		− func[i]->aux->subprog_start = subprog_start + subprog_start_adjustment;
	1108	+ func[i]->aux->subprog_start = subprog_start;
1060	1109	func[i]->aux->func_info = prog->aux->func_info;
1061	1110	func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
1062	1111	func[i]->aux->poke_tab = prog->aux->poke_tab;
@@ -1110,17 +1159,10 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 47 unmodified lines
1110	1159	func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
1111	1160	func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
1112	1161	func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
	1162	+ func[i]->aux->token = prog->aux->token;
1113	1163	if (!i)
1114	1164	func[i]->aux->exception_boundary = env->seen_exception;
1115		−
1116		− /*
1117		− * To properly pass the absolute subprog start to jit
1118		− * all instruction adjustments should be accumulated
1119		− */
1120		− old_len = func[i]->len;
1121		− func[i] = bpf_int_jit_compile(func[i]);
1122		− subprog_start_adjustment += func[i]->len - old_len;
1123		−
	1165	+ func[i] = bpf_int_jit_compile(env, func[i]);
1124	1166	if (!func[i]->jited) {
1125	1167	err = -ENOTSUPP;
1126	1168	goto out_free;
@@ -1164,7 +1206,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 37 unmodified lines
1164	1206	}
1165	1207	for (i = 0; i < env->subprog_cnt; i++) {
1166	1208	old_bpf_func = func[i]->bpf_func;
1167		− tmp = bpf_int_jit_compile(func[i]);
	1209	+ tmp = bpf_int_jit_compile(env, func[i]);
1168	1210	if (tmp != func[i] \|\| func[i]->bpf_func != old_bpf_func) {
1169	1211	verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
1170	1212	err = -ENOTSUPP;
@@ -1246,16 +1288,87 @@ out_free: · 75 unmodified lines
1246	1288	}
1247	1289	kfree(func);
1248	1290	out_undo_insn:
	1291	+ bpf_prog_jit_attempt_done(prog);
	1292	+ return err;
	1293	+}
	1294	+
	1295	+int bpf_jit_subprogs(struct bpf_verifier_env *env)
	1296	+{
	1297	+ int err, i;
	1298	+ bool blinded = false;
	1299	+ struct bpf_insn *insn;
	1300	+ struct bpf_prog prog, orig_prog;
	1301	+ struct bpf_insn_aux_data *orig_insn_aux;
	1302	+ u32 *orig_subprog_starts;
	1303	+
	1304	+ if (env->subprog_cnt <= 1)
	1305	+ return 0;
	1306	+
	1307	+ prog = orig_prog = env->prog;
	1308	+ if (bpf_prog_need_blind(prog)) {
	1309	+ orig_insn_aux = bpf_dup_insn_aux_data(env);
	1310	+ if (!orig_insn_aux) {
	1311	+ err = -ENOMEM;
	1312	+ goto out_cleanup;
	1313	+ }
	1314	+ orig_subprog_starts = bpf_dup_subprog_starts(env);
	1315	+ if (!orig_subprog_starts) {
	1316	+ vfree(orig_insn_aux);
	1317	+ err = -ENOMEM;
	1318	+ goto out_cleanup;
	1319	+ }
	1320	+ prog = bpf_jit_blind_constants(env, prog);
	1321	+ if (IS_ERR(prog)) {
	1322	+ err = -ENOMEM;
	1323	+ prog = orig_prog;
	1324	+ goto out_restore;
	1325	+ }
	1326	+ blinded = true;
	1327	+ }
	1328	+
	1329	+ err = jit_subprogs(env);
	1330	+ if (err)
	1331	+ goto out_jit_err;
	1332	+
	1333	+ if (blinded) {
	1334	+ bpf_jit_prog_release_other(prog, orig_prog);
	1335	+ kvfree(orig_subprog_starts);
	1336	+ vfree(orig_insn_aux);
	1337	+ }
	1338	+
	1339	+ return 0;
	1340	+
	1341	+out_jit_err:
	1342	+ if (blinded) {
	1343	+ bpf_jit_prog_release_other(orig_prog, prog);
	1344	+ /* roll back to the clean original prog */
	1345	+ prog = env->prog = orig_prog;
	1346	+ goto out_restore;
	1347	+ } else {
	1348	+ if (err != -EFAULT) {
	1349	+ /*
	1350	+ * We will fall back to interpreter mode when err is not -EFAULT, before
	1351	+ * that, insn->off and insn->imm should be restored to their original
	1352	+ * values since they were modified by jit_subprogs.
	1353	+ */
	1354	+ for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
	1355	+ if (!bpf_pseudo_call(insn))
	1356	+ continue;
	1357	+ insn->off = 0;
	1358	+ insn->imm = env->insn_aux_data[i].call_imm;
	1359	+ }
	1360	+ }
	1361	+ goto out_cleanup;
	1362	+ }
	1363	+
	1364	+out_restore:
	1365	+ bpf_restore_subprog_starts(env, orig_subprog_starts);
	1366	+ bpf_restore_insn_aux_data(env, orig_insn_aux);
	1367	+ kvfree(orig_subprog_starts);
	1368	+out_cleanup:
1249	1369	/* cleanup main prog to be interpreted */
1250	1370	prog->jit_requested = 0;
1251	1371	prog->blinding_requested = 0;
1252		− for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
1253		− if (!bpf_pseudo_call(insn))
1254		− continue;
1255		− insn->off = 0;
1256		− insn->imm = env->insn_aux_data[i].call_imm;
1257		− }
1258		− bpf_prog_jit_attempt_done(prog);
1259	1372	return err;
1260	1373	}
1261	1374

kernel/bpf/helpers.c+16−1

@@ -845,7 +845,13 @@ int bpf_bprintf_prepare(const char fmt, u32 fmt_size, const u64 raw_args, · 844 unmodified lines
845	845	data->buf = buffers->buf;
846	846
847	847	for (i = 0; i < fmt_size; i++) {
848		− if ((!isprint(fmt[i]) && !isspace(fmt[i])) \|\| !isascii(fmt[i])) {
	848	+ unsigned char c = fmt[i];
	849	+
	850	+ /*
	851	+ * Permit bytes >= 0x80 in plain text so UTF-8 literals can pass
	852	+ * through unchanged, while still rejecting ASCII control bytes.
	853	+ */
	854	+ if (isascii(c) && !isprint(c) && !isspace(c)) {
849	855	err = -EINVAL;
850	856	goto out;
851	857	}
@@ -867,6 +873,15 @@ int bpf_bprintf_prepare(const char fmt, u32 fmt_size, const u64 raw_args, · 15 unmodified lines
867	873	* always access fmt[i + 1], in the worst case it will be a 0
868	874	*/
869	875	i++;
	876	+ c = fmt[i];
	877	+ /*
	878	+ * The format parser below only understands ASCII conversion
	879	+ * specifiers and modifiers, so reject non-ASCII after '%'.
	880	+ */
	881	+ if (!isascii(c)) {
	882	+ err = -EINVAL;
	883	+ goto out;
	884	+ }
870	885
871	886	/* skip optional "[0 +-][num]" width formatting field */
872	887	while (fmt[i] == '0' \|\| fmt[i] == '+' \|\| fmt[i] == '-' \|\|

kernel/bpf/liveness.c+62−52

@@ -574,7 +574,7 @@ static int print_instances(struct bpf_verifier_env *env) · 573 unmodified lines
574	574	*
575	575	* precise {frame=N, off=V} -- known absolute frame index and byte offset
576	576	* \|
577		− * offset-imprecise {frame=N, off=OFF_IMPRECISE}
	577	+ * offset-imprecise {frame=N, cnt=0}
578	578	* \| -- known frame identity, unknown offset
579	579	* fully-imprecise {frame=ARG_IMPRECISE, mask=bitmask}
580	580	* -- unknown frame identity; .mask is a
@@ -607,8 +607,6 @@ enum arg_track_state { · 26 unmodified lines
607	607	ARG_IMPRECISE = -3, /* lost identity; .mask is arg bitmask */
608	608	};
609	609
610		−#define OFF_IMPRECISE S16_MIN /* arg identity known but offset unknown */
611		−
612	610	/* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */
613	611	#define MAX_ARG_SPILL_SLOTS 64
614	612
@@ -622,28 +620,6 @@ static bool arg_is_fp(const struct arg_track *at) · 7 unmodified lines
622	620	return at->frame >= 0 \|\| at->frame == ARG_IMPRECISE;
623	621	}
624	622
625		−/*
626		− * Clear all tracked callee stack slots overlapping the byte range
627		− * [off, off+sz-1] where off is a negative FP-relative offset.
628		− */
629		−static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz)
630		−{
631		− struct arg_track none = { .frame = ARG_NONE };
632		−
633		− if (off == OFF_IMPRECISE) {
634		− for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
635		− at_stack[i] = none;
636		− return;
637		− }
638		− for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
639		− int slot_start = -((i + 1) * 8);
640		− int slot_end = slot_start + 8;
641		−
642		− if (slot_start < off + (int)sz && slot_end > off)
643		− at_stack[i] = none;
644		− }
645		−}
646		−
647	623	static void verbose_arg_track(struct bpf_verifier_env env, struct arg_track at)
648	624	{
649	625	int i;
@@ -863,16 +839,13 @@ static void arg_track_alu64(struct arg_track dst, const struct arg_track src) · 213 unmodified lines
863	839	dst = arg_join_imprecise(dst, *src);
864	840	}
865	841
866		−static s16 arg_add(s16 off, s64 delta)
	842	+static bool arg_add(s16 off, s64 delta, s16 *out)
867	843	{
868		− s64 res;
869		−
870		− if (off == OFF_IMPRECISE)
871		− return OFF_IMPRECISE;
872		− res = (s64)off + delta;
873		− if (res < S16_MIN + 1 \|\| res > S16_MAX)
874		− return OFF_IMPRECISE;
875		− return res;
	844	+ s16 d = delta;
	845	+
	846	+ if (d != delta)
	847	+ return true;
	848	+ return check_add_overflow(off, d, out);
876	849	}
877	850
878	851	static void arg_padd(struct arg_track *at, s64 delta)
@@ -882,9 +855,9 @@ static void arg_padd(struct arg_track *at, s64 delta) · 3 unmodified lines
882	855	if (at->off_cnt == 0)
883	856	return;
884	857	for (i = 0; i < at->off_cnt; i++) {
885		− s16 new_off = arg_add(at->off[i], delta);
	858	+ s16 new_off;
886	859
887		− if (new_off == OFF_IMPRECISE) {
	860	+ if (arg_add(at->off[i], delta, &new_off)) {
888	861	at->off_cnt = 0;
889	862	return;
890	863	}
@@ -899,8 +872,6 @@ static void arg_padd(struct arg_track *at, s64 delta) · 8 unmodified lines
899	872	*/
900	873	static int fp_off_to_slot(s16 off)
901	874	{
902		− if (off == OFF_IMPRECISE)
903		− return -1;
904	875	if (off >= 0 \|\| off < -(int)(MAX_ARG_SPILL_SLOTS * 8))
905	876	return -1;
906	877	if (off % 8)
@@ -930,9 +901,11 @@ static struct arg_track fill_from_stack(struct bpf_insn *insn, · 23 unmodified lines
930	901	return imp;
931	902
932	903	for (i = 0; i < cnt; i++) {
933		− s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
934		− int slot = fp_off_to_slot(fp_off);
	904	+ s16 fp_off, slot;
935	905
	906	+ if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
	907	+ return imp;
	908	+ slot = fp_off_to_slot(fp_off);
936	909	if (slot < 0)
937	910	return imp;
938	911	result = __arg_track_join(result, at_stack_out[slot]);
@@ -968,9 +941,12 @@ static void spill_to_stack(struct bpf_insn insn, struct arg_track at_out, · 29 unmodified lines
968	941	return;
969	942	}
970	943	for (i = 0; i < cnt; i++) {
971		− s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
972		− int slot = fp_off_to_slot(fp_off);
	944	+ s16 fp_off;
	945	+ int slot;
973	946
	947	+ if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
	948	+ continue;
	949	+ slot = fp_off_to_slot(fp_off);
974	950	if (slot < 0)
975	951	continue;
976	952	if (cnt == 1)
@@ -980,6 +956,32 @@ static void spill_to_stack(struct bpf_insn insn, struct arg_track at_out, · 3 unmodified lines
980	956	}
981	957	}
982	958
	959	+/*
	960	+ * Clear all tracked callee stack slots overlapping the byte range
	961	+ * [off, off+sz-1] where off is a negative FP-relative offset.
	962	+ */
	963	+static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt)
	964	+{
	965	+ struct arg_track none = { .frame = ARG_NONE };
	966	+
	967	+ if (cnt == 0) {
	968	+ for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
	969	+ at_stack[i] = __arg_track_join(at_stack[i], none);
	970	+ return;
	971	+ }
	972	+ for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
	973	+ int slot_start = -((i + 1) * 8);
	974	+ int slot_end = slot_start + 8;
	975	+
	976	+ if (slot_start < off + (int)sz && slot_end > off) {
	977	+ if (cnt == 1)
	978	+ at_stack[i] = none;
	979	+ else
	980	+ at_stack[i] = __arg_track_join(at_stack[i], none);
	981	+ }
	982	+ }
	983	+}
	984	+
983	985	/*
984	986	* Clear stack slots overlapping all possible FP offsets in @reg.
985	987	*/
@@ -990,18 +992,22 @@ static void clear_stack_for_all_offs(struct bpf_insn *insn, · 4 unmodified lines
990	992	int cnt, i;
991	993
992	994	if (reg == BPF_REG_FP) {
993		− clear_overlapping_stack_slots(at_stack_out, insn->off, sz);
	995	+ clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1);
994	996	return;
995	997	}
996	998	cnt = at_out[reg].off_cnt;
997	999	if (cnt == 0) {
998		− clear_overlapping_stack_slots(at_stack_out, OFF_IMPRECISE, sz);
	1000	+ clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt);
999	1001	return;
1000	1002	}
1001	1003	for (i = 0; i < cnt; i++) {
1002		− s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
	1004	+ s16 fp_off;
1003	1005
1004		− clear_overlapping_stack_slots(at_stack_out, fp_off, sz);
	1006	+ if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) {
	1007	+ clear_overlapping_stack_slots(at_stack_out, 0, sz, 0);
	1008	+ break;
	1009	+ }
	1010	+ clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt);
1005	1011	}
1006	1012	}
1007	1013
@@ -1042,6 +1048,12 @@ static void arg_track_log(struct bpf_verifier_env env, struct bpf_insn insn, i · 34 unmodified lines
1042	1048	verbose(env, "\n");
1043	1049	}
1044	1050
	1051	+static bool can_be_local_fp(int depth, int regno, struct arg_track *at)
	1052	+{
	1053	+ return regno == BPF_REG_FP \|\| at->frame == depth \|\|
	1054	+ (at->frame == ARG_IMPRECISE && (at->mask & BIT(depth)));
	1055	+}
	1056	+
1045	1057	/*
1046	1058	* Pure dataflow transfer function for arg_track state.
1047	1059	* Updates at_out[] based on how the instruction modifies registers.
@@ -1111,8 +1123,7 @@ static void arg_track_xfer(struct bpf_verifier_env env, struct bpf_insn insn, · 63 unmodified lines
1111	1123	at_out[r] = none;
1112	1124	} else if (class == BPF_LDX) {
1113	1125	u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1114		− bool src_is_local_fp = insn->src_reg == BPF_REG_FP \|\| src->frame == depth \|\|
1115		− (src->frame == ARG_IMPRECISE && (src->mask & BIT(depth)));
	1126	+ bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src);
1116	1127
1117	1128	/*
1118	1129	* Reload from callee stack: if src is current-frame FP-derived
@@ -1147,7 +1158,7 @@ static void arg_track_xfer(struct bpf_verifier_env env, struct bpf_insn insn, · 28 unmodified lines
1147	1158	bool dst_is_local_fp;
1148	1159
1149	1160	/* Track spills to current-frame FP-derived callee stack */
1150		− dst_is_local_fp = insn->dst_reg == BPF_REG_FP \|\| dst->frame == depth;
	1161	+ dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
1151	1162	if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM)
1152	1163	spill_to_stack(insn, at_out, insn->dst_reg,
1153	1164	at_stack_out, src, sz);
@@ -1166,7 +1177,7 @@ static void arg_track_xfer(struct bpf_verifier_env env, struct bpf_insn insn, · 12 unmodified lines
1166	1177	}
1167	1178	} else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) {
1168	1179	u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
1169		− bool dst_is_local_fp = insn->dst_reg == BPF_REG_FP \|\| dst->frame == depth;
	1180	+ bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
1170	1181
1171	1182	/* BPF_ST to FP-derived dst: clear overlapping stack slots */
1172	1183	if (dst_is_local_fp)
@@ -1316,8 +1327,7 @@ static int record_load_store_access(struct bpf_verifier_env *env, · 143 unmodified lines
1316	1327	resolved.off_cnt = ptr->off_cnt;
1317	1328	resolved.frame = ptr->frame;
1318	1329	for (oi = 0; oi < ptr->off_cnt; oi++) {
1319		− resolved.off[oi] = arg_add(ptr->off[oi], insn->off);
1320		− if (resolved.off[oi] == OFF_IMPRECISE) {
	1330	+ if (arg_add(ptr->off[oi], insn->off, &resolved.off[oi])) {
1321	1331	resolved.off_cnt = 0;
1322	1332	break;
1323	1333	}

kernel/bpf/syscall.c+0−4

@@ -3083,10 +3083,6 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) · 3082 unmodified lines
3083	3083	if (err < 0)
3084	3084	goto free_used_maps;
3085	3085
3086		− prog = bpf_prog_select_runtime(prog, &err);
3087		− if (err < 0)
3088		− goto free_used_maps;
3089		−
3090	3086	err = bpf_prog_mark_insn_arrays_ready(prog);
3091	3087	if (err < 0)
3092	3088	goto free_used_maps;

kernel/bpf/verifier.c+19−7

@@ -3497,6 +3497,11 @@ static int insn_stack_access_flags(int frameno, int spi) · 3496 unmodified lines
3497	3497	return INSN_F_STACK_ACCESS \| (spi << INSN_F_SPI_SHIFT) \| frameno;
3498	3498	}
3499	3499
	3500	+static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
	3501	+{
	3502	+ env->insn_aux_data[idx].indirect_target = true;
	3503	+}
	3504	+
3500	3505	#define LR_FRAMENO_BITS 3
3501	3506	#define LR_SPI_BITS 6
3502	3507	#define LR_ENTRY_BITS (LR_SPI_BITS + LR_FRAMENO_BITS + 1)
@@ -4544,6 +4549,9 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, · 1041 unmodified lines
4544	4549	int perm_flags;
4545	4550	const char *reg_name = "";
4546	4551
	4552	+ if (base_type(reg->type) != PTR_TO_BTF_ID)
	4553	+ goto bad_type;
	4554	+
4547	4555	if (btf_is_kernel(reg->btf)) {
4548	4556	perm_flags = PTR_MAYBE_NULL \| PTR_TRUSTED \| MEM_RCU;
4549	4557
@@ -4556,7 +4564,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, · 6 unmodified lines
4556	4564	perm_flags \|= MEM_PERCPU;
4557	4565	}
4558	4566
4559		− if (base_type(reg->type) != PTR_TO_BTF_ID \|\| (type_flag(reg->type) & ~perm_flags))
	4567	+ if (type_flag(reg->type) & ~perm_flags)
4560	4568	goto bad_type;
4561	4569
4562	4570	/* We need to verify reg->type and reg->btf, before accessing reg->btf */
@@ -17545,12 +17553,14 @@ static int check_indirect_jump(struct bpf_verifier_env env, struct bpf_insn in · 12982 unmodified lines
17545	17553	}
17546	17554
17547	17555	for (i = 0; i < n - 1; i++) {
	17556	+ mark_indirect_target(env, env->gotox_tmp_buf->items[i]);
17548	17557	other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
17549	17558	env->insn_idx, env->cur_state->speculative);
17550	17559	if (IS_ERR(other_branch))
17551	17560	return PTR_ERR(other_branch);
17552	17561	}
17553	17562	env->insn_idx = env->gotox_tmp_buf->items[n-1];
	17563	+ mark_indirect_target(env, env->insn_idx);
17554	17564	return INSN_IDX_UPDATED;
17555	17565	}
17556	17566
@@ -20155,6 +20165,14 @@ skip_full_check: · 2598 unmodified lines
20155	20165
20156	20166	adjust_btf_func(env);
20157	20167
	20168	+ /* extension progs temporarily inherit the attach_type of their targets
	20169	+ for verification purposes, so set it back to zero before returning
	20170	+ */
	20171	+ if (env->prog->type == BPF_PROG_TYPE_EXT)
	20172	+ env->prog->expected_attach_type = 0;
	20173	+
	20174	+ env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
	20175	+
20158	20176	err_release_maps:
20159	20177	if (ret)
20160	20178	release_insn_arrays(env);
@@ -20166,12 +20184,6 @@ err_release_maps: · 5 unmodified lines
20166	20184	if (!env->prog->aux->used_btfs)
20167	20185	release_btfs(env);
20168	20186
20169		− /* extension progs temporarily inherit the attach_type of their targets
20170		− for verification purposes, so set it back to zero before returning
20171		− */
20172		− if (env->prog->type == BPF_PROG_TYPE_EXT)
20173		− env->prog->expected_attach_type = 0;
20174		−
20175	20187	*prog = env->prog;
20176	20188
20177	20189	module_put(env->attach_btf_mod);

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code