Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull bpf fixes from Alexei Starovoitov: "Most of the diff stat comes from Xu Kuohai's fix to emit ENDBR/BTI, since all JITs had to be touched to move constant blinding out and pass bpf_verifier_env in. - Fix use-after-free in arena_vm_close on fork (Alexei Starovoitov) - Dissociate struct_ops program with map if map_update fails (Amery Hung) - Fix out-of-range and off-by-one bugs in arm64 JIT (Daniel Borkmann) - Fix precedence bug in convert_bpf_ld_abs alignment check (Daniel Borkmann) - Fix arg tracking for imprecise/multi-offset in BPF_ST/STX insns (Eduard Zingerman) - Copy token from main to subprogs to fix missing kallsyms (Eduard Zingerman) - Prevent double close and leak of btf objects in libbpf (Jiri Olsa) - Fix af_unix null-ptr-deref in sockmap (Michal Luczaj) - Fix NULL deref in map_kptr_match_type for scalar regs (Mykyta Yatsenko) - Avoid unnecessary IPIs. Remove redundant bpf_flush_icache() in arm64 and riscv JITs (Puranjay Mohan) - Fix out of bounds access. Validate node_id in arena_alloc_pages() (Puranjay Mohan) - Reject BPF-to-BPF calls and callbacks in arm32 JIT (Puranjay Mohan) - Refactor all JITs to pass bpf_verifier_env to emit ENDBR/BTI for indirect jump targets on x86-64, arm64 JITs (Xu Kuohai) - Allow UTF-8 literals in bpf_bprintf_prepare() (Yihan Ding)" * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: (32 commits) bpf, arm32: Reject BPF-to-BPF calls and callbacks in the JIT bpf: Dissociate struct_ops program with map if map_update fails bpf: Validate node_id in arena_alloc_pages() libbpf: Prevent double close and leak of btf objects selftests/bpf: cover UTF-8 trace_printk output bpf: allow UTF-8 literals in bpf_bprintf_prepare() selftests/bpf: Reject scalar store into kptr slot bpf: Fix NULL deref in map_kptr_match_type for scalar regs bpf: Fix precedence bug in convert_bpf_ld_abs alignment check bpf, arm64: Emit BTI for indirect jump target bpf, x86: Emit ENDBR for indirect jump targets bpf: Add helper to detect indirect jump targets bpf: Pass bpf_verifier_env to JIT bpf: Move constants blinding out of arch-specific JITs bpf, sockmap: Take state lock for af_unix iter bpf, sockmap: Fix af_unix null-ptr-deref in proto update selftests/bpf: Extend bpf_iter_unix to attempt deadlocking bpf, sockmap: Fix af_unix iter deadlock bpf, sockmap: Annotate af_unix sock:: Sk_state data-races selftests/bpf: verify kallsyms entries for token-loaded subprograms ...

Linus Torvalds committed 7 months ago

Commit eb0d6d92 parents 12bffae e1d486446 files · +1173 −707Browse files

Showing the changes to kernel (8 of 46 files). Show all

Unified · Split

8 files

kernel/bpf/arena.c +20 −3
kernel/bpf/bpf_struct_ops.c +4 −3
kernel/bpf/core.c +99 −39
kernel/bpf/fixups.c +138 −25
kernel/bpf/helpers.c +16 −1
kernel/bpf/liveness.c +62 −52
kernel/bpf/syscall.c +0 −4
kernel/bpf/verifier.c +19 −7

kernel/bpf/arena.c+20−3

 	refcount_inc(&vml->mmap_count);
 }
+static int arena_vm_may_split(struct vm_area_struct *vma, unsigned long addr)
+{
+	return -EINVAL;
+}
+static int arena_vm_mremap(struct vm_area_struct *vma)
+{
+	return -EINVAL;
+}
 static void arena_vm_close(struct vm_area_struct *vma)
 {
 	struct bpf_map *map = vma->vm_file->private_data;
 static const struct vm_operations_struct arena_vm_ops = {
 	.open		= arena_vm_open,
+	.may_split	= arena_vm_may_split,
+	.mremap		= arena_vm_mremap,
 	.close		= arena_vm_close,
 	.fault          = arena_vm_fault,
 };
 	arena->user_vm_end = vma->vm_end;
 	/*
 	 * bpf_map_mmap() checks that it's being mmaped as VM_SHARED and
-	 * clears VM_MAYEXEC. Set VM_DONTEXPAND as well to avoid
+	 * clears VM_MAYEXEC. Set VM_DONTEXPAND to avoid potential change
-	 * potential change of user_vm_start.
+	 * of user_vm_start. Set VM_DONTCOPY to prevent arena VMA from
+	 * being copied into the child process on fork.
 	 */
-	vm_flags_set(vma, VM_DONTEXPAND);
+	vm_flags_set(vma, VM_DONTEXPAND | VM_DONTCOPY);
 	vma->vm_ops = &arena_vm_ops;
 	return 0;
 }
 	u32 uaddr32;
 	int ret, i;
+	if (node_id != NUMA_NO_NODE &&
+	    ((unsigned int)node_id >= nr_node_ids || !node_online(node_id)))
+		return 0;
 	if (page_cnt > page_cnt_max)
 		return 0;

kernel/bpf/bpf_struct_ops.c+4−3

 			goto reset_unlock;
 		}
-		/* Poison pointer on error instead of return for backward compatibility */
-		bpf_prog_assoc_struct_ops(prog, &st_map->map);
 		link = kzalloc_obj(*link, GFP_USER);
 		if (!link) {
 			bpf_prog_put(prog);
 			      &bpf_struct_ops_link_lops, prog, prog->expected_attach_type);
 		*plink++ = &link->link;
+		/* Poison pointer on error instead of return for backward compatibility */
+		bpf_prog_assoc_struct_ops(prog, &st_map->map);
 		ksym = kzalloc_obj(*ksym, GFP_USER);
 		if (!ksym) {
 			err = -ENOMEM;
 reset_unlock:
 	bpf_struct_ops_map_free_ksyms(st_map);
 	bpf_struct_ops_map_free_image(st_map);
+	bpf_struct_ops_map_dissoc_progs(st_map);
 	bpf_struct_ops_map_put_progs(st_map);
 	memset(uvalue, 0, map->value_size);
 	memset(kvalue, 0, map->value_size);

kernel/bpf/core.c+99−39

@@ -1491,24 +1491,11 @@ void bpf_jit_prog_release_other(struct bpf_prog fp, struct bpf_prog fp_other) · 1490 unmodified lines
1491	bpf_prog_clone_free(fp_other);	1491	bpf_prog_clone_free(fp_other);
1492	}	1492	}
1493		1493
1494	static void adjust_insn_arrays(struct bpf_prog *prog, u32 off, u32 len)	1494	/*
1495	{	1495	* Now this function is used only to blind the main prog and must be invoked only when
1496	#ifdef CONFIG_BPF_SYSCALL	1496	* bpf_prog_need_blind() returns true.
1497	struct bpf_map *map;	1497	*/
1498	int i;	1498	struct bpf_prog bpf_jit_blind_constants(struct bpf_verifier_env env, struct bpf_prog *prog)
1499
1500	if (len <= 1)
1501	return;
1502
1503	for (i = 0; i < prog->aux->used_map_cnt; i++) {
1504	map = prog->aux->used_maps[i];
1505	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY)
1506	bpf_insn_array_adjust(map, off, len);
1507	}
1508	#endif
1509	}
1510
1511	struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog)
1512	{	1499	{
1513	struct bpf_insn insn_buff[16], aux[2];	1500	struct bpf_insn insn_buff[16], aux[2];
1514	struct bpf_prog clone, tmp;	1501	struct bpf_prog clone, tmp;
@@ -1516,13 +1503,17 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 1 unmodified line
1516	struct bpf_insn *insn;	1503	struct bpf_insn *insn;
1517	int i, rewritten;	1504	int i, rewritten;
1518		1505
1519	if (!prog->blinding_requested \|\| prog->blinded)	1506	if (WARN_ON_ONCE(env && env->prog != prog))
1520	return prog;	1507	return ERR_PTR(-EINVAL);
1521		1508
1522	clone = bpf_prog_clone_create(prog, GFP_USER);	1509	clone = bpf_prog_clone_create(prog, GFP_USER);
1523	if (!clone)	1510	if (!clone)
1524	return ERR_PTR(-ENOMEM);	1511	return ERR_PTR(-ENOMEM);
1525		1512
		1513	/* make sure bpf_patch_insn_data() patches the correct prog */
		1514	if (env)
		1515	env->prog = clone;
		1516
1526	insn_cnt = clone->len;	1517	insn_cnt = clone->len;
1527	insn = clone->insnsi;	1518	insn = clone->insnsi;
1528		1519
@@ -1550,21 +1541,28 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 21 unmodified lines
1550	if (!rewritten)	1541	if (!rewritten)
1551	continue;	1542	continue;
1552		1543
1553	tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);	1544	if (env)
1554	if (IS_ERR(tmp)) {	1545	tmp = bpf_patch_insn_data(env, i, insn_buff, rewritten);
		1546	else
		1547	tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
		1548
		1549	if (IS_ERR_OR_NULL(tmp)) {
		1550	if (env)
		1551	/* restore the original prog */
		1552	env->prog = prog;
1555	/* Patching may have repointed aux->prog during	1553	/* Patching may have repointed aux->prog during
1556	* realloc from the original one, so we need to	1554	* realloc from the original one, so we need to
1557	* fix it up here on error.	1555	* fix it up here on error.
1558	*/	1556	*/
1559	bpf_jit_prog_release_other(prog, clone);	1557	bpf_jit_prog_release_other(prog, clone);
1560	return tmp;	1558	return IS_ERR(tmp) ? tmp : ERR_PTR(-ENOMEM);
1561	}	1559	}
1562		1560
1563	clone = tmp;	1561	clone = tmp;
1564	insn_delta = rewritten - 1;	1562	insn_delta = rewritten - 1;
1565		1563
1566	/* Instructions arrays must be updated using absolute xlated offsets */	1564	if (env)
1567	adjust_insn_arrays(clone, prog->aux->subprog_start + i, rewritten);	1565	env->prog = clone;
1568		1566
1569	/* Walk new program and skip insns we just inserted. */	1567	/* Walk new program and skip insns we just inserted. */
1570	insn = clone->insnsi + i + insn_delta;	1568	insn = clone->insnsi + i + insn_delta;
@@ -1575,6 +1573,15 @@ struct bpf_prog bpf_jit_blind_constants(struct bpf_prog prog) · 4 unmodified lines
1575	clone->blinded = 1;	1573	clone->blinded = 1;
1576	return clone;	1574	return clone;
1577	}	1575	}
		1576
		1577	bool bpf_insn_is_indirect_target(const struct bpf_verifier_env env, const struct bpf_prog prog,
		1578	int insn_idx)
		1579	{
		1580	if (!env)
		1581	return false;
		1582	insn_idx += prog->aux->subprog_start;
		1583	return env->insn_aux_data[insn_idx].indirect_target;
		1584	}
1578	#endif /* CONFIG_BPF_JIT */	1585	#endif /* CONFIG_BPF_JIT */
1579		1586
1580	/* Base function for offset calculation. Needs to go into .text section,	1587	/* Base function for offset calculation. Needs to go into .text section,
@@ -2533,18 +2540,55 @@ static bool bpf_prog_select_interpreter(struct bpf_prog *fp) · 952 unmodified lines
2533	return select_interpreter;	2540	return select_interpreter;
2534	}	2541	}
2535		2542
2536	/**	2543	static struct bpf_prog bpf_prog_jit_compile(struct bpf_verifier_env env, struct bpf_prog *prog)
2537	* bpf_prog_select_runtime - select exec runtime for BPF program	2544	{
2538	* @fp: bpf_prog populated with BPF program	2545	#ifdef CONFIG_BPF_JIT
2539	* @err: pointer to error variable	2546	struct bpf_prog *orig_prog;
2540	*	2547	struct bpf_insn_aux_data *orig_insn_aux;
2541	* Try to JIT eBPF program, if JIT is not available, use interpreter.	2548
2542	* The BPF program will be executed via bpf_prog_run() function.	2549	if (!bpf_prog_need_blind(prog))
2543	*	2550	return bpf_int_jit_compile(env, prog);
2544	* Return: the &fp argument along with &err set to 0 for success or	2551
2545	* a negative errno code on failure	2552	if (env) {
2546	*/	2553	/*
2547	struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err)	2554	* If env is not NULL, we are called from the end of bpf_check(), at this
		2555	* point, only insn_aux_data is used after failure, so it should be restored
		2556	* on failure.
		2557	*/
		2558	orig_insn_aux = bpf_dup_insn_aux_data(env);
		2559	if (!orig_insn_aux)
		2560	return prog;
		2561	}
		2562
		2563	orig_prog = prog;
		2564	prog = bpf_jit_blind_constants(env, prog);
		2565	/*
		2566	* If blinding was requested and we failed during blinding, we must fall
		2567	* back to the interpreter.
		2568	*/
		2569	if (IS_ERR(prog))
		2570	goto out_restore;
		2571
		2572	prog = bpf_int_jit_compile(env, prog);
		2573	if (prog->jited) {
		2574	bpf_jit_prog_release_other(prog, orig_prog);
		2575	if (env)
		2576	vfree(orig_insn_aux);
		2577	return prog;
		2578	}
		2579
		2580	bpf_jit_prog_release_other(orig_prog, prog);
		2581
		2582	out_restore:
		2583	prog = orig_prog;
		2584	if (env)
		2585	bpf_restore_insn_aux_data(env, orig_insn_aux);
		2586	#endif
		2587	return prog;
		2588	}
		2589
		2590	struct bpf_prog __bpf_prog_select_runtime(struct bpf_verifier_env env, struct bpf_prog *fp,
		2591	int *err)
2548	{	2592	{
2549	/* In case of BPF to BPF calls, verifier did all the prep	2593	/* In case of BPF to BPF calls, verifier did all the prep
2550	* work with regards to JITing, etc.	2594	* work with regards to JITing, etc.
@@ -2572,7 +2616,7 @@ struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err) · 21 unmodified lines
2572	if (*err)	2616	if (*err)
2573	return fp;	2617	return fp;
2574		2618
2575	fp = bpf_int_jit_compile(fp);	2619	fp = bpf_prog_jit_compile(env, fp);
2576	bpf_prog_jit_attempt_done(fp);	2620	bpf_prog_jit_attempt_done(fp);
2577	if (!fp->jited && jit_needed) {	2621	if (!fp->jited && jit_needed) {
2578	*err = -ENOTSUPP;	2622	*err = -ENOTSUPP;
@@ -2598,6 +2642,22 @@ finalize: · 19 unmodified lines
2598		2642
2599	return fp;	2643	return fp;
2600	}	2644	}
		2645
		2646	/**
		2647	* bpf_prog_select_runtime - select exec runtime for BPF program
		2648	* @fp: bpf_prog populated with BPF program
		2649	* @err: pointer to error variable
		2650	*
		2651	* Try to JIT eBPF program, if JIT is not available, use interpreter.
		2652	* The BPF program will be executed via bpf_prog_run() function.
		2653	*
		2654	* Return: the &fp argument along with &err set to 0 for success or
		2655	* a negative errno code on failure
		2656	*/
		2657	struct bpf_prog bpf_prog_select_runtime(struct bpf_prog fp, int *err)
		2658	{
		2659	return __bpf_prog_select_runtime(NULL, fp, err);
		2660	}
2601	EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);	2661	EXPORT_SYMBOL_GPL(bpf_prog_select_runtime);
2602		2662
2603	static unsigned int __bpf_prog_ret1(const void *ctx,	2663	static unsigned int __bpf_prog_ret1(const void *ctx,
@@ -3085,7 +3145,7 @@ const struct bpf_func_proto bpf_tail_call_proto = { · 481 unmodified lines
3085	* It is encouraged to implement bpf_int_jit_compile() instead, so that	3145	* It is encouraged to implement bpf_int_jit_compile() instead, so that
3086	* eBPF and implicitly also cBPF can get JITed!	3146	* eBPF and implicitly also cBPF can get JITed!
3087	*/	3147	*/
3088	struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog)	3148	struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_prog prog)
3089	{	3149	{
3090	return prog;	3150	return prog;
3091	}	3151	}

kernel/bpf/fixups.c+138−25

@@ -183,6 +183,18 @@ static void adjust_insn_aux_data(struct bpf_verifier_env *env, · 182 unmodified lines
183	data[i].seen = old_seen;	183	data[i].seen = old_seen;
184	data[i].zext_dst = insn_has_def32(insn + i);	184	data[i].zext_dst = insn_has_def32(insn + i);
185	}	185	}
		186
		187	/*
		188	* The indirect_target flag of the original instruction was moved to the last of the
		189	* new instructions by the above memmove and memset, but the indirect jump target is
		190	* actually the first instruction, so move it back. This also matches with the behavior
		191	* of bpf_insn_array_adjust(), which preserves xlated_off to point to the first new
		192	* instruction.
		193	*/
		194	if (data[off + cnt - 1].indirect_target) {
		195	data[off].indirect_target = 1;
		196	data[off + cnt - 1].indirect_target = 0;
		197	}
186	}	198	}
187		199
188	static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)	200	static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
@@ -232,8 +244,8 @@ static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len) · 43 unmodified lines
232	}	244	}
233	}	245	}
234		246
235	static struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,	247	struct bpf_prog bpf_patch_insn_data(struct bpf_verifier_env env, u32 off,
236	const struct bpf_insn *patch, u32 len)	248	const struct bpf_insn *patch, u32 len)
237	{	249	{
238	struct bpf_prog *new_prog;	250	struct bpf_prog *new_prog;
239	struct bpf_insn_aux_data *new_data = NULL;	251	struct bpf_insn_aux_data *new_data = NULL;
@@ -973,7 +985,47 @@ patch_insn_buf: · 733 unmodified lines
973	return 0;	985	return 0;
974	}	986	}
975		987
976	int bpf_jit_subprogs(struct bpf_verifier_env *env)	988	static u32 bpf_dup_subprog_starts(struct bpf_verifier_env env)
		989	{
		990	u32 *starts = NULL;
		991
		992	starts = kvmalloc_objs(u32, env->subprog_cnt, GFP_KERNEL_ACCOUNT);
		993	if (starts) {
		994	for (int i = 0; i < env->subprog_cnt; i++)
		995	starts[i] = env->subprog_info[i].start;
		996	}
		997	return starts;
		998	}
		999
		1000	static void bpf_restore_subprog_starts(struct bpf_verifier_env env, u32 orig_starts)
		1001	{
		1002	for (int i = 0; i < env->subprog_cnt; i++)
		1003	env->subprog_info[i].start = orig_starts[i];
		1004	/* restore the start of fake 'exit' subprog as well */
		1005	env->subprog_info[env->subprog_cnt].start = env->prog->len;
		1006	}
		1007
		1008	struct bpf_insn_aux_data bpf_dup_insn_aux_data(struct bpf_verifier_env env)
		1009	{
		1010	size_t size;
		1011	void *new_aux;
		1012
		1013	size = array_size(sizeof(struct bpf_insn_aux_data), env->prog->len);
		1014	new_aux = __vmalloc(size, GFP_KERNEL_ACCOUNT);
		1015	if (new_aux)
		1016	memcpy(new_aux, env->insn_aux_data, size);
		1017	return new_aux;
		1018	}
		1019
		1020	void bpf_restore_insn_aux_data(struct bpf_verifier_env *env,
		1021	struct bpf_insn_aux_data *orig_insn_aux)
		1022	{
		1023	/* the expanded elements are zero-filled, so no special handling is required */
		1024	vfree(env->insn_aux_data);
		1025	env->insn_aux_data = orig_insn_aux;
		1026	}
		1027
		1028	static int jit_subprogs(struct bpf_verifier_env *env)
977	{	1029	{
978	struct bpf_prog prog = env->prog, func, tmp;	1030	struct bpf_prog prog = env->prog, func, tmp;
979	int i, j, subprog_start, subprog_end = 0, len, subprog;	1031	int i, j, subprog_start, subprog_end = 0, len, subprog;
@@ -981,10 +1033,6 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 1 unmodified line
981	struct bpf_insn *insn;	1033	struct bpf_insn *insn;
982	void *old_bpf_func;	1034	void *old_bpf_func;
983	int err, num_exentries;	1035	int err, num_exentries;
984	int old_len, subprog_start_adjustment = 0;
985
986	if (env->subprog_cnt <= 1)
987	return 0;
988		1036
989	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {	1037	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
990	if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))	1038	if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
@@ -1053,10 +1101,11 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 62 unmodified lines
1053	goto out_free;	1101	goto out_free;
1054	func[i]->is_func = 1;	1102	func[i]->is_func = 1;
1055	func[i]->sleepable = prog->sleepable;	1103	func[i]->sleepable = prog->sleepable;
		1104	func[i]->blinded = prog->blinded;
1056	func[i]->aux->func_idx = i;	1105	func[i]->aux->func_idx = i;
1057	/* Below members will be freed only at prog->aux */	1106	/* Below members will be freed only at prog->aux */
1058	func[i]->aux->btf = prog->aux->btf;	1107	func[i]->aux->btf = prog->aux->btf;
1059	func[i]->aux->subprog_start = subprog_start + subprog_start_adjustment;	1108	func[i]->aux->subprog_start = subprog_start;
1060	func[i]->aux->func_info = prog->aux->func_info;	1109	func[i]->aux->func_info = prog->aux->func_info;
1061	func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;	1110	func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
1062	func[i]->aux->poke_tab = prog->aux->poke_tab;	1111	func[i]->aux->poke_tab = prog->aux->poke_tab;
@@ -1110,17 +1159,10 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 47 unmodified lines
1110	func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;	1159	func[i]->aux->exception_cb = env->subprog_info[i].is_exception_cb;
1111	func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;	1160	func[i]->aux->changes_pkt_data = env->subprog_info[i].changes_pkt_data;
1112	func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;	1161	func[i]->aux->might_sleep = env->subprog_info[i].might_sleep;
		1162	func[i]->aux->token = prog->aux->token;
1113	if (!i)	1163	if (!i)
1114	func[i]->aux->exception_boundary = env->seen_exception;	1164	func[i]->aux->exception_boundary = env->seen_exception;
1115		1165	func[i] = bpf_int_jit_compile(env, func[i]);
1116	/*
1117	* To properly pass the absolute subprog start to jit
1118	* all instruction adjustments should be accumulated
1119	*/
1120	old_len = func[i]->len;
1121	func[i] = bpf_int_jit_compile(func[i]);
1122	subprog_start_adjustment += func[i]->len - old_len;
1123
1124	if (!func[i]->jited) {	1166	if (!func[i]->jited) {
1125	err = -ENOTSUPP;	1167	err = -ENOTSUPP;
1126	goto out_free;	1168	goto out_free;
@@ -1164,7 +1206,7 @@ int bpf_jit_subprogs(struct bpf_verifier_env *env) · 37 unmodified lines
1164	}	1206	}
1165	for (i = 0; i < env->subprog_cnt; i++) {	1207	for (i = 0; i < env->subprog_cnt; i++) {
1166	old_bpf_func = func[i]->bpf_func;	1208	old_bpf_func = func[i]->bpf_func;
1167	tmp = bpf_int_jit_compile(func[i]);	1209	tmp = bpf_int_jit_compile(env, func[i]);
1168	if (tmp != func[i] \|\| func[i]->bpf_func != old_bpf_func) {	1210	if (tmp != func[i] \|\| func[i]->bpf_func != old_bpf_func) {
1169	verbose(env, "JIT doesn't support bpf-to-bpf calls\n");	1211	verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
1170	err = -ENOTSUPP;	1212	err = -ENOTSUPP;
@@ -1246,16 +1288,87 @@ out_free: · 75 unmodified lines
1246	}	1288	}
1247	kfree(func);	1289	kfree(func);
1248	out_undo_insn:	1290	out_undo_insn:
		1291	bpf_prog_jit_attempt_done(prog);
		1292	return err;
		1293	}
		1294
		1295	int bpf_jit_subprogs(struct bpf_verifier_env *env)
		1296	{
		1297	int err, i;
		1298	bool blinded = false;
		1299	struct bpf_insn *insn;
		1300	struct bpf_prog prog, orig_prog;
		1301	struct bpf_insn_aux_data *orig_insn_aux;
		1302	u32 *orig_subprog_starts;
		1303
		1304	if (env->subprog_cnt <= 1)
		1305	return 0;
		1306
		1307	prog = orig_prog = env->prog;
		1308	if (bpf_prog_need_blind(prog)) {
		1309	orig_insn_aux = bpf_dup_insn_aux_data(env);
		1310	if (!orig_insn_aux) {
		1311	err = -ENOMEM;
		1312	goto out_cleanup;
		1313	}
		1314	orig_subprog_starts = bpf_dup_subprog_starts(env);
		1315	if (!orig_subprog_starts) {
		1316	vfree(orig_insn_aux);
		1317	err = -ENOMEM;
		1318	goto out_cleanup;
		1319	}
		1320	prog = bpf_jit_blind_constants(env, prog);
		1321	if (IS_ERR(prog)) {
		1322	err = -ENOMEM;
		1323	prog = orig_prog;
		1324	goto out_restore;
		1325	}
		1326	blinded = true;
		1327	}
		1328
		1329	err = jit_subprogs(env);
		1330	if (err)
		1331	goto out_jit_err;
		1332
		1333	if (blinded) {
		1334	bpf_jit_prog_release_other(prog, orig_prog);
		1335	kvfree(orig_subprog_starts);
		1336	vfree(orig_insn_aux);
		1337	}
		1338
		1339	return 0;
		1340
		1341	out_jit_err:
		1342	if (blinded) {
		1343	bpf_jit_prog_release_other(orig_prog, prog);
		1344	/* roll back to the clean original prog */
		1345	prog = env->prog = orig_prog;
		1346	goto out_restore;
		1347	} else {
		1348	if (err != -EFAULT) {
		1349	/*
		1350	* We will fall back to interpreter mode when err is not -EFAULT, before
		1351	* that, insn->off and insn->imm should be restored to their original
		1352	* values since they were modified by jit_subprogs.
		1353	*/
		1354	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
		1355	if (!bpf_pseudo_call(insn))
		1356	continue;
		1357	insn->off = 0;
		1358	insn->imm = env->insn_aux_data[i].call_imm;
		1359	}
		1360	}
		1361	goto out_cleanup;
		1362	}
		1363
		1364	out_restore:
		1365	bpf_restore_subprog_starts(env, orig_subprog_starts);
		1366	bpf_restore_insn_aux_data(env, orig_insn_aux);
		1367	kvfree(orig_subprog_starts);
		1368	out_cleanup:
1249	/* cleanup main prog to be interpreted */	1369	/* cleanup main prog to be interpreted */
1250	prog->jit_requested = 0;	1370	prog->jit_requested = 0;
1251	prog->blinding_requested = 0;	1371	prog->blinding_requested = 0;
1252	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
1253	if (!bpf_pseudo_call(insn))
1254	continue;
1255	insn->off = 0;
1256	insn->imm = env->insn_aux_data[i].call_imm;
1257	}
1258	bpf_prog_jit_attempt_done(prog);
1259	return err;	1372	return err;
1260	}	1373	}
1261		1374

kernel/bpf/helpers.c+16−1

@@ -845,7 +845,13 @@ int bpf_bprintf_prepare(const char fmt, u32 fmt_size, const u64 raw_args, · 844 unmodified lines
845	data->buf = buffers->buf;	845	data->buf = buffers->buf;
846		846
847	for (i = 0; i < fmt_size; i++) {	847	for (i = 0; i < fmt_size; i++) {
848	if ((!isprint(fmt[i]) && !isspace(fmt[i])) \|\| !isascii(fmt[i])) {	848	unsigned char c = fmt[i];
		849
		850	/*
		851	* Permit bytes >= 0x80 in plain text so UTF-8 literals can pass
		852	* through unchanged, while still rejecting ASCII control bytes.
		853	*/
		854	if (isascii(c) && !isprint(c) && !isspace(c)) {
849	err = -EINVAL;	855	err = -EINVAL;
850	goto out;	856	goto out;
851	}	857	}
@@ -867,6 +873,15 @@ int bpf_bprintf_prepare(const char fmt, u32 fmt_size, const u64 raw_args, · 15 unmodified lines
867	* always access fmt[i + 1], in the worst case it will be a 0	873	* always access fmt[i + 1], in the worst case it will be a 0
868	*/	874	*/
869	i++;	875	i++;
		876	c = fmt[i];
		877	/*
		878	* The format parser below only understands ASCII conversion
		879	* specifiers and modifiers, so reject non-ASCII after '%'.
		880	*/
		881	if (!isascii(c)) {
		882	err = -EINVAL;
		883	goto out;
		884	}
870		885
871	/* skip optional "[0 +-][num]" width formatting field */	886	/* skip optional "[0 +-][num]" width formatting field */
872	while (fmt[i] == '0' \|\| fmt[i] == '+' \|\| fmt[i] == '-' \|\|	887	while (fmt[i] == '0' \|\| fmt[i] == '+' \|\| fmt[i] == '-' \|\|

kernel/bpf/liveness.c+62−52

  *
  *   precise {frame=N, off=V}      -- known absolute frame index and byte offset
  *        |
- *   offset-imprecise {frame=N, off=OFF_IMPRECISE}
+ *   offset-imprecise {frame=N, cnt=0}
  *        |                        -- known frame identity, unknown offset
  *   fully-imprecise {frame=ARG_IMPRECISE, mask=bitmask}
  *                                 -- unknown frame identity; .mask is a
 	ARG_IMPRECISE	= -3,	/* lost identity; .mask is arg bitmask */
 };
-#define OFF_IMPRECISE	S16_MIN	/* arg identity known but offset unknown */
 /* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */
 #define MAX_ARG_SPILL_SLOTS 64
 	return at->frame >= 0 || at->frame == ARG_IMPRECISE;
 }
-/*
- * Clear all tracked callee stack slots overlapping the byte range
- * [off, off+sz-1] where off is a negative FP-relative offset.
- */
-static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz)
-{
-	struct arg_track none = { .frame = ARG_NONE };
-	if (off == OFF_IMPRECISE) {
-		for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
-			at_stack[i] = none;
-		return;
-	}
-	for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
-		int slot_start = -((i + 1) * 8);
-		int slot_end = slot_start + 8;
-		if (slot_start < off + (int)sz && slot_end > off)
-			at_stack[i] = none;
-	}
-}
 static void verbose_arg_track(struct bpf_verifier_env *env, struct arg_track *at)
 {
 	int i;
 	*dst = arg_join_imprecise(*dst, *src);
 }
-static s16 arg_add(s16 off, s64 delta)
+static bool arg_add(s16 off, s64 delta, s16 *out)
 {
-	s64 res;
+	s16 d = delta;
-	if (off == OFF_IMPRECISE)
+	if (d != delta)
-		return OFF_IMPRECISE;
+		return true;
-	res = (s64)off + delta;
+	return check_add_overflow(off, d, out);
-	if (res < S16_MIN + 1 || res > S16_MAX)
-		return OFF_IMPRECISE;
-	return res;
 }
 static void arg_padd(struct arg_track *at, s64 delta)
 	if (at->off_cnt == 0)
 		return;
 	for (i = 0; i < at->off_cnt; i++) {
-		s16 new_off = arg_add(at->off[i], delta);
+		s16 new_off;
-		if (new_off == OFF_IMPRECISE) {
+		if (arg_add(at->off[i], delta, &new_off)) {
 			at->off_cnt = 0;
 			return;
 		}
  */
 static int fp_off_to_slot(s16 off)
 {
-	if (off == OFF_IMPRECISE)
-		return -1;
 	if (off >= 0 || off < -(int)(MAX_ARG_SPILL_SLOTS * 8))
 		return -1;
 	if (off % 8)
 		return imp;
 	for (i = 0; i < cnt; i++) {
-		s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
+		s16 fp_off, slot;
-		int slot = fp_off_to_slot(fp_off);
+		if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
+			return imp;
+		slot = fp_off_to_slot(fp_off);
 		if (slot < 0)
 			return imp;
 		result = __arg_track_join(result, at_stack_out[slot]);
 		return;
 	}
 	for (i = 0; i < cnt; i++) {
-		s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
+		s16 fp_off;
-		int slot = fp_off_to_slot(fp_off);
+		int slot;
+		if (arg_add(at_out[reg].off[i], insn->off, &fp_off))
+			continue;
+		slot = fp_off_to_slot(fp_off);
 		if (slot < 0)
 			continue;
 		if (cnt == 1)
 	}
 }
+/*
+ * Clear all tracked callee stack slots overlapping the byte range
+ * [off, off+sz-1] where off is a negative FP-relative offset.
+ */
+static void clear_overlapping_stack_slots(struct arg_track *at_stack, s16 off, u32 sz, int cnt)
+{
+	struct arg_track none = { .frame = ARG_NONE };
+	if (cnt == 0) {
+		for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++)
+			at_stack[i] = __arg_track_join(at_stack[i], none);
+		return;
+	}
+	for (int i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
+		int slot_start = -((i + 1) * 8);
+		int slot_end = slot_start + 8;
+		if (slot_start < off + (int)sz && slot_end > off) {
+			if (cnt == 1)
+				at_stack[i] = none;
+			else
+				at_stack[i] = __arg_track_join(at_stack[i], none);
+		}
+	}
+}
 /*
  * Clear stack slots overlapping all possible FP offsets in @reg.
  */
 	int cnt, i;
 	if (reg == BPF_REG_FP) {
-		clear_overlapping_stack_slots(at_stack_out, insn->off, sz);
+		clear_overlapping_stack_slots(at_stack_out, insn->off, sz, 1);
 		return;
 	}
 	cnt = at_out[reg].off_cnt;
 	if (cnt == 0) {
-		clear_overlapping_stack_slots(at_stack_out, OFF_IMPRECISE, sz);
+		clear_overlapping_stack_slots(at_stack_out, 0, sz, cnt);
 		return;
 	}
 	for (i = 0; i < cnt; i++) {
-		s16 fp_off = arg_add(at_out[reg].off[i], insn->off);
+		s16 fp_off;
-		clear_overlapping_stack_slots(at_stack_out, fp_off, sz);
+		if (arg_add(at_out[reg].off[i], insn->off, &fp_off)) {
+			clear_overlapping_stack_slots(at_stack_out, 0, sz, 0);
+			break;
+		}
+		clear_overlapping_stack_slots(at_stack_out, fp_off, sz, cnt);
 	}
 }
 		verbose(env, "\n");
 }
+static bool can_be_local_fp(int depth, int regno, struct arg_track *at)
+{
+	return regno == BPF_REG_FP || at->frame == depth ||
+	       (at->frame == ARG_IMPRECISE && (at->mask & BIT(depth)));
+}
 /*
  * Pure dataflow transfer function for arg_track state.
  * Updates at_out[] based on how the instruction modifies registers.
 			at_out[r] = none;
 	} else if (class == BPF_LDX) {
 		u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
-		bool src_is_local_fp = insn->src_reg == BPF_REG_FP || src->frame == depth ||
+		bool src_is_local_fp = can_be_local_fp(depth, insn->src_reg, src);
-				       (src->frame == ARG_IMPRECISE && (src->mask & BIT(depth)));
 		/*
 		 * Reload from callee stack: if src is current-frame FP-derived
 		bool dst_is_local_fp;
 		/* Track spills to current-frame FP-derived callee stack */
-		dst_is_local_fp = insn->dst_reg == BPF_REG_FP || dst->frame == depth;
+		dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
 		if (dst_is_local_fp && BPF_MODE(insn->code) == BPF_MEM)
 			spill_to_stack(insn, at_out, insn->dst_reg,
 				       at_stack_out, src, sz);
 		}
 	} else if (class == BPF_ST && BPF_MODE(insn->code) == BPF_MEM) {
 		u32 sz = bpf_size_to_bytes(BPF_SIZE(insn->code));
-		bool dst_is_local_fp = insn->dst_reg == BPF_REG_FP || dst->frame == depth;
+		bool dst_is_local_fp = can_be_local_fp(depth, insn->dst_reg, dst);
 		/* BPF_ST to FP-derived dst: clear overlapping stack slots */
 		if (dst_is_local_fp)
 		resolved.off_cnt = ptr->off_cnt;
 		resolved.frame = ptr->frame;
 		for (oi = 0; oi < ptr->off_cnt; oi++) {
-			resolved.off[oi] = arg_add(ptr->off[oi], insn->off);
+			if (arg_add(ptr->off[oi], insn->off, &resolved.off[oi])) {
-			if (resolved.off[oi] == OFF_IMPRECISE) {
 				resolved.off_cnt = 0;
 				break;
 			}

kernel/bpf/syscall.c+0−4

 	if (err < 0)
 		goto free_used_maps;
-	prog = bpf_prog_select_runtime(prog, &err);
-	if (err < 0)
-		goto free_used_maps;
 	err = bpf_prog_mark_insn_arrays_ready(prog);
 	if (err < 0)
 		goto free_used_maps;

kernel/bpf/verifier.c+19−7

 	return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
 }
+static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
+{
+	env->insn_aux_data[idx].indirect_target = true;
+}
 #define LR_FRAMENO_BITS	3
 #define LR_SPI_BITS	6
 #define LR_ENTRY_BITS	(LR_SPI_BITS + LR_FRAMENO_BITS + 1)
 	int perm_flags;
 	const char *reg_name = "";
+	if (base_type(reg->type) != PTR_TO_BTF_ID)
+		goto bad_type;
 	if (btf_is_kernel(reg->btf)) {
 		perm_flags = PTR_MAYBE_NULL | PTR_TRUSTED | MEM_RCU;
 			perm_flags |= MEM_PERCPU;
 	}
-	if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags))
+	if (type_flag(reg->type) & ~perm_flags)
 		goto bad_type;
 	/* We need to verify reg->type and reg->btf, before accessing reg->btf */
 	}
 	for (i = 0; i < n - 1; i++) {
+		mark_indirect_target(env, env->gotox_tmp_buf->items[i]);
 		other_branch = push_stack(env, env->gotox_tmp_buf->items[i],
 					  env->insn_idx, env->cur_state->speculative);
 		if (IS_ERR(other_branch))
 			return PTR_ERR(other_branch);
 	}
 	env->insn_idx = env->gotox_tmp_buf->items[n-1];
+	mark_indirect_target(env, env->insn_idx);
 	return INSN_IDX_UPDATED;
 }
 	adjust_btf_func(env);
+	/* extension progs temporarily inherit the attach_type of their targets
+	   for verification purposes, so set it back to zero before returning
+	 */
+	if (env->prog->type == BPF_PROG_TYPE_EXT)
+		env->prog->expected_attach_type = 0;
+	env->prog = __bpf_prog_select_runtime(env, env->prog, &ret);
 err_release_maps:
 	if (ret)
 		release_insn_arrays(env);
 	if (!env->prog->aux->used_btfs)
 		release_btfs(env);
-	/* extension progs temporarily inherit the attach_type of their targets
-	   for verification purposes, so set it back to zero before returning
-	 */
-	if (env->prog->type == BPF_PROG_TYPE_EXT)
-		env->prog->expected_attach_type = 0;
 	*prog = env->prog;
 	module_put(env->attach_btf_mod);

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code