Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull BPF fixes from Daniel Borkmann: - Initialize task local storage before fork bails out to free the task (Jann Horn) - Fix insn_aux_data leak on verifier error path (KaFai Wan) - Reject BPF inode storage map creation when BPF LSM is uninitialized (Matt Bobrowski) - Mask pseudo pointer values in verifier logs when pointer leaks are not allowed (Nuoqi Gui) - Harden BPF JIT against spraying via IBPB flush (Pawan Gupta) - Reject a skb-modifying SK_SKB stream parser since the latter is only meant to measure the next message (Sechang Lim) - Fix bpf_refcount_acquire to reject refcounted allocation arguments with a non-zero fixed offset (Yiyang Chen) * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: bpf: Prefer dirty packs for eBPF allocations bpf: Prefer packs that won't trigger an IBPB flush on allocation bpf: Skip redundant IBPB in pack allocator bpf: Restrict JIT predictor flush to cBPF x86/bugs: Enable IBPB flush on BPF JIT allocation bpf: Support for hardening against JIT spraying bpf: Reject BPF_MAP_TYPE_INODE_STORAGE creation if BPF LSM is uninitialized bpf,fork: wipe ->bpf_storage before bailouts that access it bpf: Fix insn_aux_data leak on verifier err_free_env path selftests/bpf: Cover pseudo-BTF ksym log masking bpf: Mask pseudo pointer values in verifier logs selftests/bpf: Cover refcount acquire node offsets bpf: Reject offset refcount acquire arguments selftests/bpf: test rejection of a packet-modifying SK_SKB stream parser bpf, sockmap: reject a packet-modifying SK_SKB stream parser selftests/bpf: don't modify the skb in the strparser parser prog

Linus Torvalds committed 4 months ago

Commit 51512e22 parents 826eec5 b72e29e23 files · +297 −62Browse files

Unified · Split

23 files

arch/arm64/net/bpf_jit_comp.c +2 −2
arch/loongarch/net/bpf_jit.c +3 −2
arch/powerpc/net/bpf_jit_comp.c +2 −2
arch/riscv/net/bpf_jit_comp64.c +1 −1
arch/riscv/net/bpf_jit_core.c +2 −1
arch/x86/include/asm/nospec-branch.h +4 −0
arch/x86/kernel/cpu/bugs.c +45 −5
arch/x86/net/bpf_jit_comp.c +3 −2
include/linux/bpf_lsm.h +4 −0
include/linux/filter.h +13 −2
kernel/bpf/bpf_inode_storage.c +9 −0
kernel/bpf/core.c +62 −6
kernel/bpf/disasm.c +4 −1
kernel/bpf/dispatcher.c +1 −1
kernel/bpf/verifier.c +23 −11
kernel/fork.c +5 −4
net/core/sock_map.c +20 −0
security/bpf/hooks.c +3 −0
tools/testing/selftests/bpf/prog_tests/sockmap_strp.c +31 −0
tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +34 −0
tools/testing/selftests/bpf/progs/sockmap_parse_prog.c +0 −22
tools/testing/selftests/bpf/progs/test_sockmap_strp.c +7 −0
tools/testing/selftests/bpf/progs/verifier_unpriv.c +19 −0

arch/arm64/net/bpf_jit_comp.c+2−2

@@ -2177,7 +2177,7 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_pr · 2176 unmodified lines
2177	2177	image_size = extable_offset + extable_size;
2178	2178	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,
2179	2179	sizeof(u64), &header, &image_ptr,
2180		− jit_fill_hole);
	2180	+ jit_fill_hole, was_classic);
2181	2181	if (!ro_header)
2182	2182	goto out_off;
2183	2183
@@ -2870,7 +2870,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 686 unmodified lines
2870	2870
2871	2871	void *arch_alloc_bpf_trampoline(unsigned int size)
2872	2872	{
2873		− return bpf_prog_pack_alloc(size, jit_fill_hole);
	2873	+ return bpf_prog_pack_alloc(size, jit_fill_hole, false);
2874	2874	}
2875	2875
2876	2876	void arch_free_bpf_trampoline(void *image, unsigned int size)

arch/loongarch/net/bpf_jit.c+3−2

@@ -1790,7 +1790,7 @@ static int invoke_bpf(struct jit_ctx ctx, struct bpf_tramp_nodes tn, · 1789 unmodified lines
1790	1790
1791	1791	void *arch_alloc_bpf_trampoline(unsigned int size)
1792	1792	{
1793		− return bpf_prog_pack_alloc(size, jit_fill_hole);
	1793	+ return bpf_prog_pack_alloc(size, jit_fill_hole, false);
1794	1794	}
1795	1795
1796	1796	void arch_free_bpf_trampoline(void *image, unsigned int size)
@@ -2256,7 +2256,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_pr · 459 unmodified lines
2256	2256	image_size = prog_size + extable_size;
2257	2257	/* Now we know the size of the structure to make */
2258	2258	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
2259		− &header, &image_ptr, jit_fill_hole);
	2259	+ &header, &image_ptr, jit_fill_hole,
	2260	+ bpf_prog_was_classic(prog));
2260	2261	if (!ro_header)
2261	2262	goto out_offset;
2262	2263

arch/powerpc/net/bpf_jit_comp.c+2−2

@@ -295,7 +295,7 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_pr · 294 unmodified lines
295	295	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
296	296
297	297	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
298		− bpf_jit_fill_ill_insns);
	298	+ bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp));
299	299	if (!fhdr)
300	300	goto out_err;
301	301
@@ -588,7 +588,7 @@ bool bpf_jit_inlines_helper_call(s32 imm) · 286 unmodified lines
588	588
589	589	void *arch_alloc_bpf_trampoline(unsigned int size)
590	590	{
591		− return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns);
	591	+ return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false);
592	592	}
593	593
594	594	void arch_free_bpf_trampoline(void *image, unsigned int size)

arch/riscv/net/bpf_jit_comp64.c+1−1

@@ -1321,7 +1321,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 1320 unmodified lines
1321	1321
1322	1322	void *arch_alloc_bpf_trampoline(unsigned int size)
1323	1323	{
1324		− return bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
	1324	+ return bpf_prog_pack_alloc(size, bpf_fill_ill_insns, false);
1325	1325	}
1326	1326
1327	1327	void arch_free_bpf_trampoline(void *image, unsigned int size)

arch/riscv/net/bpf_jit_core.c+2−1

@@ -109,7 +109,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_pr · 108 unmodified lines
109	109	bpf_jit_binary_pack_alloc(prog_size + extable_size,
110	110	&jit_data->ro_image, sizeof(u32),
111	111	&jit_data->header, &jit_data->image,
112		− bpf_fill_ill_insns);
	112	+ bpf_fill_ill_insns,
	113	+ bpf_prog_was_classic(prog));
113	114	if (!jit_data->ro_header)
114	115	goto out_offset;
115	116

arch/x86/include/asm/nospec-branch.h+4−0

@@ -388,6 +388,10 @@ extern void srso_alias_return_thunk(void); · 387 unmodified lines
388	388	extern void entry_untrain_ret(void);
389	389	extern void write_ibpb(void);
390	390
	391	+#ifdef CONFIG_BPF_JIT
	392	+extern void bpf_arch_ibpb(void);
	393	+#endif
	394	+
391	395	#ifdef CONFIG_X86_64
392	396	extern void clear_bhb_loop(void);
393	397	#endif

arch/x86/kernel/cpu/bugs.c+45−5

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#include <linux/sched/smt.h>
17	17	#include <linux/pgtable.h>
18	18	#include <linux/bpf.h>
	19	+#include <linux/filter.h>
19	20	#include <linux/kvm_types.h>
20	21
21	22	#include <asm/spec-ctrl.h>
@@ -1651,8 +1652,21 @@ static inline const char *spectre_v2_module_string(void) · 1629 unmodified lines
1651	1652	{
1652	1653	return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
1653	1654	}
	1655	+
	1656	+/*
	1657	+ * The "retpoline sequence" is the "call;mov;ret" sequence that
	1658	+ * replaces normal indirect branch instructions. Differentiate
	1659	+ * the retpoline sequence from the LFENCE-prefixed indirect
	1660	+ * branches that simply use the retpoline infrastructure.
	1661	+ */
	1662	+static inline bool retpoline_seq_enabled(void)
	1663	+{
	1664	+ return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE);
	1665	+}
	1666	+
1654	1667	#else
1655	1668	static inline const char *spectre_v2_module_string(void) { return ""; }
	1669	+static inline bool retpoline_seq_enabled(void) { return false; }
1656	1670	#endif
1657	1671
1658	1672	#define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
@@ -2095,8 +2109,7 @@ static void __init bhi_apply_mitigation(void) · 436 unmodified lines
2095	2109	return;
2096	2110
2097	2111	/* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
2098		− if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
2099		− !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
	2112	+ if (retpoline_seq_enabled()) {
2100	2113	spec_ctrl_disable_kernel_rrsba();
2101	2114	if (rrsba_disabled)
2102	2115	return;
@@ -2238,6 +2251,27 @@ static void __init spectre_v2_update_mitigation(void) · 135 unmodified lines
2238	2251	pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]);
2239	2252	}
2240	2253
	2254	+#ifdef CONFIG_BPF_JIT
	2255	+static void __bpf_arch_ibpb(void *unused)
	2256	+{
	2257	+ write_ibpb();
	2258	+}
	2259	+
	2260	+void bpf_arch_ibpb(void)
	2261	+{
	2262	+ on_each_cpu(__bpf_arch_ibpb, NULL, 1);
	2263	+}
	2264	+
	2265	+static bool __init cpu_wants_ibpb_bpf(void)
	2266	+{
	2267	+ /* A genuine retpoline already neutralizes ring0 indirect predictions */
	2268	+ if (retpoline_seq_enabled())
	2269	+ return false;
	2270	+
	2271	+ return boot_cpu_has(X86_FEATURE_IBPB);
	2272	+}
	2273	+#endif
	2274	+
2241	2275	static void __init spectre_v2_apply_mitigation(void)
2242	2276	{
2243	2277	if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
@@ -2314,6 +2348,14 @@ static void __init spectre_v2_apply_mitigation(void) · 70 unmodified lines
2314	2348	setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
2315	2349	pr_info("Enabling Restricted Speculation for firmware calls\n");
2316	2350	}
	2351	+
	2352	+#ifdef CONFIG_BPF_JIT
	2353	+ if (cpu_wants_ibpb_bpf()) {
	2354	+ static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb);
	2355	+ static_branch_enable(&bpf_pred_flush_enabled);
	2356	+ pr_info("Enabling IBPB for BPF\n");
	2357	+ }
	2358	+#endif
2317	2359	}
2318	2360
2319	2361	static void update_stibp_msr(void * __unused)
@@ -3490,9 +3532,7 @@ static const char *spectre_bhi_state(void) · 1170 unmodified lines
3490	3532	return "; BHI: BHI_DIS_S";
3491	3533	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
3492	3534	return "; BHI: SW loop, KVM: SW loop";
3493		− else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
3494		− !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
3495		− rrsba_disabled)
	3535	+ else if (retpoline_seq_enabled() && rrsba_disabled)
3496	3536	return "; BHI: Retpoline";
3497	3537	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT))
3498	3538	return "; BHI: Vulnerable, KVM: SW loop";

arch/x86/net/bpf_jit_comp.c+3−2

@@ -3653,7 +3653,7 @@ cleanup: · 3652 unmodified lines
3653	3653
3654	3654	void *arch_alloc_bpf_trampoline(unsigned int size)
3655	3655	{
3656		− return bpf_prog_pack_alloc(size, jit_fill_hole);
	3656	+ return bpf_prog_pack_alloc(size, jit_fill_hole, false);
3657	3657	}
3658	3658
3659	3659	void arch_free_bpf_trampoline(void *image, unsigned int size)
@@ -3965,7 +3965,8 @@ out_image: · 305 unmodified lines
3965	3965	/* allocate module memory for x86 insns and extable */
3966	3966	header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
3967	3967	&image, align, &rw_header, &rw_image,
3968		− jit_fill_hole);
	3968	+ jit_fill_hole,
	3969	+ bpf_prog_was_classic(prog));
3969	3970	if (!header)
3970	3971	goto out_addrs;
3971	3972	prog->aux->extable = (void *) image + roundup(proglen, align);

include/linux/bpf_lsm.h+4−0

@@ -14,6 +14,8 @@ · 13 unmodified lines
14	14
15	15	#ifdef CONFIG_BPF_LSM
16	16
	17	+extern bool bpf_lsm_initialized __ro_after_init;
	18	+
17	19	#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
18	20	RET bpf_lsm_##NAME(__VA_ARGS__);
19	21	#include <linux/lsm_hook_defs.h>
@@ -56,6 +58,8 @@ bool bpf_lsm_hook_returns_errno(u32 btf_id); · 36 unmodified lines
56	58
57	59	#else /* !CONFIG_BPF_LSM */
58	60
	61	+#define bpf_lsm_initialized false
	62	+
59	63	static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
60	64	{
61	65	return false;

include/linux/filter.h+13−2

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#include <linux/if_vlan.h>
22	22	#include <linux/vmalloc.h>
23	23	#include <linux/sockptr.h>
	24	+#include <linux/static_call.h>
24	25	#include <linux/u64_stats_sync.h>
25	26
26	27	#include <net/sch_generic.h>
@@ -1314,6 +1315,15 @@ extern long bpf_jit_limit_max; · 1287 unmodified lines
1314	1315
1315	1316	typedef void (bpf_jit_fill_hole_t)(void area, unsigned int size);
1316	1317
	1318	+/*
	1319	+ * Flush the indirect branch predictors before reusing JIT memory, so that
	1320	+ * indirect jumps into a newly written program don't reuse predictions left
	1321	+ * behind by an old program that occupied the same space.
	1322	+ */
	1323	+void bpf_arch_pred_flush(void);
	1324	+DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush);
	1325	+DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled);
	1326	+
1317	1327	void bpf_jit_fill_hole_with_zero(void *area, unsigned int size);
1318	1328
1319	1329	struct bpf_binary_header *
@@ -1328,7 +1338,7 @@ void bpf_jit_free(struct bpf_prog *fp); · 8 unmodified lines
1328	1338	struct bpf_binary_header *
1329	1339	bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);
1330	1340
1331		−void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns);
	1341	+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic);
1332	1342	void bpf_prog_pack_free(void *ptr, u32 size);
1333	1343
1334	1344	static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
@@ -1342,7 +1352,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, · 7 unmodified lines
1342	1352	unsigned int alignment,
1343	1353	struct bpf_binary_header **rw_hdr,
1344	1354	u8 **rw_image,
1345		− bpf_jit_fill_hole_t bpf_fill_ill_insns);
	1355	+ bpf_jit_fill_hole_t bpf_fill_ill_insns,
	1356	+ bool was_classic);
1346	1357	int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header,
1347	1358	struct bpf_binary_header *rw_header);
1348	1359	void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header,

kernel/bpf/bpf_inode_storage.c+9−0

@@ -178,6 +178,15 @@ static int notsupp_get_next_key(struct bpf_map map, void key, · 177 unmodified lines
178	178
179	179	static struct bpf_map inode_storage_map_alloc(union bpf_attr attr)
180	180	{
	181	+ /*
	182	+ * Do not allow allocation of BPF_MAP_TYPE_INODE_STORAGE if the BPF LSM
	183	+ * was not initialized by the LSM framework at boot. Without proper
	184	+ * initialization, the BPF inode security blob offset remains unprepared,
	185	+ * causing bpf_inode() to calculate an invalid memory offset and corrupt
	186	+ * inode->i_security.
	187	+ */
	188	+ if (!bpf_lsm_initialized)
	189	+ return ERR_PTR(-EOPNOTSUPP);
181	190	return bpf_local_storage_map_alloc(attr, &inode_cache);
182	191	}
183	192

kernel/bpf/core.c+62−6

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	20	#include <uapi/linux/btf.h>
21	21	#include <linux/filter.h>
22	22	#include <linux/skbuff.h>
	23	+#include <linux/static_call.h>
23	24	#include <linux/vmalloc.h>
24	25	#include <linux/prandom.h>
25	26	#include <linux/bpf.h>
@@ -875,6 +876,7 @@ int bpf_jit_add_poke_descriptor(struct bpf_prog *prog, · 849 unmodified lines
875	876	struct bpf_prog_pack {
876	877	struct list_head list;
877	878	void *ptr;
	879	+ bool arch_flush_needed;
878	880	unsigned long bitmap[];
879	881	};
880	882
@@ -883,6 +885,15 @@ void bpf_jit_fill_hole_with_zero(void *area, unsigned int size) · 2 unmodified lines
883	885	memset(area, 0, size);
884	886	}
885	887
	888	+DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush);
	889	+
	890	+/*
	891	+ * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the
	892	+ * pack allocator test "is a predictor flush wired up at all" with a cheap
	893	+ * static branch instead of repeatedly querying the static call target.
	894	+ */
	895	+DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled);
	896	+
886	897	#define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)
887	898
888	899	static DEFINE_MUTEX(pack_mutex);
@@ -918,6 +929,8 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins · 29 unmodified lines
918	929	bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
919	930	bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
920	931
	932	+ if (static_branch_unlikely(&bpf_pred_flush_enabled))
	933	+ pack->arch_flush_needed = true;
921	934	set_vm_flush_reset_perms(pack->ptr);
922	935	err = set_memory_rox((unsigned long)pack->ptr,
923	936	BPF_PROG_PACK_SIZE / PAGE_SIZE);
@@ -932,15 +945,23 @@ out: · 8 unmodified lines
932	945	return NULL;
933	946	}
934	947
935		−void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
	948	+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic)
936	949	{
937	950	unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size);
938		− struct bpf_prog_pack *pack;
939		− unsigned long pos;
	951	+ struct bpf_prog_pack pack, fallback_pack = NULL;
	952	+ unsigned long pos, fallback_pos = 0;
940	953	void *ptr = NULL;
941	954
942	955	mutex_lock(&pack_mutex);
943	956	if (size > BPF_PROG_PACK_SIZE) {
	957	+ /*
	958	+ * Allocations larger than a pack get their own pages, and
	959	+ * predictors are not flushed for such allocation. This is only
	960	+ * safe because cBPF programs (the unprivileged attack surface)
	961	+ * are bounded well below a pack size.
	962	+ */
	963	+ if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled))
	964	+ pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n");
944	965	size = round_up(size, PAGE_SIZE);
945	966	ptr = bpf_jit_alloc_exec(size);
946	967	if (ptr) {
@@ -960,8 +981,29 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) · 13 unmodified lines
960	981	list_for_each_entry(pack, &pack_list, list) {
961	982	pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
962	983	nbits, 0);
963		− if (pos < BPF_PROG_CHUNK_COUNT)
	984	+ if (pos >= BPF_PROG_CHUNK_COUNT)
	985	+ continue;
	986	+ /* Flush not enabled, use any pack */
	987	+ if (!static_branch_unlikely(&bpf_pred_flush_enabled))
964	988	goto found_free_area;
	989	+ /*
	990	+ * cBPF reuse of a dirty pack triggers a flush, so prefer a
	991	+ * clean pack for cBPF. eBPF never flushes, so steer it to a
	992	+ * dirty pack and keep clean packs free for cBPF.
	993	+ */
	994	+ if (was_classic ^ pack->arch_flush_needed)
	995	+ goto found_free_area;
	996	+ if (!fallback_pack) {
	997	+ fallback_pack = pack;
	998	+ fallback_pos = pos;
	999	+ }
	1000	+ }
	1001	+
	1002	+ /* No preferred pack found */
	1003	+ if (fallback_pack) {
	1004	+ pack = fallback_pack;
	1005	+ pos = fallback_pos;
	1006	+ goto found_free_area;
965	1007	}
966	1008
967	1009	pack = alloc_new_pack(bpf_fill_ill_insns);
@@ -971,6 +1013,16 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) · 3 unmodified lines
971	1013	pos = 0;
972	1014
973	1015	found_free_area:
	1016	+ /* Flush only for cBPF as it may contain a crafted gadget */
	1017	+ if (static_branch_unlikely(&bpf_pred_flush_enabled) &&
	1018	+ pack->arch_flush_needed &&
	1019	+ was_classic) {
	1020	+ struct bpf_prog_pack *p;
	1021	+
	1022	+ static_call_cond(bpf_arch_pred_flush)();
	1023	+ list_for_each_entry(p, &pack_list, list)
	1024	+ p->arch_flush_needed = false;
	1025	+ }
974	1026	bitmap_set(pack->bitmap, pos, nbits);
975	1027	ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT);
976	1028
@@ -1008,6 +1060,9 @@ void bpf_prog_pack_free(void *ptr, u32 size) · 31 unmodified lines
1008	1060	"bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");
1009	1061
1010	1062	bitmap_clear(pack->bitmap, pos, nbits);
	1063	+
	1064	+ if (static_branch_unlikely(&bpf_pred_flush_enabled))
	1065	+ pack->arch_flush_needed = true;
1011	1066	if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
1012	1067	BPF_PROG_CHUNK_COUNT, 0) == 0) {
1013	1068	list_del(&pack->list);
@@ -1130,7 +1185,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, · 116 unmodified lines
1130	1185	unsigned int alignment,
1131	1186	struct bpf_binary_header **rw_header,
1132	1187	u8 **rw_image,
1133		− bpf_jit_fill_hole_t bpf_fill_ill_insns)
	1188	+ bpf_jit_fill_hole_t bpf_fill_ill_insns,
	1189	+ bool was_classic)
1134	1190	{
1135	1191	struct bpf_binary_header *ro_header;
1136	1192	u32 size, hole, start;
@@ -1143,7 +1199,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr, · 6 unmodified lines
1143	1199
1144	1200	if (bpf_jit_charge_modmem(size))
1145	1201	return NULL;
1146		− ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
	1202	+ ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic);
1147	1203	if (!ro_header) {
1148	1204	bpf_jit_uncharge_modmem(size);
1149	1205	return NULL;

kernel/bpf/disasm.c+4−1

@@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 322 unmodified lines
323	323	*/
324	324	u64 imm = ((u64)(insn + 1)->imm << 32) \| (u32)insn->imm;
325	325	bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD \|\|
326		− insn->src_reg == BPF_PSEUDO_MAP_VALUE;
	326	+ insn->src_reg == BPF_PSEUDO_MAP_VALUE \|\|
	327	+ insn->src_reg == BPF_PSEUDO_MAP_IDX \|\|
	328	+ insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE \|\|
	329	+ insn->src_reg == BPF_PSEUDO_BTF_ID;
327	330	char tmp[64];
328	331
329	332	if (is_ptr && !allow_ptr_leaks)

kernel/bpf/dispatcher.c+1−1

@@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher d, struct bpf_prog from, · 144 unmodified lines
145	145
146	146	mutex_lock(&d->mutex);
147	147	if (!d->image) {
148		− d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
	148	+ d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false);
149	149	if (!d->image)
150	150	goto out;
151	151	d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);

kernel/bpf/verifier.c+23−11

@@ -7996,9 +7996,10 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields) · 7995 unmodified lines
7996	7996	return field;
7997	7997	}
7998	7998
7999		−static int check_func_arg_reg_off(struct bpf_verifier_env *env,
8000		− const struct bpf_reg_state *reg, argno_t argno,
8001		− enum bpf_arg_type arg_type)
	7999	+static int __check_func_arg_reg_off(struct bpf_verifier_env *env,
	8000	+ const struct bpf_reg_state *reg, argno_t argno,
	8001	+ enum bpf_arg_type arg_type,
	8002	+ bool btf_id_fixed_off_ok)
8002	8003	{
8003	8004	u32 type = reg->type;
8004	8005
@@ -8055,12 +8056,11 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, · 50 unmodified lines
8055	8056	case PTR_TO_BTF_ID \| MEM_ALLOC \| NON_OWN_REF \| MEM_RCU:
8056	8057	/* When referenced PTR_TO_BTF_ID is passed to release function,
8057	8058	* its fixed offset must be 0. In the other cases, fixed offset
8058		− * can be non-zero. This was already checked above. So pass
8059		− * fixed_off_ok as true to allow fixed offset for all other
8060		− * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we
8061		− * still need to do checks instead of returning.
	8059	+ * can be non-zero unless the caller requires otherwise.
	8060	+ * var_off always must be 0 for PTR_TO_BTF_ID, hence we still
	8061	+ * need to do checks instead of returning.
8062	8062	*/
8063		− return __check_ptr_off_reg(env, reg, argno, true);
	8063	+ return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok);
8064	8064	case PTR_TO_CTX:
8065	8065	/*
8066	8066	* Allow fixed and variable offsets for syscall context, but
@@ -8076,6 +8076,13 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env, · 9 unmodified lines
8076	8076	}
8077	8077	}
8078	8078
	8079	+static int check_func_arg_reg_off(struct bpf_verifier_env *env,
	8080	+ const struct bpf_reg_state *reg, argno_t argno,
	8081	+ enum bpf_arg_type arg_type)
	8082	+{
	8083	+ return __check_func_arg_reg_off(env, reg, argno, arg_type, true);
	8084	+}
	8085	+
8079	8086	static int check_arg_const_str(struct bpf_verifier_env *env,
8080	8087	struct bpf_reg_state *reg, argno_t argno)
8081	8088	{
@@ -11947,6 +11954,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 3865 unmodified lines
11947	11954	enum bpf_arg_type arg_type = ARG_DONTCARE;
11948	11955	argno_t argno = argno_from_arg(i + 1);
11949	11956	int regno = reg_from_argno(argno);
	11957	+ bool btf_id_fixed_off_ok = true;
11950	11958	u32 ref_id, type_size;
11951	11959	bool is_ret_buf_sz = false;
11952	11960	int kf_arg_type;
@@ -12120,7 +12128,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 167 unmodified lines
12120	12128	case KF_ARG_PTR_TO_MEM:
12121	12129	case KF_ARG_PTR_TO_MEM_SIZE:
12122	12130	case KF_ARG_PTR_TO_CALLBACK:
12123		− case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
12124	12131	case KF_ARG_PTR_TO_CONST_STR:
12125	12132	case KF_ARG_PTR_TO_WORKQUEUE:
12126	12133	case KF_ARG_PTR_TO_TIMER:
@@ -12134,6 +12141,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 7 unmodified lines
12134	12141	case KF_ARG_PTR_TO_CTX:
12135	12142	arg_type = ARG_PTR_TO_CTX;
12136	12143	break;
	12144	+ case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
	12145	+ arg_type = ARG_PTR_TO_BTF_ID;
	12146	+ btf_id_fixed_off_ok = false;
	12147	+ break;
12137	12148	default:
12138	12149	verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type);
12139	12150	return -EFAULT;
@@ -12141,7 +12152,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ · 1 unmodified line
12141	12152
12142	12153	if (regno == meta->release_regno)
12143	12154	arg_type \|= OBJ_RELEASE;
12144		− ret = check_func_arg_reg_off(env, reg, argno, arg_type);
	12155	+ ret = __check_func_arg_reg_off(env, reg, argno, arg_type,
	12156	+ btf_id_fixed_off_ok);
12145	12157	if (ret < 0)
12146	12158	return ret;
12147	12159
@@ -19994,13 +20006,13 @@ err_unlock: · 7846 unmodified lines
19994	20006	if (!is_priv)
19995	20007	mutex_unlock(&bpf_verifier_lock);
19996	20008	bpf_clear_insn_aux_data(env, 0, env->prog->len);
19997		− vfree(env->insn_aux_data);
19998	20009	err_free_env:
19999	20010	bpf_stack_liveness_free(env);
20000	20011	kvfree(env->cfg.insn_postorder);
20001	20012	kvfree(env->scc_info);
20002	20013	kvfree(env->succ);
20003	20014	kvfree(env->gotox_tmp_buf);
	20015	+ vfree(env->insn_aux_data);
20004	20016	kvfree(env);
20005	20017	return ret;
20006	20018	}

kernel/fork.c+5−4

@@ -1009,6 +1009,11 @@ static struct task_struct dup_task_struct(struct task_struct orig, int node) · 1008 unmodified lines
1009	1009	tsk->mm_cid.active = 0;
1010	1010	INIT_HLIST_NODE(&tsk->mm_cid.node);
1011	1011	#endif
	1012	+
	1013	+#ifdef CONFIG_BPF_SYSCALL
	1014	+ RCU_INIT_POINTER(tsk->bpf_storage, NULL);
	1015	+ tsk->bpf_ctx = NULL;
	1016	+#endif
1012	1017	return tsk;
1013	1018
1014	1019	free_stack:
@@ -2247,10 +2252,6 @@ __latent_entropy struct task_struct *copy_process( · 1232 unmodified lines
2247	2252	p->sequential_io = 0;
2248	2253	p->sequential_io_avg = 0;
2249	2254	#endif
2250		−#ifdef CONFIG_BPF_SYSCALL
2251		− RCU_INIT_POINTER(p->bpf_storage, NULL);
2252		− p->bpf_ctx = NULL;
2253		−#endif
2254	2255
2255	2256	unwind_task_init(p);
2256	2257

net/core/sock_map.c+20−0

@@ -1515,6 +1515,17 @@ static int sock_map_prog_link_lookup(struct bpf_map map, struct bpf_prog **ppr · 1514 unmodified lines
1515	1515	return 0;
1516	1516	}
1517	1517
	1518	+static int sock_map_prog_attach_check(enum bpf_attach_type attach_type,
	1519	+ struct bpf_prog *prog)
	1520	+{
	1521	+ /* A stream parser must not modify the skb, only measure it. */
	1522	+ if (prog && attach_type == BPF_SK_SKB_STREAM_PARSER &&
	1523	+ prog->aux->changes_pkt_data)
	1524	+ return -EINVAL;
	1525	+
	1526	+ return 0;
	1527	+}
	1528	+
1518	1529	/* Handle the following four cases:
1519	1530	* prog_attach: prog != NULL, old == NULL, link == NULL
1520	1531	* prog_detach: prog == NULL, old != NULL, link == NULL
@@ -1533,6 +1544,10 @@ static int sock_map_prog_update(struct bpf_map map, struct bpf_prog prog, · 12 unmodified lines
1533	1544	if (ret)
1534	1545	return ret;
1535	1546
	1547	+ ret = sock_map_prog_attach_check(which, prog);
	1548	+ if (ret)
	1549	+ return ret;
	1550	+
1536	1551	/* for prog_attach/prog_detach/link_attach, return error if a bpf_link
1537	1552	* exists for that prog.
1538	1553	*/
@@ -1776,6 +1791,11 @@ static int sock_map_link_update_prog(struct bpf_link *link, · 237 unmodified lines
1776	1791	ret = -EINVAL;
1777	1792	goto out;
1778	1793	}
	1794	+
	1795	+ ret = sock_map_prog_attach_check(link->attach_type, prog);
	1796	+ if (ret)
	1797	+ goto out;
	1798	+
1779	1799	if (!sockmap_link->map) {
1780	1800	ret = -ENOLINK;
1781	1801	goto out;

security/bpf/hooks.c+3−0

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	7	#include <linux/bpf_lsm.h>
8	8	#include <uapi/linux/lsm.h>
9	9
	10	+bool bpf_lsm_initialized __ro_after_init;
	11	+
10	12	static struct security_hook_list bpf_lsm_hooks[] __ro_after_init = {
11	13	#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
12	14	LSM_HOOK_INIT(NAME, bpf_lsm_##NAME),
@@ -24,6 +26,7 @@ static int __init bpf_lsm_init(void) · 11 unmodified lines
24	26	{
25	27	security_add_hooks(bpf_lsm_hooks, ARRAY_SIZE(bpf_lsm_hooks),
26	28	&bpf_lsmid);
	29	+ bpf_lsm_initialized = true;
27	30	pr_info("LSM support for eBPF active\n");
28	31	return 0;
29	32	}

tools/testing/selftests/bpf/prog_tests/sockmap_strp.c+31−0

@@ -431,6 +431,35 @@ out: · 430 unmodified lines
431	431	test_sockmap_strp__destroy(strp);
432	432	}
433	433
	434	+static void test_sockmap_strp_parser_reject(void)
	435	+{
	436	+ struct test_sockmap_strp *strp = NULL;
	437	+ int parser_mod, parser_ro, link;
	438	+ int err, map;
	439	+
	440	+ strp = test_sockmap_strp__open_and_load();
	441	+ if (!ASSERT_OK_PTR(strp, "test_sockmap_strp__open_and_load"))
	442	+ return;
	443	+
	444	+ map = bpf_map__fd(strp->maps.sock_map);
	445	+ parser_mod = bpf_program__fd(strp->progs.prog_skb_parser_resize);
	446	+ parser_ro = bpf_program__fd(strp->progs.prog_skb_parser);
	447	+
	448	+ err = bpf_prog_attach(parser_mod, map, BPF_SK_SKB_STREAM_PARSER, 0);
	449	+ ASSERT_ERR(err, "bpf_prog_attach parser_mod");
	450	+
	451	+ link = bpf_link_create(parser_ro, map, BPF_SK_SKB_STREAM_PARSER, NULL);
	452	+ if (!ASSERT_GE(link, 0, "bpf_link_create parser_ro"))
	453	+ goto out;
	454	+
	455	+ err = bpf_link_update(link, parser_mod, NULL);
	456	+ ASSERT_ERR(err, "bpf_link_update parser_mod");
	457	+out:
	458	+ if (link >= 0)
	459	+ close(link);
	460	+ test_sockmap_strp__destroy(strp);
	461	+}
	462	+
434	463	void test_sockmap_strp(void)
435	464	{
436	465	if (test__start_subtest("sockmap strp tcp pass"))
@@ -451,4 +480,6 @@ void test_sockmap_strp(void) · 14 unmodified lines
451	480	test_sockmap_strp_multiple_pkt(AF_INET, SOCK_STREAM);
452	481	if (test__start_subtest("sockmap strp tcp dispatch"))
453	482	test_sockmap_strp_dispatch_pkt(AF_INET, SOCK_STREAM);
	483	+ if (test__start_subtest("sockmap strp parser reject pkt mod"))
	484	+ test_sockmap_strp_parser_reject();
454	485	}

tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c+34−0

@@ -13,12 +13,20 @@ struct node_acquire { · 12 unmodified lines
13	13	struct bpf_refcount refcount;
14	14	};
15	15
	16	+struct node_refcounted {
	17	+ long key;
	18	+ struct bpf_list_node list;
	19	+ struct bpf_refcount refcount;
	20	+};
	21	+
16	22	extern void bpf_rcu_read_lock(void) __ksym;
17	23	extern void bpf_rcu_read_unlock(void) __ksym;
18	24
19	25	#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
20	26	private(A) struct bpf_spin_lock glock;
21	27	private(A) struct bpf_rb_root groot __contains(node_acquire, node);
	28	+private(B) struct bpf_spin_lock lock;
	29	+private(B) struct bpf_list_head head __contains(node_refcounted, list);
22	30
23	31	static bool less(struct bpf_rb_node a, const struct bpf_rb_node b)
24	32	{
@@ -93,6 +101,32 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) · 68 unmodified lines
93	101	return 0;
94	102	}
95	103
	104	+SEC("?tc")
	105	+__failure __msg("dereference of modified ptr_ ptr R1")
	106	+long refcount_acquire_list_node_offset(void *ctx)
	107	+{
	108	+ struct node_refcounted node, base, *ref;
	109	+ struct bpf_list_node *list_node;
	110	+
	111	+ node = bpf_obj_new(typeof(*node));
	112	+ if (!node)
	113	+ return 1;
	114	+
	115	+ bpf_spin_lock(&lock);
	116	+ bpf_list_push_front(&head, &node->list);
	117	+ list_node = bpf_list_pop_front(&head);
	118	+ bpf_spin_unlock(&lock);
	119	+ if (!list_node)
	120	+ return 2;
	121	+
	122	+ base = container_of(list_node, struct node_refcounted, list);
	123	+ ref = bpf_refcount_acquire(list_node);
	124	+ if (ref)
	125	+ bpf_obj_drop(ref);
	126	+ bpf_obj_drop(base);
	127	+ return 0;
	128	+}
	129	+
96	130	SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
97	131	__failure __msg("function calls are not allowed while holding a lock")
98	132	int BPF_PROG(rbtree_fail_sleepable_lock_across_rcu,

tools/testing/selftests/bpf/progs/sockmap_parse_prog.c+0−22

@@ -5,28 +5,6 @@ · 4 unmodified lines
5	5	SEC("sk_skb1")
6	6	int bpf_prog1(struct __sk_buff *skb)
7	7	{
8		− void data_end = (void )(long) skb->data_end;
9		− void data = (void )(long) skb->data;
10		− __u8 *d = data;
11		− int err;
12		−
13		− if (data + 10 > data_end) {
14		− err = bpf_skb_pull_data(skb, 10);
15		− if (err)
16		− return SK_DROP;
17		−
18		− data_end = (void *)(long)skb->data_end;
19		− data = (void *)(long)skb->data;
20		− if (data + 10 > data_end)
21		− return SK_DROP;
22		− }
23		−
24		− /* This write/read is a bit pointless but tests the verifier and
25		− * strparser handler for read/write pkt data and access into sk
26		− * fields.
27		− */
28		− d = data;
29		− d[7] = 1;
30	8	return skb->len;
31	9	}
32	10

tools/testing/selftests/bpf/progs/test_sockmap_strp.c+7−0

@@ -50,4 +50,11 @@ int prog_skb_parser_partial(struct __sk_buff *skb) · 49 unmodified lines
50	50	return 10;
51	51	}
52	52
	53	+SEC("sk_skb/stream_parser")
	54	+int prog_skb_parser_resize(struct __sk_buff *skb)
	55	+{
	56	+ bpf_skb_change_tail(skb, skb->len, 0);
	57	+ return skb->len;
	58	+}
	59	+
53	60	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/verifier_unpriv.c+19−0

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	6	#include "../../../include/linux/filter.h"
7	7	#include "bpf_misc.h"
8	8
	9	+extern const int bpf_prog_active __ksym;
	10	+
9	11	#define BPF_SK_LOOKUP(func) \
10	12	/* struct bpf_sock_tuple tuple = {} */ \
11	13	"r2 = 0;" \
@@ -77,6 +79,23 @@ __naked void dummy_prog_loop1_socket(void) · 65 unmodified lines
77	79	: __clobber_all);
78	80	}
79	81
	82	+SEC("socket")
	83	+__description("unpriv: pseudo btf id log masks address")
	84	+__success_unpriv
	85	+__msg_unpriv("0: (18) r1 = 0x0")
	86	+__not_msg_unpriv("0: (18) r1 = 0x{{[1-9a-f][0-9a-f]*}}")
	87	+__retval_unpriv(0)
	88	+__log_level(2)
	89	+__naked void pseudo_btf_id_log_masks_address(void)
	90	+{
	91	+ asm volatile ("r1 = %[bpf_prog_active] ll;"
	92	+ "r0 = 0;"
	93	+ "exit;"
	94	+ :
	95	+ : __imm_addr(bpf_prog_active)
	96	+ : __clobber_all);
	97	+}
	98	+
80	99	SEC("socket")
81	100	__description("unpriv: return pointer")
82	101	__success __failure_unpriv __msg_unpriv("R0 leaks addr")

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code