Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Pull BPF fixes from Daniel Borkmann: - Initialize task local storage before fork bails out to free the task (Jann Horn) - Fix insn_aux_data leak on verifier error path (KaFai Wan) - Reject BPF inode storage map creation when BPF LSM is uninitialized (Matt Bobrowski) - Mask pseudo pointer values in verifier logs when pointer leaks are not allowed (Nuoqi Gui) - Harden BPF JIT against spraying via IBPB flush (Pawan Gupta) - Reject a skb-modifying SK_SKB stream parser since the latter is only meant to measure the next message (Sechang Lim) - Fix bpf_refcount_acquire to reject refcounted allocation arguments with a non-zero fixed offset (Yiyang Chen) * tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: bpf: Prefer dirty packs for eBPF allocations bpf: Prefer packs that won't trigger an IBPB flush on allocation bpf: Skip redundant IBPB in pack allocator bpf: Restrict JIT predictor flush to cBPF x86/bugs: Enable IBPB flush on BPF JIT allocation bpf: Support for hardening against JIT spraying bpf: Reject BPF_MAP_TYPE_INODE_STORAGE creation if BPF LSM is uninitialized bpf,fork: wipe ->bpf_storage before bailouts that access it bpf: Fix insn_aux_data leak on verifier err_free_env path selftests/bpf: Cover pseudo-BTF ksym log masking bpf: Mask pseudo pointer values in verifier logs selftests/bpf: Cover refcount acquire node offsets bpf: Reject offset refcount acquire arguments selftests/bpf: test rejection of a packet-modifying SK_SKB stream parser bpf, sockmap: reject a packet-modifying SK_SKB stream parser selftests/bpf: don't modify the skb in the strparser parser prog

Linus Torvalds committed 4 months ago

Commit 51512e22 parents 826eec5 b72e29e23 files · +297 −62Browse files

Unified · Split

23 files

arch/arm64/net/bpf_jit_comp.c +2 −2
arch/loongarch/net/bpf_jit.c +3 −2
arch/powerpc/net/bpf_jit_comp.c +2 −2
arch/riscv/net/bpf_jit_comp64.c +1 −1
arch/riscv/net/bpf_jit_core.c +2 −1
arch/x86/include/asm/nospec-branch.h +4 −0
arch/x86/kernel/cpu/bugs.c +45 −5
arch/x86/net/bpf_jit_comp.c +3 −2
include/linux/bpf_lsm.h +4 −0
include/linux/filter.h +13 −2
kernel/bpf/bpf_inode_storage.c +9 −0
kernel/bpf/core.c +62 −6
kernel/bpf/disasm.c +4 −1
kernel/bpf/dispatcher.c +1 −1
kernel/bpf/verifier.c +23 −11
kernel/fork.c +5 −4
net/core/sock_map.c +20 −0
security/bpf/hooks.c +3 −0
tools/testing/selftests/bpf/prog_tests/sockmap_strp.c +31 −0
tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c +34 −0
tools/testing/selftests/bpf/progs/sockmap_parse_prog.c +0 −22
tools/testing/selftests/bpf/progs/test_sockmap_strp.c +7 −0
tools/testing/selftests/bpf/progs/verifier_unpriv.c +19 −0

arch/arm64/net/bpf_jit_comp.c+2−2

@@ -2177,7 +2177,7 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_pr · 2176 unmodified lines
2177	image_size = extable_offset + extable_size;	2177	image_size = extable_offset + extable_size;
2178	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,	2178	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr,
2179	sizeof(u64), &header, &image_ptr,	2179	sizeof(u64), &header, &image_ptr,
2180	jit_fill_hole);	2180	jit_fill_hole, was_classic);
2181	if (!ro_header)	2181	if (!ro_header)
2182	goto out_off;	2182	goto out_off;
2183		2183
@@ -2870,7 +2870,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 686 unmodified lines
2870		2870
2871	void *arch_alloc_bpf_trampoline(unsigned int size)	2871	void *arch_alloc_bpf_trampoline(unsigned int size)
2872	{	2872	{
2873	return bpf_prog_pack_alloc(size, jit_fill_hole);	2873	return bpf_prog_pack_alloc(size, jit_fill_hole, false);
2874	}	2874	}
2875		2875
2876	void arch_free_bpf_trampoline(void *image, unsigned int size)	2876	void arch_free_bpf_trampoline(void *image, unsigned int size)

arch/loongarch/net/bpf_jit.c+3−2

 void *arch_alloc_bpf_trampoline(unsigned int size)
 {
-	return bpf_prog_pack_alloc(size, jit_fill_hole);
+	return bpf_prog_pack_alloc(size, jit_fill_hole, false);
 }
 void arch_free_bpf_trampoline(void *image, unsigned int size)
 	image_size = prog_size + extable_size;
 	/* Now we know the size of the structure to make */
 	ro_header = bpf_jit_binary_pack_alloc(image_size, &ro_image_ptr, sizeof(u32),
-					      &header, &image_ptr, jit_fill_hole);
+					      &header, &image_ptr, jit_fill_hole,
+					      bpf_prog_was_classic(prog));
 	if (!ro_header)
 		goto out_offset;

arch/powerpc/net/bpf_jit_comp.c+2−2

@@ -295,7 +295,7 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_pr · 294 unmodified lines
295	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;	295	alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
296		296
297	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,	297	fhdr = bpf_jit_binary_pack_alloc(alloclen, &fimage, 4, &hdr, &image,
298	bpf_jit_fill_ill_insns);	298	bpf_jit_fill_ill_insns, bpf_prog_was_classic(fp));
299	if (!fhdr)	299	if (!fhdr)
300	goto out_err;	300	goto out_err;
301		301
@@ -588,7 +588,7 @@ bool bpf_jit_inlines_helper_call(s32 imm) · 286 unmodified lines
588		588
589	void *arch_alloc_bpf_trampoline(unsigned int size)	589	void *arch_alloc_bpf_trampoline(unsigned int size)
590	{	590	{
591	return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns);	591	return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns, false);
592	}	592	}
593		593
594	void arch_free_bpf_trampoline(void *image, unsigned int size)	594	void arch_free_bpf_trampoline(void *image, unsigned int size)

arch/riscv/net/bpf_jit_comp64.c+1−1

@@ -1321,7 +1321,7 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, · 1320 unmodified lines
1321		1321
1322	void *arch_alloc_bpf_trampoline(unsigned int size)	1322	void *arch_alloc_bpf_trampoline(unsigned int size)
1323	{	1323	{
1324	return bpf_prog_pack_alloc(size, bpf_fill_ill_insns);	1324	return bpf_prog_pack_alloc(size, bpf_fill_ill_insns, false);
1325	}	1325	}
1326		1326
1327	void arch_free_bpf_trampoline(void *image, unsigned int size)	1327	void arch_free_bpf_trampoline(void *image, unsigned int size)

arch/riscv/net/bpf_jit_core.c+2−1

@@ -109,7 +109,8 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_verifier_env env, struct bpf_pr · 108 unmodified lines
109	bpf_jit_binary_pack_alloc(prog_size + extable_size,	109	bpf_jit_binary_pack_alloc(prog_size + extable_size,
110	&jit_data->ro_image, sizeof(u32),	110	&jit_data->ro_image, sizeof(u32),
111	&jit_data->header, &jit_data->image,	111	&jit_data->header, &jit_data->image,
112	bpf_fill_ill_insns);	112	bpf_fill_ill_insns,
		113	bpf_prog_was_classic(prog));
113	if (!jit_data->ro_header)	114	if (!jit_data->ro_header)
114	goto out_offset;	115	goto out_offset;
115		116

arch/x86/include/asm/nospec-branch.h+4−0

 extern void entry_untrain_ret(void);
 extern void write_ibpb(void);
+#ifdef CONFIG_BPF_JIT
+extern void bpf_arch_ibpb(void);
+#endif
 #ifdef CONFIG_X86_64
 extern void clear_bhb_loop(void);
 #endif

arch/x86/kernel/cpu/bugs.c+45−5

 #include <linux/sched/smt.h>
 #include <linux/pgtable.h>
 #include <linux/bpf.h>
+#include <linux/filter.h>
 #include <linux/kvm_types.h>
 #include <asm/spec-ctrl.h>
 {
 	return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 }
+/*
+ * The "retpoline sequence" is the "call;mov;ret" sequence that
+ * replaces normal indirect branch instructions. Differentiate
+ * *the* retpoline sequence from the LFENCE-prefixed indirect
+ * branches that simply use the retpoline infrastructure.
+ */
+static inline bool retpoline_seq_enabled(void)
+{
+	return boot_cpu_has(X86_FEATURE_RETPOLINE) && !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE);
+}
 #else
 static inline const char *spectre_v2_module_string(void) { return ""; }
+static inline bool retpoline_seq_enabled(void) { return false; }
 #endif
 #define SPECTRE_V2_LFENCE_MSG "WARNING: LFENCE mitigation is not recommended for this CPU, data leaks possible!\n"
 		return;
 	/* Retpoline mitigates against BHI unless the CPU has RRSBA behavior */
-	if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+	if (retpoline_seq_enabled()) {
-	    !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE)) {
 		spec_ctrl_disable_kernel_rrsba();
 		if (rrsba_disabled)
 			return;
 		pr_info("%s\n", spectre_v2_strings[spectre_v2_enabled]);
 }
+#ifdef CONFIG_BPF_JIT
+static void __bpf_arch_ibpb(void *unused)
+{
+	write_ibpb();
+}
+void bpf_arch_ibpb(void)
+{
+	on_each_cpu(__bpf_arch_ibpb, NULL, 1);
+}
+static bool __init cpu_wants_ibpb_bpf(void)
+{
+	/* A genuine retpoline already neutralizes ring0 indirect predictions */
+	if (retpoline_seq_enabled())
+		return false;
+	return boot_cpu_has(X86_FEATURE_IBPB);
+}
+#endif
 static void __init spectre_v2_apply_mitigation(void)
 {
 	if (spectre_v2_enabled == SPECTRE_V2_EIBRS && unprivileged_ebpf_enabled())
 		setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 		pr_info("Enabling Restricted Speculation for firmware calls\n");
 	}
+#ifdef CONFIG_BPF_JIT
+	if (cpu_wants_ibpb_bpf()) {
+		static_call_update(bpf_arch_pred_flush, bpf_arch_ibpb);
+		static_branch_enable(&bpf_pred_flush_enabled);
+		pr_info("Enabling IBPB for BPF\n");
+	}
+#endif
 }
 static void update_stibp_msr(void * __unused)
 		return "; BHI: BHI_DIS_S";
 	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_LOOP))
 		return "; BHI: SW loop, KVM: SW loop";
-	else if (boot_cpu_has(X86_FEATURE_RETPOLINE) &&
+	else if (retpoline_seq_enabled() && rrsba_disabled)
-		 !boot_cpu_has(X86_FEATURE_RETPOLINE_LFENCE) &&
-		 rrsba_disabled)
 		return "; BHI: Retpoline";
 	else if (boot_cpu_has(X86_FEATURE_CLEAR_BHB_VMEXIT))
 		return "; BHI: Vulnerable, KVM: SW loop";

arch/x86/net/bpf_jit_comp.c+3−2

 void *arch_alloc_bpf_trampoline(unsigned int size)
 {
-	return bpf_prog_pack_alloc(size, jit_fill_hole);
+	return bpf_prog_pack_alloc(size, jit_fill_hole, false);
 }
 void arch_free_bpf_trampoline(void *image, unsigned int size)
 			/* allocate module memory for x86 insns and extable */
 			header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
 							   &image, align, &rw_header, &rw_image,
-							   jit_fill_hole);
+							   jit_fill_hole,
+							   bpf_prog_was_classic(prog));
 			if (!header)
 				goto out_addrs;
 			prog->aux->extable = (void *) image + roundup(proglen, align);

include/linux/bpf_lsm.h+4−0

 #ifdef CONFIG_BPF_LSM
+extern bool bpf_lsm_initialized __ro_after_init;
 #define LSM_HOOK(RET, DEFAULT, NAME, ...) \
 	RET bpf_lsm_##NAME(__VA_ARGS__);
 #include <linux/lsm_hook_defs.h>
 #else /* !CONFIG_BPF_LSM */
+#define bpf_lsm_initialized false
 static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
 {
 	return false;

include/linux/filter.h+13−2

 #include <linux/if_vlan.h>
 #include <linux/vmalloc.h>
 #include <linux/sockptr.h>
+#include <linux/static_call.h>
 #include <linux/u64_stats_sync.h>
 #include <net/sch_generic.h>
 typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);
+/*
+ * Flush the indirect branch predictors before reusing JIT memory, so that
+ * indirect jumps into a newly written program don't reuse predictions left
+ * behind by an old program that occupied the same space.
+ */
+void bpf_arch_pred_flush(void);
+DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush);
+DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled);
 void bpf_jit_fill_hole_with_zero(void *area, unsigned int size);
 struct bpf_binary_header *
 struct bpf_binary_header *
 bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);
-void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns);
+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic);
 void bpf_prog_pack_free(void *ptr, u32 size);
 static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
 			  unsigned int alignment,
 			  struct bpf_binary_header **rw_hdr,
 			  u8 **rw_image,
-			  bpf_jit_fill_hole_t bpf_fill_ill_insns);
+			  bpf_jit_fill_hole_t bpf_fill_ill_insns,
+			  bool was_classic);
 int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header,
 				 struct bpf_binary_header *rw_header);
 void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header,

kernel/bpf/bpf_inode_storage.c+9−0

@@ -178,6 +178,15 @@ static int notsupp_get_next_key(struct bpf_map map, void key, · 177 unmodified lines
178		178
179	static struct bpf_map inode_storage_map_alloc(union bpf_attr attr)	179	static struct bpf_map inode_storage_map_alloc(union bpf_attr attr)
180	{	180	{
		181	/*
		182	* Do not allow allocation of BPF_MAP_TYPE_INODE_STORAGE if the BPF LSM
		183	* was not initialized by the LSM framework at boot. Without proper
		184	* initialization, the BPF inode security blob offset remains unprepared,
		185	* causing bpf_inode() to calculate an invalid memory offset and corrupt
		186	* inode->i_security.
		187	*/
		188	if (!bpf_lsm_initialized)
		189	return ERR_PTR(-EOPNOTSUPP);
181	return bpf_local_storage_map_alloc(attr, &inode_cache);	190	return bpf_local_storage_map_alloc(attr, &inode_cache);
182	}	191	}
183		192

kernel/bpf/core.c+62−6

 #include <uapi/linux/btf.h>
 #include <linux/filter.h>
 #include <linux/skbuff.h>
+#include <linux/static_call.h>
 #include <linux/vmalloc.h>
 #include <linux/prandom.h>
 #include <linux/bpf.h>
 struct bpf_prog_pack {
 	struct list_head list;
 	void *ptr;
+	bool arch_flush_needed;
 	unsigned long bitmap[];
 };
 	memset(area, 0, size);
 }
+DEFINE_STATIC_CALL_NULL(bpf_arch_pred_flush, bpf_arch_pred_flush);
+/*
+ * Enabled once bpf_arch_pred_flush points at a real flush routine. Lets the
+ * pack allocator test "is a predictor flush wired up at all" with a cheap
+ * static branch instead of repeatedly querying the static call target.
+ */
+DEFINE_STATIC_KEY_FALSE(bpf_pred_flush_enabled);
 #define BPF_PROG_SIZE_TO_NBITS(size)	(round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)
 static DEFINE_MUTEX(pack_mutex);
 	bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
 	bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
+	if (static_branch_unlikely(&bpf_pred_flush_enabled))
+		pack->arch_flush_needed = true;
 	set_vm_flush_reset_perms(pack->ptr);
 	err = set_memory_rox((unsigned long)pack->ptr,
 			     BPF_PROG_PACK_SIZE / PAGE_SIZE);
 	return NULL;
 }
-void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic)
 {
 	unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size);
-	struct bpf_prog_pack *pack;
+	struct bpf_prog_pack *pack, *fallback_pack = NULL;
-	unsigned long pos;
+	unsigned long pos, fallback_pos = 0;
 	void *ptr = NULL;
 	mutex_lock(&pack_mutex);
 	if (size > BPF_PROG_PACK_SIZE) {
+		/*
+		 * Allocations larger than a pack get their own pages, and
+		 * predictors are not flushed for such allocation. This is only
+		 * safe because cBPF programs (the unprivileged attack surface)
+		 * are bounded well below a pack size.
+		 */
+		if (was_classic && static_branch_unlikely(&bpf_pred_flush_enabled))
+			pr_warn_once("BPF: Predictors not flushed for allocations greater than BPF_PROG_PACK_SIZE\n");
 		size = round_up(size, PAGE_SIZE);
 		ptr = bpf_jit_alloc_exec(size);
 		if (ptr) {
 	list_for_each_entry(pack, &pack_list, list) {
 		pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
 						 nbits, 0);
-		if (pos < BPF_PROG_CHUNK_COUNT)
+		if (pos >= BPF_PROG_CHUNK_COUNT)
+			continue;
+		/* Flush not enabled, use any pack */
+		if (!static_branch_unlikely(&bpf_pred_flush_enabled))
 			goto found_free_area;
+		/*
+		 * cBPF reuse of a dirty pack triggers a flush, so prefer a
+		 * clean pack for cBPF. eBPF never flushes, so steer it to a
+		 * dirty pack and keep clean packs free for cBPF.
+		 */
+		if (was_classic ^ pack->arch_flush_needed)
+			goto found_free_area;
+		if (!fallback_pack) {
+			fallback_pack = pack;
+			fallback_pos = pos;
+		}
+	}
+	/* No preferred pack found */
+	if (fallback_pack) {
+		pack = fallback_pack;
+		pos = fallback_pos;
+		goto found_free_area;
 	}
 	pack = alloc_new_pack(bpf_fill_ill_insns);
 	pos = 0;
 found_free_area:
+	/* Flush only for cBPF as it may contain a crafted gadget */
+	if (static_branch_unlikely(&bpf_pred_flush_enabled) &&
+	    pack->arch_flush_needed &&
+	    was_classic) {
+		struct bpf_prog_pack *p;
+		static_call_cond(bpf_arch_pred_flush)();
+		list_for_each_entry(p, &pack_list, list)
+			p->arch_flush_needed = false;
+	}
 	bitmap_set(pack->bitmap, pos, nbits);
 	ptr = (void *)(pack->ptr) + (pos << BPF_PROG_CHUNK_SHIFT);
 		  "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");
 	bitmap_clear(pack->bitmap, pos, nbits);
+	if (static_branch_unlikely(&bpf_pred_flush_enabled))
+		pack->arch_flush_needed = true;
 	if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
 				       BPF_PROG_CHUNK_COUNT, 0) == 0) {
 		list_del(&pack->list);
 			  unsigned int alignment,
 			  struct bpf_binary_header **rw_header,
 			  u8 **rw_image,
-			  bpf_jit_fill_hole_t bpf_fill_ill_insns)
+			  bpf_jit_fill_hole_t bpf_fill_ill_insns,
+			  bool was_classic)
 {
 	struct bpf_binary_header *ro_header;
 	u32 size, hole, start;
 	if (bpf_jit_charge_modmem(size))
 		return NULL;
-	ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
+	ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns, was_classic);
 	if (!ro_header) {
 		bpf_jit_uncharge_modmem(size);
 		return NULL;

kernel/bpf/disasm.c+4−1

 			 */
 			u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
 			bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD ||
-				      insn->src_reg == BPF_PSEUDO_MAP_VALUE;
+				      insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
+				      insn->src_reg == BPF_PSEUDO_MAP_IDX ||
+				      insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE ||
+				      insn->src_reg == BPF_PSEUDO_BTF_ID;
 			char tmp[64];
 			if (is_ptr && !allow_ptr_leaks)

kernel/bpf/dispatcher.c+1−1

@@ -145,7 +145,7 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher d, struct bpf_prog from, · 144 unmodified lines
145		145
146	mutex_lock(&d->mutex);	146	mutex_lock(&d->mutex);
147	if (!d->image) {	147	if (!d->image) {
148	d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);	148	d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero, false);
149	if (!d->image)	149	if (!d->image)
150	goto out;	150	goto out;
151	d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);	151	d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);

kernel/bpf/verifier.c+23−11

 	return field;
 }
-static int check_func_arg_reg_off(struct bpf_verifier_env *env,
+static int __check_func_arg_reg_off(struct bpf_verifier_env *env,
-				  const struct bpf_reg_state *reg, argno_t argno,
+				    const struct bpf_reg_state *reg, argno_t argno,
-				  enum bpf_arg_type arg_type)
+				    enum bpf_arg_type arg_type,
+				    bool btf_id_fixed_off_ok)
 {
 	u32 type = reg->type;
 	case PTR_TO_BTF_ID | MEM_ALLOC | NON_OWN_REF | MEM_RCU:
 		/* When referenced PTR_TO_BTF_ID is passed to release function,
 		 * its fixed offset must be 0. In the other cases, fixed offset
-		 * can be non-zero. This was already checked above. So pass
+		 * can be non-zero unless the caller requires otherwise.
-		 * fixed_off_ok as true to allow fixed offset for all other
+		 * var_off always must be 0 for PTR_TO_BTF_ID, hence we still
-		 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we
+		 * need to do checks instead of returning.
-		 * still need to do checks instead of returning.
 		 */
-		return __check_ptr_off_reg(env, reg, argno, true);
+		return __check_ptr_off_reg(env, reg, argno, btf_id_fixed_off_ok);
 	case PTR_TO_CTX:
 		/*
 		 * Allow fixed and variable offsets for syscall context, but
 	}
 }
+static int check_func_arg_reg_off(struct bpf_verifier_env *env,
+				  const struct bpf_reg_state *reg, argno_t argno,
+				  enum bpf_arg_type arg_type)
+{
+	return __check_func_arg_reg_off(env, reg, argno, arg_type, true);
+}
 static int check_arg_const_str(struct bpf_verifier_env *env,
 			       struct bpf_reg_state *reg, argno_t argno)
 {
 		enum bpf_arg_type arg_type = ARG_DONTCARE;
 		argno_t argno = argno_from_arg(i + 1);
 		int regno = reg_from_argno(argno);
+		bool btf_id_fixed_off_ok = true;
 		u32 ref_id, type_size;
 		bool is_ret_buf_sz = false;
 		int kf_arg_type;
 		case KF_ARG_PTR_TO_MEM:
 		case KF_ARG_PTR_TO_MEM_SIZE:
 		case KF_ARG_PTR_TO_CALLBACK:
-		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
 		case KF_ARG_PTR_TO_CONST_STR:
 		case KF_ARG_PTR_TO_WORKQUEUE:
 		case KF_ARG_PTR_TO_TIMER:
 		case KF_ARG_PTR_TO_CTX:
 			arg_type = ARG_PTR_TO_CTX;
 			break;
+		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
+			arg_type = ARG_PTR_TO_BTF_ID;
+			btf_id_fixed_off_ok = false;
+			break;
 		default:
 			verifier_bug(env, "unknown kfunc arg type %d", kf_arg_type);
 			return -EFAULT;
 		if (regno == meta->release_regno)
 			arg_type |= OBJ_RELEASE;
-		ret = check_func_arg_reg_off(env, reg, argno, arg_type);
+		ret = __check_func_arg_reg_off(env, reg, argno, arg_type,
+					       btf_id_fixed_off_ok);
 		if (ret < 0)
 			return ret;
 	if (!is_priv)
 		mutex_unlock(&bpf_verifier_lock);
 	bpf_clear_insn_aux_data(env, 0, env->prog->len);
-	vfree(env->insn_aux_data);
 err_free_env:
 	bpf_stack_liveness_free(env);
 	kvfree(env->cfg.insn_postorder);
 	kvfree(env->scc_info);
 	kvfree(env->succ);
 	kvfree(env->gotox_tmp_buf);
+	vfree(env->insn_aux_data);
 	kvfree(env);
 	return ret;
 }

kernel/fork.c+5−4

 	tsk->mm_cid.active = 0;
 	INIT_HLIST_NODE(&tsk->mm_cid.node);
 #endif
+#ifdef CONFIG_BPF_SYSCALL
+	RCU_INIT_POINTER(tsk->bpf_storage, NULL);
+	tsk->bpf_ctx = NULL;
+#endif
 	return tsk;
 free_stack:
 	p->sequential_io	= 0;
 	p->sequential_io_avg	= 0;
 #endif
-#ifdef CONFIG_BPF_SYSCALL
-	RCU_INIT_POINTER(p->bpf_storage, NULL);
-	p->bpf_ctx = NULL;
-#endif
 	unwind_task_init(p);

net/core/sock_map.c+20−0

@@ -1515,6 +1515,17 @@ static int sock_map_prog_link_lookup(struct bpf_map map, struct bpf_prog **ppr · 1514 unmodified lines
1515	return 0;	1515	return 0;
1516	}	1516	}
1517		1517
		1518	static int sock_map_prog_attach_check(enum bpf_attach_type attach_type,
		1519	struct bpf_prog *prog)
		1520	{
		1521	/* A stream parser must not modify the skb, only measure it. */
		1522	if (prog && attach_type == BPF_SK_SKB_STREAM_PARSER &&
		1523	prog->aux->changes_pkt_data)
		1524	return -EINVAL;
		1525
		1526	return 0;
		1527	}
		1528
1518	/* Handle the following four cases:	1529	/* Handle the following four cases:
1519	* prog_attach: prog != NULL, old == NULL, link == NULL	1530	* prog_attach: prog != NULL, old == NULL, link == NULL
1520	* prog_detach: prog == NULL, old != NULL, link == NULL	1531	* prog_detach: prog == NULL, old != NULL, link == NULL
@@ -1533,6 +1544,10 @@ static int sock_map_prog_update(struct bpf_map map, struct bpf_prog prog, · 12 unmodified lines
1533	if (ret)	1544	if (ret)
1534	return ret;	1545	return ret;
1535		1546
		1547	ret = sock_map_prog_attach_check(which, prog);
		1548	if (ret)
		1549	return ret;
		1550
1536	/* for prog_attach/prog_detach/link_attach, return error if a bpf_link	1551	/* for prog_attach/prog_detach/link_attach, return error if a bpf_link
1537	* exists for that prog.	1552	* exists for that prog.
1538	*/	1553	*/
@@ -1776,6 +1791,11 @@ static int sock_map_link_update_prog(struct bpf_link *link, · 237 unmodified lines
1776	ret = -EINVAL;	1791	ret = -EINVAL;
1777	goto out;	1792	goto out;
1778	}	1793	}
		1794
		1795	ret = sock_map_prog_attach_check(link->attach_type, prog);
		1796	if (ret)
		1797	goto out;
		1798
1779	if (!sockmap_link->map) {	1799	if (!sockmap_link->map) {
1780	ret = -ENOLINK;	1800	ret = -ENOLINK;
1781	goto out;	1801	goto out;

security/bpf/hooks.c+3−0

 #include <linux/bpf_lsm.h>
 #include <uapi/linux/lsm.h>
+bool bpf_lsm_initialized __ro_after_init;
 static struct security_hook_list bpf_lsm_hooks[] __ro_after_init = {
 	#define LSM_HOOK(RET, DEFAULT, NAME, ...) \
 	LSM_HOOK_INIT(NAME, bpf_lsm_##NAME),
 {
 	security_add_hooks(bpf_lsm_hooks, ARRAY_SIZE(bpf_lsm_hooks),
 			   &bpf_lsmid);
+	bpf_lsm_initialized = true;
 	pr_info("LSM support for eBPF active\n");
 	return 0;
 }

tools/testing/selftests/bpf/prog_tests/sockmap_strp.c+31−0

@@ -431,6 +431,35 @@ out: · 430 unmodified lines
431	test_sockmap_strp__destroy(strp);	431	test_sockmap_strp__destroy(strp);
432	}	432	}
433		433
		434	static void test_sockmap_strp_parser_reject(void)
		435	{
		436	struct test_sockmap_strp *strp = NULL;
		437	int parser_mod, parser_ro, link;
		438	int err, map;
		439
		440	strp = test_sockmap_strp__open_and_load();
		441	if (!ASSERT_OK_PTR(strp, "test_sockmap_strp__open_and_load"))
		442	return;
		443
		444	map = bpf_map__fd(strp->maps.sock_map);
		445	parser_mod = bpf_program__fd(strp->progs.prog_skb_parser_resize);
		446	parser_ro = bpf_program__fd(strp->progs.prog_skb_parser);
		447
		448	err = bpf_prog_attach(parser_mod, map, BPF_SK_SKB_STREAM_PARSER, 0);
		449	ASSERT_ERR(err, "bpf_prog_attach parser_mod");
		450
		451	link = bpf_link_create(parser_ro, map, BPF_SK_SKB_STREAM_PARSER, NULL);
		452	if (!ASSERT_GE(link, 0, "bpf_link_create parser_ro"))
		453	goto out;
		454
		455	err = bpf_link_update(link, parser_mod, NULL);
		456	ASSERT_ERR(err, "bpf_link_update parser_mod");
		457	out:
		458	if (link >= 0)
		459	close(link);
		460	test_sockmap_strp__destroy(strp);
		461	}
		462
434	void test_sockmap_strp(void)	463	void test_sockmap_strp(void)
435	{	464	{
436	if (test__start_subtest("sockmap strp tcp pass"))	465	if (test__start_subtest("sockmap strp tcp pass"))
@@ -451,4 +480,6 @@ void test_sockmap_strp(void) · 14 unmodified lines
451	test_sockmap_strp_multiple_pkt(AF_INET, SOCK_STREAM);	480	test_sockmap_strp_multiple_pkt(AF_INET, SOCK_STREAM);
452	if (test__start_subtest("sockmap strp tcp dispatch"))	481	if (test__start_subtest("sockmap strp tcp dispatch"))
453	test_sockmap_strp_dispatch_pkt(AF_INET, SOCK_STREAM);	482	test_sockmap_strp_dispatch_pkt(AF_INET, SOCK_STREAM);
		483	if (test__start_subtest("sockmap strp parser reject pkt mod"))
		484	test_sockmap_strp_parser_reject();
454	}	485	}

tools/testing/selftests/bpf/progs/refcounted_kptr_fail.c+34−0

@@ -13,12 +13,20 @@ struct node_acquire { · 12 unmodified lines
13	struct bpf_refcount refcount;	13	struct bpf_refcount refcount;
14	};	14	};
15		15
		16	struct node_refcounted {
		17	long key;
		18	struct bpf_list_node list;
		19	struct bpf_refcount refcount;
		20	};
		21
16	extern void bpf_rcu_read_lock(void) __ksym;	22	extern void bpf_rcu_read_lock(void) __ksym;
17	extern void bpf_rcu_read_unlock(void) __ksym;	23	extern void bpf_rcu_read_unlock(void) __ksym;
18		24
19	#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))	25	#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
20	private(A) struct bpf_spin_lock glock;	26	private(A) struct bpf_spin_lock glock;
21	private(A) struct bpf_rb_root groot __contains(node_acquire, node);	27	private(A) struct bpf_rb_root groot __contains(node_acquire, node);
		28	private(B) struct bpf_spin_lock lock;
		29	private(B) struct bpf_list_head head __contains(node_refcounted, list);
22		30
23	static bool less(struct bpf_rb_node a, const struct bpf_rb_node b)	31	static bool less(struct bpf_rb_node a, const struct bpf_rb_node b)
24	{	32	{
@@ -93,6 +101,32 @@ long rbtree_refcounted_node_ref_escapes_owning_input(void *ctx) · 68 unmodified lines
93	return 0;	101	return 0;
94	}	102	}
95		103
		104	SEC("?tc")
		105	__failure __msg("dereference of modified ptr_ ptr R1")
		106	long refcount_acquire_list_node_offset(void *ctx)
		107	{
		108	struct node_refcounted node, base, *ref;
		109	struct bpf_list_node *list_node;
		110
		111	node = bpf_obj_new(typeof(*node));
		112	if (!node)
		113	return 1;
		114
		115	bpf_spin_lock(&lock);
		116	bpf_list_push_front(&head, &node->list);
		117	list_node = bpf_list_pop_front(&head);
		118	bpf_spin_unlock(&lock);
		119	if (!list_node)
		120	return 2;
		121
		122	base = container_of(list_node, struct node_refcounted, list);
		123	ref = bpf_refcount_acquire(list_node);
		124	if (ref)
		125	bpf_obj_drop(ref);
		126	bpf_obj_drop(base);
		127	return 0;
		128	}
		129
96	SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")	130	SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
97	__failure __msg("function calls are not allowed while holding a lock")	131	__failure __msg("function calls are not allowed while holding a lock")
98	int BPF_PROG(rbtree_fail_sleepable_lock_across_rcu,	132	int BPF_PROG(rbtree_fail_sleepable_lock_across_rcu,

tools/testing/selftests/bpf/progs/sockmap_parse_prog.c+0−22

@@ -5,28 +5,6 @@ · 4 unmodified lines
5	SEC("sk_skb1")	5	SEC("sk_skb1")
6	int bpf_prog1(struct __sk_buff *skb)	6	int bpf_prog1(struct __sk_buff *skb)
7	{	7	{
8	void data_end = (void )(long) skb->data_end;
9	void data = (void )(long) skb->data;
10	__u8 *d = data;
11	int err;
12
13	if (data + 10 > data_end) {
14	err = bpf_skb_pull_data(skb, 10);
15	if (err)
16	return SK_DROP;
17
18	data_end = (void *)(long)skb->data_end;
19	data = (void *)(long)skb->data;
20	if (data + 10 > data_end)
21	return SK_DROP;
22	}
23
24	/* This write/read is a bit pointless but tests the verifier and
25	* strparser handler for read/write pkt data and access into sk
26	* fields.
27	*/
28	d = data;
29	d[7] = 1;
30	return skb->len;	8	return skb->len;
31	}	9	}
32		10

tools/testing/selftests/bpf/progs/test_sockmap_strp.c+7−0

@@ -50,4 +50,11 @@ int prog_skb_parser_partial(struct __sk_buff *skb) · 49 unmodified lines
50	return 10;	50	return 10;
51	}	51	}
52		52
		53	SEC("sk_skb/stream_parser")
		54	int prog_skb_parser_resize(struct __sk_buff *skb)
		55	{
		56	bpf_skb_change_tail(skb, skb->len, 0);
		57	return skb->len;
		58	}
		59
53	char _license[] SEC("license") = "GPL";	60	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/verifier_unpriv.c+19−0

@@ -6,6 +6,8 @@ · 5 unmodified lines
6	#include "../../../include/linux/filter.h"	6	#include "../../../include/linux/filter.h"
7	#include "bpf_misc.h"	7	#include "bpf_misc.h"
8		8
		9	extern const int bpf_prog_active __ksym;
		10
9	#define BPF_SK_LOOKUP(func) \	11	#define BPF_SK_LOOKUP(func) \
10	/* struct bpf_sock_tuple tuple = {} */ \	12	/* struct bpf_sock_tuple tuple = {} */ \
11	"r2 = 0;" \	13	"r2 = 0;" \
@@ -77,6 +79,23 @@ __naked void dummy_prog_loop1_socket(void) · 65 unmodified lines
77	: __clobber_all);	79	: __clobber_all);
78	}	80	}
79		81
		82	SEC("socket")
		83	__description("unpriv: pseudo btf id log masks address")
		84	__success_unpriv
		85	__msg_unpriv("0: (18) r1 = 0x0")
		86	__not_msg_unpriv("0: (18) r1 = 0x{{[1-9a-f][0-9a-f]*}}")
		87	__retval_unpriv(0)
		88	__log_level(2)
		89	__naked void pseudo_btf_id_log_masks_address(void)
		90	{
		91	asm volatile ("r1 = %[bpf_prog_active] ll;"
		92	"r0 = 0;"
		93	"exit;"
		94	:
		95	: __imm_addr(bpf_prog_active)
		96	: __clobber_all);
		97	}
		98
80	SEC("socket")	99	SEC("socket")
81	__description("unpriv: return pointer")	100	__description("unpriv: return pointer")
82	__success __failure_unpriv __msg_unpriv("R0 leaks addr")	101	__success __failure_unpriv __msg_unpriv("R0 leaks addr")

Merge tag 'bpf-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code