bpf: Introduce load-acquire and store-release instructions

Introduce BPF instructions with load-acquire and store-release semantics, as discussed in [1]. Define 2 new flags: #define BPF_LOAD_ACQ 0x100 #define BPF_STORE_REL 0x110 A "load-acquire" is a BPF_STX | BPF_ATOMIC instruction with the 'imm' field set to BPF_LOAD_ACQ (0x100). Similarly, a "store-release" is a BPF_STX | BPF_ATOMIC instruction with the 'imm' field set to BPF_STORE_REL (0x110). Unlike existing atomic read-modify-write operations that only support BPF_W (32-bit) and BPF_DW (64-bit) size modifiers, load-acquires and store-releases also support BPF_B (8-bit) and BPF_H (16-bit). As an exception, however, 64-bit load-acquires/store-releases are not supported on 32-bit architectures (to fix a build error reported by the kernel test robot). An 8- or 16-bit load-acquire zero-extends the value before writing it to a 32-bit register, just like ARM64 instruction LDARH and friends. Similar to existing atomic read-modify-write operations, misaligned load-acquires/store-releases are not allowed (even if BPF_F_ANY_ALIGNMENT is set). As an example, consider the following 64-bit load-acquire BPF instruction (assuming little-endian): db 10 00 00 00 01 00 00 r0 = load_acquire((u64 *)(r1 + 0x0)) opcode (0xdb): BPF_ATOMIC | BPF_DW | BPF_STX imm (0x00000100): BPF_LOAD_ACQ Similarly, a 16-bit BPF store-release: cb 21 00 00 10 01 00 00 store_release((u16 *)(r1 + 0x0), w2) opcode (0xcb): BPF_ATOMIC | BPF_H | BPF_STX imm (0x00000110): BPF_STORE_REL In arch/{arm64,s390,x86}/net/bpf_jit_comp.c, have bpf_jit_supports_insn(..., /*in_arena=*/true) return false for the new instructions, until the corresponding JIT compiler supports them in arena. [1] https://lore.kernel.org/all/20240729183246.4110549-1-yepeilin@google.com/

Peilin Ye committed 1 year ago

Commit 88044231 parent 3a6fa5710 files · +166 −13Browse files

Unified · Split

10 files

arch/arm64/net/bpf_jit_comp.c +4 −0
arch/s390/net/bpf_jit_comp.c +10 −4
arch/x86/net/bpf_jit_comp.c +4 −0
include/linux/bpf.h +15 −0
include/linux/filter.h +2 −0
include/uapi/linux/bpf.h +3 −0
kernel/bpf/core.c +61 −6
kernel/bpf/disasm.c +12 −0
kernel/bpf/verifier.c +52 −3
tools/include/uapi/linux/bpf.h +3 −0

arch/arm64/net/bpf_jit_comp.c+4−0

@@ -2667,8 +2667,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) · 2666 unmodified lines
2667	2667	if (!in_arena)
2668	2668	return true;
2669	2669	switch (insn->code) {
	2670	+ case BPF_STX \| BPF_ATOMIC \| BPF_B:
	2671	+ case BPF_STX \| BPF_ATOMIC \| BPF_H:
2670	2672	case BPF_STX \| BPF_ATOMIC \| BPF_W:
2671	2673	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
	2674	+ if (bpf_atomic_is_load_store(insn))
	2675	+ return false;
2672	2676	if (!cpus_have_cap(ARM64_HAS_LSE_ATOMICS))
2673	2677	return false;
2674	2678	}

arch/s390/net/bpf_jit_comp.c+10−4

@@ -2919,10 +2919,16 @@ bool bpf_jit_supports_arena(void) · 2918 unmodified lines
2919	2919
2920	2920	bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
2921	2921	{
2922		− /*
2923		− * Currently the verifier uses this function only to check which
2924		− * atomic stores to arena are supported, and they all are.
2925		− */
	2922	+ if (!in_arena)
	2923	+ return true;
	2924	+ switch (insn->code) {
	2925	+ case BPF_STX \| BPF_ATOMIC \| BPF_B:
	2926	+ case BPF_STX \| BPF_ATOMIC \| BPF_H:
	2927	+ case BPF_STX \| BPF_ATOMIC \| BPF_W:
	2928	+ case BPF_STX \| BPF_ATOMIC \| BPF_DW:
	2929	+ if (bpf_atomic_is_load_store(insn))
	2930	+ return false;
	2931	+ }
2926	2932	return true;
2927	2933	}
2928	2934

arch/x86/net/bpf_jit_comp.c+4−0

@@ -3771,8 +3771,12 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) · 3770 unmodified lines
3771	3771	if (!in_arena)
3772	3772	return true;
3773	3773	switch (insn->code) {
	3774	+ case BPF_STX \| BPF_ATOMIC \| BPF_B:
	3775	+ case BPF_STX \| BPF_ATOMIC \| BPF_H:
3774	3776	case BPF_STX \| BPF_ATOMIC \| BPF_W:
3775	3777	case BPF_STX \| BPF_ATOMIC \| BPF_DW:
	3778	+ if (bpf_atomic_is_load_store(insn))
	3779	+ return false;
3776	3780	if (insn->imm == (BPF_AND \| BPF_FETCH) \|\|
3777	3781	insn->imm == (BPF_OR \| BPF_FETCH) \|\|
3778	3782	insn->imm == (BPF_XOR \| BPF_FETCH))

include/linux/bpf.h+15−0

@@ -991,6 +991,21 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) · 990 unmodified lines
991	991	return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
992	992	}
993	993
	994	+/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an
	995	+ * atomic load or store, and false if it is a read-modify-write instruction.
	996	+ */
	997	+static inline bool
	998	+bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn)
	999	+{
	1000	+ switch (atomic_insn->imm) {
	1001	+ case BPF_LOAD_ACQ:
	1002	+ case BPF_STORE_REL:
	1003	+ return true;
	1004	+ default:
	1005	+ return false;
	1006	+ }
	1007	+}
	1008	+
994	1009	struct bpf_prog_ops {
995	1010	int (test_run)(struct bpf_prog prog, const union bpf_attr *kattr,
996	1011	union bpf_attr __user *uattr);

include/linux/filter.h+2−0

@@ -364,6 +364,8 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) · 363 unmodified lines
364	364	* BPF_XOR \| BPF_FETCH src_reg = atomic_fetch_xor(dst_reg + off16, src_reg);
365	365	* BPF_XCHG src_reg = atomic_xchg(dst_reg + off16, src_reg)
366	366	* BPF_CMPXCHG r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg)
	367	+ * BPF_LOAD_ACQ dst_reg = smp_load_acquire(src_reg + off16)
	368	+ * BPF_STORE_REL smp_store_release(dst_reg + off16, src_reg)
367	369	*/
368	370
369	371	#define BPF_ATOMIC_OP(SIZE, OP, DST, SRC, OFF) \

include/uapi/linux/bpf.h+3−0

@@ -51,6 +51,9 @@ · 50 unmodified lines
51	51	#define BPF_XCHG (0xe0 \| BPF_FETCH) /* atomic exchange */
52	52	#define BPF_CMPXCHG (0xf0 \| BPF_FETCH) /* atomic compare-and-write */
53	53
	54	+#define BPF_LOAD_ACQ 0x100 /* load-acquire */
	55	+#define BPF_STORE_REL 0x110 /* store-release */
	56	+
54	57	enum bpf_cond_pseudo_jmp {
55	58	BPF_MAY_GOTO = 0,
56	59	};

kernel/bpf/core.c+61−6

@@ -1663,14 +1663,17 @@ EXPORT_SYMBOL_GPL(__bpf_call_base); · 1662 unmodified lines
1663	1663	INSN_3(JMP, JSET, K), \
1664	1664	INSN_2(JMP, JA), \
1665	1665	INSN_2(JMP32, JA), \
	1666	+ /* Atomic operations. */ \
	1667	+ INSN_3(STX, ATOMIC, B), \
	1668	+ INSN_3(STX, ATOMIC, H), \
	1669	+ INSN_3(STX, ATOMIC, W), \
	1670	+ INSN_3(STX, ATOMIC, DW), \
1666	1671	/* Store instructions. */ \
1667	1672	/* Register based. */ \
1668	1673	INSN_3(STX, MEM, B), \
1669	1674	INSN_3(STX, MEM, H), \
1670	1675	INSN_3(STX, MEM, W), \
1671	1676	INSN_3(STX, MEM, DW), \
1672		− INSN_3(STX, ATOMIC, W), \
1673		− INSN_3(STX, ATOMIC, DW), \
1674	1677	/* Immediate based. */ \
1675	1678	INSN_3(ST, MEM, B), \
1676	1679	INSN_3(ST, MEM, H), \
@@ -2152,24 +2155,33 @@ out: · 475 unmodified lines
2152	2155	if (BPF_SIZE(insn->code) == BPF_W) \
2153	2156	atomic_##KOP((u32) SRC, (atomic_t *)(unsigned long) \
2154	2157	(DST + insn->off)); \
2155		− else \
	2158	+ else if (BPF_SIZE(insn->code) == BPF_DW) \
2156	2159	atomic64_##KOP((u64) SRC, (atomic64_t *)(unsigned long) \
2157	2160	(DST + insn->off)); \
	2161	+ else \
	2162	+ goto default_label; \
2158	2163	break; \
2159	2164	case BOP \| BPF_FETCH: \
2160	2165	if (BPF_SIZE(insn->code) == BPF_W) \
2161	2166	SRC = (u32) atomic_fetch_##KOP( \
2162	2167	(u32) SRC, \
2163	2168	(atomic_t *)(unsigned long) (DST + insn->off)); \
2164		− else \
	2169	+ else if (BPF_SIZE(insn->code) == BPF_DW) \
2165	2170	SRC = (u64) atomic64_fetch_##KOP( \
2166	2171	(u64) SRC, \
2167	2172	(atomic64_t *)(unsigned long) (DST + insn->off)); \
	2173	+ else \
	2174	+ goto default_label; \
2168	2175	break;
2169	2176
2170	2177	STX_ATOMIC_DW:
2171	2178	STX_ATOMIC_W:
	2179	+ STX_ATOMIC_H:
	2180	+ STX_ATOMIC_B:
2172	2181	switch (IMM) {
	2182	+ /* Atomic read-modify-write instructions support only W and DW
	2183	+ * size modifiers.
	2184	+ */
2173	2185	ATOMIC_ALU_OP(BPF_ADD, add)
2174	2186	ATOMIC_ALU_OP(BPF_AND, and)
2175	2187	ATOMIC_ALU_OP(BPF_OR, or)
@@ -2181,20 +2193,63 @@ out: · 5 unmodified lines
2181	2193	SRC = (u32) atomic_xchg(
2182	2194	(atomic_t *)(unsigned long) (DST + insn->off),
2183	2195	(u32) SRC);
2184		− else
	2196	+ else if (BPF_SIZE(insn->code) == BPF_DW)
2185	2197	SRC = (u64) atomic64_xchg(
2186	2198	(atomic64_t *)(unsigned long) (DST + insn->off),
2187	2199	(u64) SRC);
	2200	+ else
	2201	+ goto default_label;
2188	2202	break;
2189	2203	case BPF_CMPXCHG:
2190	2204	if (BPF_SIZE(insn->code) == BPF_W)
2191	2205	BPF_R0 = (u32) atomic_cmpxchg(
2192	2206	(atomic_t *)(unsigned long) (DST + insn->off),
2193	2207	(u32) BPF_R0, (u32) SRC);
2194		− else
	2208	+ else if (BPF_SIZE(insn->code) == BPF_DW)
2195	2209	BPF_R0 = (u64) atomic64_cmpxchg(
2196	2210	(atomic64_t *)(unsigned long) (DST + insn->off),
2197	2211	(u64) BPF_R0, (u64) SRC);
	2212	+ else
	2213	+ goto default_label;
	2214	+ break;
	2215	+ /* Atomic load and store instructions support all size
	2216	+ * modifiers.
	2217	+ */
	2218	+ case BPF_LOAD_ACQ:
	2219	+ switch (BPF_SIZE(insn->code)) {
	2220	+#define LOAD_ACQUIRE(SIZEOP, SIZE) \
	2221	+ case BPF_##SIZEOP: \
	2222	+ DST = (SIZE)smp_load_acquire( \
	2223	+ (SIZE *)(unsigned long)(SRC + insn->off)); \
	2224	+ break;
	2225	+ LOAD_ACQUIRE(B, u8)
	2226	+ LOAD_ACQUIRE(H, u16)
	2227	+ LOAD_ACQUIRE(W, u32)
	2228	+#ifdef CONFIG_64BIT
	2229	+ LOAD_ACQUIRE(DW, u64)
	2230	+#endif
	2231	+#undef LOAD_ACQUIRE
	2232	+ default:
	2233	+ goto default_label;
	2234	+ }
	2235	+ break;
	2236	+ case BPF_STORE_REL:
	2237	+ switch (BPF_SIZE(insn->code)) {
	2238	+#define STORE_RELEASE(SIZEOP, SIZE) \
	2239	+ case BPF_##SIZEOP: \
	2240	+ smp_store_release( \
	2241	+ (SIZE *)(unsigned long)(DST + insn->off), (SIZE)SRC); \
	2242	+ break;
	2243	+ STORE_RELEASE(B, u8)
	2244	+ STORE_RELEASE(H, u16)
	2245	+ STORE_RELEASE(W, u32)
	2246	+#ifdef CONFIG_64BIT
	2247	+ STORE_RELEASE(DW, u64)
	2248	+#endif
	2249	+#undef STORE_RELEASE
	2250	+ default:
	2251	+ goto default_label;
	2252	+ }
2198	2253	break;
2199	2254
2200	2255	default:

kernel/bpf/disasm.c+12−0

@@ -267,6 +267,18 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, · 266 unmodified lines
267	267	BPF_SIZE(insn->code) == BPF_DW ? "64" : "",
268	268	bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
269	269	insn->dst_reg, insn->off, insn->src_reg);
	270	+ } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
	271	+ insn->imm == BPF_LOAD_ACQ) {
	272	+ verbose(cbs->private_data, "(%02x) r%d = load_acquire((%s *)(r%d %+d))\n",
	273	+ insn->code, insn->dst_reg,
	274	+ bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
	275	+ insn->src_reg, insn->off);
	276	+ } else if (BPF_MODE(insn->code) == BPF_ATOMIC &&
	277	+ insn->imm == BPF_STORE_REL) {
	278	+ verbose(cbs->private_data, "(%02x) store_release((%s *)(r%d %+d), r%d)\n",
	279	+ insn->code,
	280	+ bpf_ldst_string[BPF_SIZE(insn->code) >> 3],
	281	+ insn->dst_reg, insn->off, insn->src_reg);
270	282	} else {
271	283	verbose(cbs->private_data, "BUG_%02x\n", insn->code);
272	284	}

kernel/bpf/verifier.c+52−3

@@ -579,6 +579,13 @@ static bool is_cmpxchg_insn(const struct bpf_insn *insn) · 578 unmodified lines
579	579	insn->imm == BPF_CMPXCHG;
580	580	}
581	581
	582	+static bool is_atomic_load_insn(const struct bpf_insn *insn)
	583	+{
	584	+ return BPF_CLASS(insn->code) == BPF_STX &&
	585	+ BPF_MODE(insn->code) == BPF_ATOMIC &&
	586	+ insn->imm == BPF_LOAD_ACQ;
	587	+}
	588	+
582	589	static int __get_spi(s32 off)
583	590	{
584	591	return (-off - 1) / BPF_REG_SIZE;
@@ -3567,7 +3574,7 @@ static bool is_reg64(struct bpf_verifier_env env, struct bpf_insn insn, · 2982 unmodified lines
3567	3574	}
3568	3575
3569	3576	if (class == BPF_STX) {
3570		− /* BPF_STX (including atomic variants) has multiple source
	3577	+ /* BPF_STX (including atomic variants) has one or more source
3571	3578	* operands, one of which is a ptr. Check whether the caller is
3572	3579	* asking about it.
3573	3580	*/
@@ -4181,7 +4188,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, · 607 unmodified lines
4181	4188	* dreg still needs precision before this insn
4182	4189	*/
4183	4190	}
4184		− } else if (class == BPF_LDX) {
	4191	+ } else if (class == BPF_LDX \|\| is_atomic_load_insn(insn)) {
4185	4192	if (!bt_is_reg_set(bt, dreg))
4186	4193	return 0;
4187	4194	bt_clear_reg(bt, dreg);
@@ -7766,6 +7773,32 @@ static int check_atomic_rmw(struct bpf_verifier_env *env, · 3578 unmodified lines
7766	7773	return 0;
7767	7774	}
7768	7775
	7776	+static int check_atomic_load(struct bpf_verifier_env *env,
	7777	+ struct bpf_insn *insn)
	7778	+{
	7779	+ if (!atomic_ptr_type_ok(env, insn->src_reg, insn)) {
	7780	+ verbose(env, "BPF_ATOMIC loads from R%d %s is not allowed\n",
	7781	+ insn->src_reg,
	7782	+ reg_type_str(env, reg_state(env, insn->src_reg)->type));
	7783	+ return -EACCES;
	7784	+ }
	7785	+
	7786	+ return check_load_mem(env, insn, true, false, false, "atomic_load");
	7787	+}
	7788	+
	7789	+static int check_atomic_store(struct bpf_verifier_env *env,
	7790	+ struct bpf_insn *insn)
	7791	+{
	7792	+ if (!atomic_ptr_type_ok(env, insn->dst_reg, insn)) {
	7793	+ verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
	7794	+ insn->dst_reg,
	7795	+ reg_type_str(env, reg_state(env, insn->dst_reg)->type));
	7796	+ return -EACCES;
	7797	+ }
	7798	+
	7799	+ return check_store_reg(env, insn, true);
	7800	+}
	7801	+
7769	7802	static int check_atomic(struct bpf_verifier_env env, struct bpf_insn insn)
7770	7803	{
7771	7804	switch (insn->imm) {
@@ -7780,6 +7813,20 @@ static int check_atomic(struct bpf_verifier_env env, struct bpf_insn insn) · 8 unmodified lines
7780	7813	case BPF_XCHG:
7781	7814	case BPF_CMPXCHG:
7782	7815	return check_atomic_rmw(env, insn);
	7816	+ case BPF_LOAD_ACQ:
	7817	+ if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
	7818	+ verbose(env,
	7819	+ "64-bit load-acquires are only supported on 64-bit arches\n");
	7820	+ return -EOPNOTSUPP;
	7821	+ }
	7822	+ return check_atomic_load(env, insn);
	7823	+ case BPF_STORE_REL:
	7824	+ if (BPF_SIZE(insn->code) == BPF_DW && BITS_PER_LONG != 64) {
	7825	+ verbose(env,
	7826	+ "64-bit store-releases are only supported on 64-bit arches\n");
	7827	+ return -EOPNOTSUPP;
	7828	+ }
	7829	+ return check_atomic_store(env, insn);
7783	7830	default:
7784	7831	verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n",
7785	7832	insn->imm);
@@ -20605,7 +20652,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 12819 unmodified lines
20605	20652	insn->code == (BPF_ST \| BPF_MEM \| BPF_W) \|\|
20606	20653	insn->code == (BPF_ST \| BPF_MEM \| BPF_DW)) {
20607	20654	type = BPF_WRITE;
20608		− } else if ((insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_W) \|\|
	20655	+ } else if ((insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_B) \|\|
	20656	+ insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_H) \|\|
	20657	+ insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_W) \|\|
20609	20658	insn->code == (BPF_STX \| BPF_ATOMIC \| BPF_DW)) &&
20610	20659	env->insn_aux_data[i + delta].ptr_type == PTR_TO_ARENA) {
20611	20660	insn->code = BPF_STX \| BPF_PROBE_ATOMIC \| BPF_SIZE(insn->code);

tools/include/uapi/linux/bpf.h+3−0

@@ -51,6 +51,9 @@ · 50 unmodified lines
51	51	#define BPF_XCHG (0xe0 \| BPF_FETCH) /* atomic exchange */
52	52	#define BPF_CMPXCHG (0xf0 \| BPF_FETCH) /* atomic compare-and-write */
53	53
	54	+#define BPF_LOAD_ACQ 0x100 /* load-acquire */
	55	+#define BPF_STORE_REL 0x110 /* store-release */
	56	+
54	57	enum bpf_cond_pseudo_jmp {
55	58	BPF_MAY_GOTO = 0,
56	59	};

bpf: Introduce load-acquire and store-release instructions

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code