Merge patch series "bpf, riscv: use BPF prog pack allocator in BPF JIT"

Puranjay Mohan <puranjay12@gmail.com> says: Here is some data to prove the V2 fixes the problem: Without this series: root@rv-selftester:~/src/kselftest/bpf# time ./test_tag test_tag: OK (40945 tests) real 7m47.562s user 0m24.145s sys 6m37.064s With this series applied: root@rv-selftester:~/src/selftest/bpf# time ./test_tag test_tag: OK (40945 tests) real 7m29.472s user 0m25.865s sys 6m18.401s BPF programs currently consume a page each on RISCV. For systems with many BPF programs, this adds significant pressure to instruction TLB. High iTLB pressure usually causes slow down for the whole system. Song Liu introduced the BPF prog pack allocator[1] to mitigate the above issue. It packs multiple BPF programs into a single huge page. It is currently only enabled for the x86_64 BPF JIT. I enabled this allocator on the ARM64 BPF JIT[2]. It is being reviewed now. This patch series enables the BPF prog pack allocator for the RISCV BPF JIT. ====================================================== Performance Analysis of prog pack allocator on RISCV64 ====================================================== Test setup: =========== Host machine: Debian GNU/Linux 11 (bullseye) Qemu Version: QEMU emulator version 8.0.3 (Debian 1:8.0.3+dfsg-1) u-boot-qemu Version: 2023.07+dfsg-1 opensbi Version: 1.3-1 To test the performance of the BPF prog pack allocator on RV, a stresser tool[4] linked below was built. This tool loads 8 BPF programs on the system and triggers 5 of them in an infinite loop by doing system calls. The runner script starts 20 instances of the above which loads 8*20=160 BPF programs on the system, 5*20=100 of which are being constantly triggered. The script is passed a command which would be run in the above environment. The script was run with following perf command: ./run.sh "perf stat -a \ -e iTLB-load-misses \ -e dTLB-load-misses \ -e dTLB-store-misses \ -e instructions \ --timeout 60000" The output of the above command is discussed below before and after enabling the BPF prog pack allocator. The tests were run on qemu-system-riscv64 with 8 cpus, 16G memory. The rootfs was created using Bjorn's riscv-cross-builder[5] docker container linked below. Results ======= Before enabling prog pack allocator: ------------------------------------ Performance counter stats for 'system wide': 4939048 iTLB-load-misses 5468689 dTLB-load-misses 465234 dTLB-store-misses 1441082097998 instructions 60.045791200 seconds time elapsed After enabling prog pack allocator: ----------------------------------- Performance counter stats for 'system wide': 3430035 iTLB-load-misses 5008745 dTLB-load-misses 409944 dTLB-store-misses 1441535637988 instructions 60.046296600 seconds time elapsed Improvements in metrics ======================= It was expected that the iTLB-load-misses would decrease as now a single huge page is used to keep all the BPF programs compared to a single page for each program earlier. -------------------------------------------- The improvement in iTLB-load-misses: -30.5 % -------------------------------------------- I repeated this expriment more than 100 times in different setups and the improvement was always greater than 30%. This patch series is boot tested on the Starfive VisionFive 2 board[6]. The performance analysis was not done on the board because it doesn't expose iTLB-load-misses, etc. The stresser program was run on the board to test the loading and unloading of BPF programs [1] https://lore.kernel.org/bpf/20220204185742.271030-1-song@kernel.org/ [2] https://lore.kernel.org/all/20230626085811.3192402-1-puranjay12@gmail.com/ [3] https://lore.kernel.org/all/20230626085811.3192402-2-puranjay12@gmail.com/ [4] https://github.com/puranjaymohan/BPF-Allocator-Bench [5] https://github.com/bjoto/riscv-cross-builder [6] https://www.starfivetech.com/en/site/boards * b4-shazam-merge: bpf, riscv: use prog pack allocator in the BPF JIT riscv: implement a memset like function for text riscv: extend patch_text_nosync() for multiple pages bpf: make bpf_prog_pack allocator portable

Palmer Dabbelt committed 3 years ago

Commit 77eea552 parents f578055 48a8f786 files · +255 −37Browse files

Unified · Split

6 files

arch/riscv/include/asm/patch.h +1 −0
arch/riscv/kernel/patch.c +109 −5
arch/riscv/net/bpf_jit.h +3 −0
arch/riscv/net/bpf_jit_comp64.c +48 −12
arch/riscv/net/bpf_jit_core.c +90 −16
kernel/bpf/core.c +4 −4

arch/riscv/include/asm/patch.h+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#define _ASM_RISCV_PATCH_H	7	#define _ASM_RISCV_PATCH_H
8		8
9	int patch_text_nosync(void addr, const void insns, size_t len);	9	int patch_text_nosync(void addr, const void insns, size_t len);
		10	int patch_text_set_nosync(void *addr, u8 c, size_t len);
10	int patch_text(void addr, u32 insns, int ninsns);	11	int patch_text(void addr, u32 insns, int ninsns);
11		12
12	extern int riscv_patch_in_stop_machine;	13	extern int riscv_patch_in_stop_machine;

arch/riscv/kernel/patch.c+109−5

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	#include <linux/spinlock.h>	6	#include <linux/spinlock.h>
7	#include <linux/mm.h>	7	#include <linux/mm.h>
8	#include <linux/memory.h>	8	#include <linux/memory.h>
		9	#include <linux/string.h>
9	#include <linux/uaccess.h>	10	#include <linux/uaccess.h>
10	#include <linux/stop_machine.h>	11	#include <linux/stop_machine.h>
11	#include <asm/kprobes.h>	12	#include <asm/kprobes.h>
@@ -53,12 +54,51 @@ static void patch_unmap(int fixmap) · 41 unmodified lines
53	}	54	}
54	NOKPROBE_SYMBOL(patch_unmap);	55	NOKPROBE_SYMBOL(patch_unmap);
55		56
56	static int patch_insn_write(void addr, const void insn, size_t len)	57	static int __patch_insn_set(void *addr, u8 c, size_t len)
		58	{
		59	void *waddr = addr;
		60	bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE;
		61
		62	/*
		63	* Only two pages can be mapped at a time for writing.
		64	*/
		65	if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
		66	return -EINVAL;
		67	/*
		68	* Before reaching here, it was expected to lock the text_mutex
		69	* already, so we don't need to give another lock here and could
		70	* ensure that it was safe between each cores.
		71	*/
		72	lockdep_assert_held(&text_mutex);
		73
		74	if (across_pages)
		75	patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
		76
		77	waddr = patch_map(addr, FIX_TEXT_POKE0);
		78
		79	memset(waddr, c, len);
		80
		81	patch_unmap(FIX_TEXT_POKE0);
		82
		83	if (across_pages)
		84	patch_unmap(FIX_TEXT_POKE1);
		85
		86	return 0;
		87	}
		88	NOKPROBE_SYMBOL(__patch_insn_set);
		89
		90	static int __patch_insn_write(void addr, const void insn, size_t len)
57	{	91	{
58	void *waddr = addr;	92	void *waddr = addr;
59	bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;	93	bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
60	int ret;	94	int ret;
61		95
		96	/*
		97	* Only two pages can be mapped at a time for writing.
		98	*/
		99	if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
		100	return -EINVAL;
		101
62	/*	102	/*
63	* Before reaching here, it was expected to lock the text_mutex	103	* Before reaching here, it was expected to lock the text_mutex
64	* already, so we don't need to give another lock here and could	104	* already, so we don't need to give another lock here and could
@@ -74,7 +114,7 @@ static int patch_insn_write(void addr, const void insn, size_t len) · 9 unmodified lines
74	lockdep_assert_held(&text_mutex);	114	lockdep_assert_held(&text_mutex);
75		115
76	if (across_pages)	116	if (across_pages)
77	patch_map(addr + len, FIX_TEXT_POKE1);	117	patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
78		118
79	waddr = patch_map(addr, FIX_TEXT_POKE0);	119	waddr = patch_map(addr, FIX_TEXT_POKE0);
80		120
@@ -87,15 +127,79 @@ static int patch_insn_write(void addr, const void insn, size_t len) · 6 unmodified lines
87		127
88	return ret;	128	return ret;
89	}	129	}
90	NOKPROBE_SYMBOL(patch_insn_write);	130	NOKPROBE_SYMBOL(__patch_insn_write);
91	#else	131	#else
92	static int patch_insn_write(void addr, const void insn, size_t len)	132	static int __patch_insn_set(void *addr, u8 c, size_t len)
		133	{
		134	memset(addr, c, len);
		135
		136	return 0;
		137	}
		138	NOKPROBE_SYMBOL(__patch_insn_set);
		139
		140	static int __patch_insn_write(void addr, const void insn, size_t len)
93	{	141	{
94	return copy_to_kernel_nofault(addr, insn, len);	142	return copy_to_kernel_nofault(addr, insn, len);
95	}	143	}
96	NOKPROBE_SYMBOL(patch_insn_write);	144	NOKPROBE_SYMBOL(__patch_insn_write);
97	#endif /* CONFIG_MMU */	145	#endif /* CONFIG_MMU */
98		146
		147	static int patch_insn_set(void *addr, u8 c, size_t len)
		148	{
		149	size_t patched = 0;
		150	size_t size;
		151	int ret = 0;
		152
		153	/*
		154	* __patch_insn_set() can only work on 2 pages at a time so call it in a
		155	* loop with len <= 2 * PAGE_SIZE.
		156	*/
		157	while (patched < len && !ret) {
		158	size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
		159	ret = __patch_insn_set(addr + patched, c, size);
		160
		161	patched += size;
		162	}
		163
		164	return ret;
		165	}
		166	NOKPROBE_SYMBOL(patch_insn_set);
		167
		168	int patch_text_set_nosync(void *addr, u8 c, size_t len)
		169	{
		170	u32 *tp = addr;
		171	int ret;
		172
		173	ret = patch_insn_set(tp, c, len);
		174
		175	if (!ret)
		176	flush_icache_range((uintptr_t)tp, (uintptr_t)tp + len);
		177
		178	return ret;
		179	}
		180	NOKPROBE_SYMBOL(patch_text_set_nosync);
		181
		182	static int patch_insn_write(void addr, const void insn, size_t len)
		183	{
		184	size_t patched = 0;
		185	size_t size;
		186	int ret = 0;
		187
		188	/*
		189	* Copy the instructions to the destination address, two pages at a time
		190	* because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE.
		191	*/
		192	while (patched < len && !ret) {
		193	size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
		194	ret = __patch_insn_write(addr + patched, insn + patched, size);
		195
		196	patched += size;
		197	}
		198
		199	return ret;
		200	}
		201	NOKPROBE_SYMBOL(patch_insn_write);
		202
99	int patch_text_nosync(void addr, const void insns, size_t len)	203	int patch_text_nosync(void addr, const void insns, size_t len)
100	{	204	{
101	u32 *tp = addr;	205	u32 *tp = addr;

arch/riscv/net/bpf_jit.h+3−0

 struct rv_jit_context {
 	struct bpf_prog *prog;
 	u16 *insns;		/* RV insns */
+	u16 *ro_insns;
 	int ninsns;
 	int prologue_len;
 	int epilogue_offset;
 struct rv_jit_data {
 	struct bpf_binary_header *header;
+	struct bpf_binary_header *ro_header;
 	u8 *image;
+	u8 *ro_image;
 	struct rv_jit_context ctx;
 };

arch/riscv/net/bpf_jit_comp64.c+48−12

 /* Emit fixed-length instructions for address */
 static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx)
 {
-	u64 ip = (u64)(ctx->insns + ctx->ninsns);
+	/*
+	 * Use the ro_insns(RX) to calculate the offset as the BPF program will
+	 * finally run from this memory region.
+	 */
+	u64 ip = (u64)(ctx->ro_insns + ctx->ninsns);
 	s64 off = addr - ip;
 	s64 upper = (off + (1 << 11)) >> 12;
 	s64 lower = off & 0xfff;
 	s64 off = 0;
 	u64 ip;
-	if (addr && ctx->insns) {
+	if (addr && ctx->insns && ctx->ro_insns) {
-		ip = (u64)(long)(ctx->insns + ctx->ninsns);
+		/*
+		 * Use the ro_insns(RX) to calculate the offset as the BPF
+		 * program will finally run from this memory region.
+		 */
+		ip = (u64)(long)(ctx->ro_insns + ctx->ninsns);
 		off = addr - ip;
 	}
 {
 	struct exception_table_entry *ex;
 	unsigned long pc;
-	off_t offset;
+	off_t ins_offset;
+	off_t fixup_offset;
-	if (!ctx->insns || !ctx->prog->aux->extable ||
+	if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable ||
 	    (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX))
 		return 0;
 		return -EINVAL;
 	ex = &ctx->prog->aux->extable[ctx->nexentries];
-	pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len];
+	pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len];
-	offset = pc - (long)&ex->insn;
+	/*
-	if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN))
+	 * This is the relative offset of the instruction that may fault from
+	 * the exception table itself. This will be written to the exception
+	 * table and if this instruction faults, the destination register will
+	 * be set to '0' and the execution will jump to the next instruction.
+	 */
+	ins_offset = pc - (long)&ex->insn;
+	if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
 		return -ERANGE;
-	ex->insn = offset;
 	/*
 	 * Since the extable follows the program, the fixup offset is always
 	 * bits. We don't need to worry about buildtime or runtime sort
 	 * modifying the upper bits because the table is already sorted, and
 	 * isn't part of the main exception table.
+	 *
+	 * The fixup_offset is set to the next instruction from the instruction
+	 * that may fault. The execution will jump to this after handling the
+	 * fault.
 	 */
-	offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
+	fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
-	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
+	if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
 		return -ERANGE;
-	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) |
+	/*
+	 * The offsets above have been calculated using the RO buffer but we
+	 * need to use the R/W buffer for writes.
+	 * switch ex to rw buffer for writing.
+	 */
+	ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns);
+	ex->insn = ins_offset;
+	ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) |
 		FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
 	ex->type = EX_TYPE_BPF;
 	ctx.ninsns = 0;
 	ctx.insns = NULL;
+	ctx.ro_insns = NULL;
 	ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
 	if (ret < 0)
 		return ret;
 		return -EFBIG;
 	ctx.ninsns = 0;
+	/*
+	 * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the
+	 * JITed instructions and later copies it to a RX region (ctx.ro_insns).
+	 * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the
+	 * trampoline image uses the same memory area for writing and execution,
+	 * both ctx.insns and ctx.ro_insns can be set to image.
+	 */
 	ctx.insns = image;
+	ctx.ro_insns = image;
 	ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
 	if (ret < 0)
 		return ret;

arch/riscv/net/bpf_jit_core.c+90−16

@@ -8,6 +8,8 @@ · 7 unmodified lines
8		8
9	#include <linux/bpf.h>	9	#include <linux/bpf.h>
10	#include <linux/filter.h>	10	#include <linux/filter.h>
		11	#include <linux/memory.h>
		12	#include <asm/patch.h>
11	#include "bpf_jit.h"	13	#include "bpf_jit.h"
12		14
13	/* Number of iterations to try until offsets converge. */	15	/* Number of iterations to try until offsets converge. */
@@ -117,16 +119,24 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 103 unmodified lines
117	sizeof(struct exception_table_entry);	119	sizeof(struct exception_table_entry);
118	prog_size = sizeof(ctx->insns) ctx->ninsns;	120	prog_size = sizeof(ctx->insns) ctx->ninsns;
119		121
120	jit_data->header =	122	jit_data->ro_header =
121	bpf_jit_binary_alloc(prog_size + extable_size,	123	bpf_jit_binary_pack_alloc(prog_size + extable_size,
122	&jit_data->image,	124	&jit_data->ro_image, sizeof(u32),
123	sizeof(u32),	125	&jit_data->header, &jit_data->image,
124	bpf_fill_ill_insns);	126	bpf_fill_ill_insns);
125	if (!jit_data->header) {	127	if (!jit_data->ro_header) {
126	prog = orig_prog;	128	prog = orig_prog;
127	goto out_offset;	129	goto out_offset;
128	}	130	}
129		131
		132	/*
		133	* Use the image(RW) for writing the JITed instructions. But also save
		134	* the ro_image(RX) for calculating the offsets in the image. The RW
		135	* image will be later copied to the RX image from where the program
		136	* will run. The bpf_jit_binary_pack_finalize() will do this copy in the
		137	* final step.
		138	*/
		139	ctx->ro_insns = (u16 *)jit_data->ro_image;
130	ctx->insns = (u16 *)jit_data->image;	140	ctx->insns = (u16 *)jit_data->image;
131	/*	141	/*
132	* Now, when the image is allocated, the image can	142	* Now, when the image is allocated, the image can
@@ -138,14 +148,12 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 5 unmodified lines
138		148
139	if (i == NR_JIT_ITERATIONS) {	149	if (i == NR_JIT_ITERATIONS) {
140	pr_err("bpf-jit: image did not converge in <%d passes!\n", i);	150	pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
141	if (jit_data->header)
142	bpf_jit_binary_free(jit_data->header);
143	prog = orig_prog;	151	prog = orig_prog;
144	goto out_offset;	152	goto out_free_hdr;
145	}	153	}
146		154
147	if (extable_size)	155	if (extable_size)
148	prog->aux->extable = (void *)ctx->insns + prog_size;	156	prog->aux->extable = (void *)ctx->ro_insns + prog_size;
149		157
150	skip_init_ctx:	158	skip_init_ctx:
151	pass++;	159	pass++;
@@ -154,23 +162,33 @@ skip_init_ctx: · 2 unmodified lines
154		162
155	bpf_jit_build_prologue(ctx);	163	bpf_jit_build_prologue(ctx);
156	if (build_body(ctx, extra_pass, NULL)) {	164	if (build_body(ctx, extra_pass, NULL)) {
157	bpf_jit_binary_free(jit_data->header);
158	prog = orig_prog;	165	prog = orig_prog;
159	goto out_offset;	166	goto out_free_hdr;
160	}	167	}
161	bpf_jit_build_epilogue(ctx);	168	bpf_jit_build_epilogue(ctx);
162		169
163	if (bpf_jit_enable > 1)	170	if (bpf_jit_enable > 1)
164	bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);	171	bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
165		172
166	prog->bpf_func = (void *)ctx->insns;	173	prog->bpf_func = (void *)ctx->ro_insns;
167	prog->jited = 1;	174	prog->jited = 1;
168	prog->jited_len = prog_size;	175	prog->jited_len = prog_size;
169		176
170	bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
171
172	if (!prog->is_func \|\| extra_pass) {	177	if (!prog->is_func \|\| extra_pass) {
173	bpf_jit_binary_lock_ro(jit_data->header);	178	if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
		179	jit_data->header))) {
		180	/* ro_header has been freed */
		181	jit_data->ro_header = NULL;
		182	prog = orig_prog;
		183	goto out_offset;
		184	}
		185	/*
		186	* The instructions have now been copied to the ROX region from
		187	* where they will execute.
		188	* Write any modified data cache blocks out to memory and
		189	* invalidate the corresponding blocks in the instruction cache.
		190	*/
		191	bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
174	for (i = 0; i < prog->len; i++)	192	for (i = 0; i < prog->len; i++)
175	ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);	193	ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
176	bpf_prog_fill_jited_linfo(prog, ctx->offset);	194	bpf_prog_fill_jited_linfo(prog, ctx->offset);
@@ -185,6 +203,14 @@ out: · 8 unmodified lines
185	bpf_jit_prog_release_other(prog, prog == orig_prog ?	203	bpf_jit_prog_release_other(prog, prog == orig_prog ?
186	tmp : orig_prog);	204	tmp : orig_prog);
187	return prog;	205	return prog;
		206
		207	out_free_hdr:
		208	if (jit_data->header) {
		209	bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
		210	sizeof(jit_data->header->size));
		211	bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
		212	}
		213	goto out_offset;
188	}	214	}
189		215
190	u64 bpf_jit_alloc_exec_limit(void)	216	u64 bpf_jit_alloc_exec_limit(void)
@@ -204,3 +230,51 @@ void bpf_jit_free_exec(void *addr) · 13 unmodified lines
204	{	230	{
205	return vfree(addr);	231	return vfree(addr);
206	}	232	}
		233
		234	void bpf_arch_text_copy(void dst, void *src, size_t len)
		235	{
		236	int ret;
		237
		238	mutex_lock(&text_mutex);
		239	ret = patch_text_nosync(dst, src, len);
		240	mutex_unlock(&text_mutex);
		241
		242	if (ret)
		243	return ERR_PTR(-EINVAL);
		244
		245	return dst;
		246	}
		247
		248	int bpf_arch_text_invalidate(void *dst, size_t len)
		249	{
		250	int ret;
		251
		252	mutex_lock(&text_mutex);
		253	ret = patch_text_set_nosync(dst, 0, len);
		254	mutex_unlock(&text_mutex);
		255
		256	return ret;
		257	}
		258
		259	void bpf_jit_free(struct bpf_prog *prog)
		260	{
		261	if (prog->jited) {
		262	struct rv_jit_data *jit_data = prog->aux->jit_data;
		263	struct bpf_binary_header *hdr;
		264
		265	/*
		266	* If we fail the final pass of JIT (from jit_subprogs),
		267	* the program may not be finalized yet. Call finalize here
		268	* before freeing it.
		269	*/
		270	if (jit_data) {
		271	bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
		272	kfree(jit_data);
		273	}
		274	hdr = bpf_jit_binary_pack_hdr(prog);
		275	bpf_jit_binary_pack_free(hdr, NULL);
		276	WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
		277	}
		278
		279	bpf_prog_unlock_free(prog);
		280	}

kernel/bpf/core.c+4−4

@@ -870,7 +870,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins · 869 unmodified lines
870	GFP_KERNEL);	870	GFP_KERNEL);
871	if (!pack)	871	if (!pack)
872	return NULL;	872	return NULL;
873	pack->ptr = module_alloc(BPF_PROG_PACK_SIZE);	873	pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE);
874	if (!pack->ptr) {	874	if (!pack->ptr) {
875	kfree(pack);	875	kfree(pack);
876	return NULL;	876	return NULL;
@@ -894,7 +894,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) · 17 unmodified lines
894	mutex_lock(&pack_mutex);	894	mutex_lock(&pack_mutex);
895	if (size > BPF_PROG_PACK_SIZE) {	895	if (size > BPF_PROG_PACK_SIZE) {
896	size = round_up(size, PAGE_SIZE);	896	size = round_up(size, PAGE_SIZE);
897	ptr = module_alloc(size);	897	ptr = bpf_jit_alloc_exec(size);
898	if (ptr) {	898	if (ptr) {
899	bpf_fill_ill_insns(ptr, size);	899	bpf_fill_ill_insns(ptr, size);
900	set_vm_flush_reset_perms(ptr);	900	set_vm_flush_reset_perms(ptr);
@@ -932,7 +932,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 31 unmodified lines
932		932
933	mutex_lock(&pack_mutex);	933	mutex_lock(&pack_mutex);
934	if (hdr->size > BPF_PROG_PACK_SIZE) {	934	if (hdr->size > BPF_PROG_PACK_SIZE) {
935	module_memfree(hdr);	935	bpf_jit_free_exec(hdr);
936	goto out;	936	goto out;
937	}	937	}
938		938
@@ -956,7 +956,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 17 unmodified lines
956	if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,	956	if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
957	BPF_PROG_CHUNK_COUNT, 0) == 0) {	957	BPF_PROG_CHUNK_COUNT, 0) == 0) {
958	list_del(&pack->list);	958	list_del(&pack->list);
959	module_memfree(pack->ptr);	959	bpf_jit_free_exec(pack->ptr);
960	kfree(pack);	960	kfree(pack);
961	}	961	}
962	out:	962	out:

Merge patch series "bpf, riscv: use BPF prog pack allocator in BPF JIT"

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code