Merge patch series "bpf, riscv: use BPF prog pack allocator in BPF JIT"

Puranjay Mohan <puranjay12@gmail.com> says: Here is some data to prove the V2 fixes the problem: Without this series: root@rv-selftester:~/src/kselftest/bpf# time ./test_tag test_tag: OK (40945 tests) real 7m47.562s user 0m24.145s sys 6m37.064s With this series applied: root@rv-selftester:~/src/selftest/bpf# time ./test_tag test_tag: OK (40945 tests) real 7m29.472s user 0m25.865s sys 6m18.401s BPF programs currently consume a page each on RISCV. For systems with many BPF programs, this adds significant pressure to instruction TLB. High iTLB pressure usually causes slow down for the whole system. Song Liu introduced the BPF prog pack allocator[1] to mitigate the above issue. It packs multiple BPF programs into a single huge page. It is currently only enabled for the x86_64 BPF JIT. I enabled this allocator on the ARM64 BPF JIT[2]. It is being reviewed now. This patch series enables the BPF prog pack allocator for the RISCV BPF JIT. ====================================================== Performance Analysis of prog pack allocator on RISCV64 ====================================================== Test setup: =========== Host machine: Debian GNU/Linux 11 (bullseye) Qemu Version: QEMU emulator version 8.0.3 (Debian 1:8.0.3+dfsg-1) u-boot-qemu Version: 2023.07+dfsg-1 opensbi Version: 1.3-1 To test the performance of the BPF prog pack allocator on RV, a stresser tool[4] linked below was built. This tool loads 8 BPF programs on the system and triggers 5 of them in an infinite loop by doing system calls. The runner script starts 20 instances of the above which loads 8*20=160 BPF programs on the system, 5*20=100 of which are being constantly triggered. The script is passed a command which would be run in the above environment. The script was run with following perf command: ./run.sh "perf stat -a \ -e iTLB-load-misses \ -e dTLB-load-misses \ -e dTLB-store-misses \ -e instructions \ --timeout 60000" The output of the above command is discussed below before and after enabling the BPF prog pack allocator. The tests were run on qemu-system-riscv64 with 8 cpus, 16G memory. The rootfs was created using Bjorn's riscv-cross-builder[5] docker container linked below. Results ======= Before enabling prog pack allocator: ------------------------------------ Performance counter stats for 'system wide': 4939048 iTLB-load-misses 5468689 dTLB-load-misses 465234 dTLB-store-misses 1441082097998 instructions 60.045791200 seconds time elapsed After enabling prog pack allocator: ----------------------------------- Performance counter stats for 'system wide': 3430035 iTLB-load-misses 5008745 dTLB-load-misses 409944 dTLB-store-misses 1441535637988 instructions 60.046296600 seconds time elapsed Improvements in metrics ======================= It was expected that the iTLB-load-misses would decrease as now a single huge page is used to keep all the BPF programs compared to a single page for each program earlier. -------------------------------------------- The improvement in iTLB-load-misses: -30.5 % -------------------------------------------- I repeated this expriment more than 100 times in different setups and the improvement was always greater than 30%. This patch series is boot tested on the Starfive VisionFive 2 board[6]. The performance analysis was not done on the board because it doesn't expose iTLB-load-misses, etc. The stresser program was run on the board to test the loading and unloading of BPF programs [1] https://lore.kernel.org/bpf/20220204185742.271030-1-song@kernel.org/ [2] https://lore.kernel.org/all/20230626085811.3192402-1-puranjay12@gmail.com/ [3] https://lore.kernel.org/all/20230626085811.3192402-2-puranjay12@gmail.com/ [4] https://github.com/puranjaymohan/BPF-Allocator-Bench [5] https://github.com/bjoto/riscv-cross-builder [6] https://www.starfivetech.com/en/site/boards * b4-shazam-merge: bpf, riscv: use prog pack allocator in the BPF JIT riscv: implement a memset like function for text riscv: extend patch_text_nosync() for multiple pages bpf: make bpf_prog_pack allocator portable

Palmer Dabbelt committed 3 years ago

Commit 77eea552 parents f578055 48a8f786 files · +255 −37Browse files

Unified · Split

6 files

arch/riscv/include/asm/patch.h +1 −0
arch/riscv/kernel/patch.c +109 −5
arch/riscv/net/bpf_jit.h +3 −0
arch/riscv/net/bpf_jit_comp64.c +48 −12
arch/riscv/net/bpf_jit_core.c +90 −16
kernel/bpf/core.c +4 −4

arch/riscv/include/asm/patch.h+1−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#define _ASM_RISCV_PATCH_H
8	8
9	9	int patch_text_nosync(void addr, const void insns, size_t len);
	10	+int patch_text_set_nosync(void *addr, u8 c, size_t len);
10	11	int patch_text(void addr, u32 insns, int ninsns);
11	12
12	13	extern int riscv_patch_in_stop_machine;

arch/riscv/kernel/patch.c+109−5

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	#include <linux/spinlock.h>
7	7	#include <linux/mm.h>
8	8	#include <linux/memory.h>
	9	+#include <linux/string.h>
9	10	#include <linux/uaccess.h>
10	11	#include <linux/stop_machine.h>
11	12	#include <asm/kprobes.h>
@@ -53,12 +54,51 @@ static void patch_unmap(int fixmap) · 41 unmodified lines
53	54	}
54	55	NOKPROBE_SYMBOL(patch_unmap);
55	56
56		−static int patch_insn_write(void addr, const void insn, size_t len)
	57	+static int __patch_insn_set(void *addr, u8 c, size_t len)
	58	+{
	59	+ void *waddr = addr;
	60	+ bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE;
	61	+
	62	+ /*
	63	+ * Only two pages can be mapped at a time for writing.
	64	+ */
	65	+ if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
	66	+ return -EINVAL;
	67	+ /*
	68	+ * Before reaching here, it was expected to lock the text_mutex
	69	+ * already, so we don't need to give another lock here and could
	70	+ * ensure that it was safe between each cores.
	71	+ */
	72	+ lockdep_assert_held(&text_mutex);
	73	+
	74	+ if (across_pages)
	75	+ patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
	76	+
	77	+ waddr = patch_map(addr, FIX_TEXT_POKE0);
	78	+
	79	+ memset(waddr, c, len);
	80	+
	81	+ patch_unmap(FIX_TEXT_POKE0);
	82	+
	83	+ if (across_pages)
	84	+ patch_unmap(FIX_TEXT_POKE1);
	85	+
	86	+ return 0;
	87	+}
	88	+NOKPROBE_SYMBOL(__patch_insn_set);
	89	+
	90	+static int __patch_insn_write(void addr, const void insn, size_t len)
57	91	{
58	92	void *waddr = addr;
59	93	bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
60	94	int ret;
61	95
	96	+ /*
	97	+ * Only two pages can be mapped at a time for writing.
	98	+ */
	99	+ if (len + offset_in_page(addr) > 2 * PAGE_SIZE)
	100	+ return -EINVAL;
	101	+
62	102	/*
63	103	* Before reaching here, it was expected to lock the text_mutex
64	104	* already, so we don't need to give another lock here and could
@@ -74,7 +114,7 @@ static int patch_insn_write(void addr, const void insn, size_t len) · 9 unmodified lines
74	114	lockdep_assert_held(&text_mutex);
75	115
76	116	if (across_pages)
77		− patch_map(addr + len, FIX_TEXT_POKE1);
	117	+ patch_map(addr + PAGE_SIZE, FIX_TEXT_POKE1);
78	118
79	119	waddr = patch_map(addr, FIX_TEXT_POKE0);
80	120
@@ -87,15 +127,79 @@ static int patch_insn_write(void addr, const void insn, size_t len) · 6 unmodified lines
87	127
88	128	return ret;
89	129	}
90		−NOKPROBE_SYMBOL(patch_insn_write);
	130	+NOKPROBE_SYMBOL(__patch_insn_write);
91	131	#else
92		−static int patch_insn_write(void addr, const void insn, size_t len)
	132	+static int __patch_insn_set(void *addr, u8 c, size_t len)
	133	+{
	134	+ memset(addr, c, len);
	135	+
	136	+ return 0;
	137	+}
	138	+NOKPROBE_SYMBOL(__patch_insn_set);
	139	+
	140	+static int __patch_insn_write(void addr, const void insn, size_t len)
93	141	{
94	142	return copy_to_kernel_nofault(addr, insn, len);
95	143	}
96		−NOKPROBE_SYMBOL(patch_insn_write);
	144	+NOKPROBE_SYMBOL(__patch_insn_write);
97	145	#endif /* CONFIG_MMU */
98	146
	147	+static int patch_insn_set(void *addr, u8 c, size_t len)
	148	+{
	149	+ size_t patched = 0;
	150	+ size_t size;
	151	+ int ret = 0;
	152	+
	153	+ /*
	154	+ * __patch_insn_set() can only work on 2 pages at a time so call it in a
	155	+ * loop with len <= 2 * PAGE_SIZE.
	156	+ */
	157	+ while (patched < len && !ret) {
	158	+ size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
	159	+ ret = __patch_insn_set(addr + patched, c, size);
	160	+
	161	+ patched += size;
	162	+ }
	163	+
	164	+ return ret;
	165	+}
	166	+NOKPROBE_SYMBOL(patch_insn_set);
	167	+
	168	+int patch_text_set_nosync(void *addr, u8 c, size_t len)
	169	+{
	170	+ u32 *tp = addr;
	171	+ int ret;
	172	+
	173	+ ret = patch_insn_set(tp, c, len);
	174	+
	175	+ if (!ret)
	176	+ flush_icache_range((uintptr_t)tp, (uintptr_t)tp + len);
	177	+
	178	+ return ret;
	179	+}
	180	+NOKPROBE_SYMBOL(patch_text_set_nosync);
	181	+
	182	+static int patch_insn_write(void addr, const void insn, size_t len)
	183	+{
	184	+ size_t patched = 0;
	185	+ size_t size;
	186	+ int ret = 0;
	187	+
	188	+ /*
	189	+ * Copy the instructions to the destination address, two pages at a time
	190	+ * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE.
	191	+ */
	192	+ while (patched < len && !ret) {
	193	+ size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
	194	+ ret = __patch_insn_write(addr + patched, insn + patched, size);
	195	+
	196	+ patched += size;
	197	+ }
	198	+
	199	+ return ret;
	200	+}
	201	+NOKPROBE_SYMBOL(patch_insn_write);
	202	+
99	203	int patch_text_nosync(void addr, const void insns, size_t len)
100	204	{
101	205	u32 *tp = addr;

arch/riscv/net/bpf_jit.h+3−0

@@ -68,6 +68,7 @@ static inline bool is_creg(u8 reg) · 67 unmodified lines
68	68	struct rv_jit_context {
69	69	struct bpf_prog *prog;
70	70	u16 insns; / RV insns */
	71	+ u16 *ro_insns;
71	72	int ninsns;
72	73	int prologue_len;
73	74	int epilogue_offset;
@@ -85,7 +86,9 @@ static inline int ninsns_rvoff(int ninsns) · 11 unmodified lines
85	86
86	87	struct rv_jit_data {
87	88	struct bpf_binary_header *header;
	89	+ struct bpf_binary_header *ro_header;
88	90	u8 *image;
	91	+ u8 *ro_image;
89	92	struct rv_jit_context ctx;
90	93	};
91	94

arch/riscv/net/bpf_jit_comp64.c+48−12

@@ -144,7 +144,11 @@ static bool in_auipc_jalr_range(s64 val) · 143 unmodified lines
144	144	/* Emit fixed-length instructions for address */
145	145	static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx)
146	146	{
147		− u64 ip = (u64)(ctx->insns + ctx->ninsns);
	147	+ /*
	148	+ * Use the ro_insns(RX) to calculate the offset as the BPF program will
	149	+ * finally run from this memory region.
	150	+ */
	151	+ u64 ip = (u64)(ctx->ro_insns + ctx->ninsns);
148	152	s64 off = addr - ip;
149	153	s64 upper = (off + (1 << 11)) >> 12;
150	154	s64 lower = off & 0xfff;
@@ -464,8 +468,12 @@ static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) · 313 unmodified lines
464	468	s64 off = 0;
465	469	u64 ip;
466	470
467		− if (addr && ctx->insns) {
468		− ip = (u64)(long)(ctx->insns + ctx->ninsns);
	471	+ if (addr && ctx->insns && ctx->ro_insns) {
	472	+ /*
	473	+ * Use the ro_insns(RX) to calculate the offset as the BPF
	474	+ * program will finally run from this memory region.
	475	+ */
	476	+ ip = (u64)(long)(ctx->ro_insns + ctx->ninsns);
469	477	off = addr - ip;
470	478	}
471	479
@@ -578,9 +586,10 @@ static int add_exception_handler(const struct bpf_insn *insn, · 106 unmodified lines
578	586	{
579	587	struct exception_table_entry *ex;
580	588	unsigned long pc;
581		− off_t offset;
	589	+ off_t ins_offset;
	590	+ off_t fixup_offset;
582	591
583		− if (!ctx->insns \|\| !ctx->prog->aux->extable \|\|
	592	+ if (!ctx->insns \|\| !ctx->ro_insns \|\| !ctx->prog->aux->extable \|\|
584	593	(BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX))
585	594	return 0;
586	595
@@ -594,12 +603,17 @@ static int add_exception_handler(const struct bpf_insn *insn, · 7 unmodified lines
594	603	return -EINVAL;
595	604
596	605	ex = &ctx->prog->aux->extable[ctx->nexentries];
597		− pc = (unsigned long)&ctx->insns[ctx->ninsns - insn_len];
	606	+ pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len];
598	607
599		− offset = pc - (long)&ex->insn;
600		− if (WARN_ON_ONCE(offset >= 0 \|\| offset < INT_MIN))
	608	+ /*
	609	+ * This is the relative offset of the instruction that may fault from
	610	+ * the exception table itself. This will be written to the exception
	611	+ * table and if this instruction faults, the destination register will
	612	+ * be set to '0' and the execution will jump to the next instruction.
	613	+ */
	614	+ ins_offset = pc - (long)&ex->insn;
	615	+ if (WARN_ON_ONCE(ins_offset >= 0 \|\| ins_offset < INT_MIN))
601	616	return -ERANGE;
602		− ex->insn = offset;
603	617
604	618	/*
605	619	* Since the extable follows the program, the fixup offset is always
@@ -608,12 +622,25 @@ static int add_exception_handler(const struct bpf_insn *insn, · 2 unmodified lines
608	622	* bits. We don't need to worry about buildtime or runtime sort
609	623	* modifying the upper bits because the table is already sorted, and
610	624	* isn't part of the main exception table.
	625	+ *
	626	+ * The fixup_offset is set to the next instruction from the instruction
	627	+ * that may fault. The execution will jump to this after handling the
	628	+ * fault.
611	629	*/
612		− offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
613		− if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, offset))
	630	+ fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16));
	631	+ if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
614	632	return -ERANGE;
615	633
616		− ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, offset) \|
	634	+ /*
	635	+ * The offsets above have been calculated using the RO buffer but we
	636	+ * need to use the R/W buffer for writes.
	637	+ * switch ex to rw buffer for writing.
	638	+ */
	639	+ ex = (void )ctx->insns + ((void )ex - (void *)ctx->ro_insns);
	640	+
	641	+ ex->insn = ins_offset;
	642	+
	643	+ ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) \|
617	644	FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
618	645	ex->type = EX_TYPE_BPF;
619	646
@@ -1007,6 +1034,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, · 387 unmodified lines
1007	1034
1008	1035	ctx.ninsns = 0;
1009	1036	ctx.insns = NULL;
	1037	+ ctx.ro_insns = NULL;
1010	1038	ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
1011	1039	if (ret < 0)
1012	1040	return ret;
@@ -1015,7 +1043,15 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image im, void image, · 2 unmodified lines
1015	1043	return -EFBIG;
1016	1044
1017	1045	ctx.ninsns = 0;
	1046	+ /*
	1047	+ * The bpf_int_jit_compile() uses a RW buffer (ctx.insns) to write the
	1048	+ * JITed instructions and later copies it to a RX region (ctx.ro_insns).
	1049	+ * It also uses ctx.ro_insns to calculate offsets for jumps etc. As the
	1050	+ * trampoline image uses the same memory area for writing and execution,
	1051	+ * both ctx.insns and ctx.ro_insns can be set to image.
	1052	+ */
1018	1053	ctx.insns = image;
	1054	+ ctx.ro_insns = image;
1019	1055	ret = __arch_prepare_bpf_trampoline(im, m, tlinks, func_addr, flags, &ctx);
1020	1056	if (ret < 0)
1021	1057	return ret;

arch/riscv/net/bpf_jit_core.c+90−16

@@ -8,6 +8,8 @@ · 7 unmodified lines
8	8
9	9	#include <linux/bpf.h>
10	10	#include <linux/filter.h>
	11	+#include <linux/memory.h>
	12	+#include <asm/patch.h>
11	13	#include "bpf_jit.h"
12	14
13	15	/* Number of iterations to try until offsets converge. */
@@ -117,16 +119,24 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 103 unmodified lines
117	119	sizeof(struct exception_table_entry);
118	120	prog_size = sizeof(ctx->insns) ctx->ninsns;
119	121
120		− jit_data->header =
121		− bpf_jit_binary_alloc(prog_size + extable_size,
122		− &jit_data->image,
123		− sizeof(u32),
124		− bpf_fill_ill_insns);
125		− if (!jit_data->header) {
	122	+ jit_data->ro_header =
	123	+ bpf_jit_binary_pack_alloc(prog_size + extable_size,
	124	+ &jit_data->ro_image, sizeof(u32),
	125	+ &jit_data->header, &jit_data->image,
	126	+ bpf_fill_ill_insns);
	127	+ if (!jit_data->ro_header) {
126	128	prog = orig_prog;
127	129	goto out_offset;
128	130	}
129	131
	132	+ /*
	133	+ * Use the image(RW) for writing the JITed instructions. But also save
	134	+ * the ro_image(RX) for calculating the offsets in the image. The RW
	135	+ * image will be later copied to the RX image from where the program
	136	+ * will run. The bpf_jit_binary_pack_finalize() will do this copy in the
	137	+ * final step.
	138	+ */
	139	+ ctx->ro_insns = (u16 *)jit_data->ro_image;
130	140	ctx->insns = (u16 *)jit_data->image;
131	141	/*
132	142	* Now, when the image is allocated, the image can
@@ -138,14 +148,12 @@ struct bpf_prog bpf_int_jit_compile(struct bpf_prog prog) · 5 unmodified lines
138	148
139	149	if (i == NR_JIT_ITERATIONS) {
140	150	pr_err("bpf-jit: image did not converge in <%d passes!\n", i);
141		− if (jit_data->header)
142		− bpf_jit_binary_free(jit_data->header);
143	151	prog = orig_prog;
144		− goto out_offset;
	152	+ goto out_free_hdr;
145	153	}
146	154
147	155	if (extable_size)
148		− prog->aux->extable = (void *)ctx->insns + prog_size;
	156	+ prog->aux->extable = (void *)ctx->ro_insns + prog_size;
149	157
150	158	skip_init_ctx:
151	159	pass++;
@@ -154,23 +162,33 @@ skip_init_ctx: · 2 unmodified lines
154	162
155	163	bpf_jit_build_prologue(ctx);
156	164	if (build_body(ctx, extra_pass, NULL)) {
157		− bpf_jit_binary_free(jit_data->header);
158	165	prog = orig_prog;
159		− goto out_offset;
	166	+ goto out_free_hdr;
160	167	}
161	168	bpf_jit_build_epilogue(ctx);
162	169
163	170	if (bpf_jit_enable > 1)
164	171	bpf_jit_dump(prog->len, prog_size, pass, ctx->insns);
165	172
166		− prog->bpf_func = (void *)ctx->insns;
	173	+ prog->bpf_func = (void *)ctx->ro_insns;
167	174	prog->jited = 1;
168	175	prog->jited_len = prog_size;
169	176
170		− bpf_flush_icache(jit_data->header, ctx->insns + ctx->ninsns);
171		−
172	177	if (!prog->is_func \|\| extra_pass) {
173		− bpf_jit_binary_lock_ro(jit_data->header);
	178	+ if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header,
	179	+ jit_data->header))) {
	180	+ /* ro_header has been freed */
	181	+ jit_data->ro_header = NULL;
	182	+ prog = orig_prog;
	183	+ goto out_offset;
	184	+ }
	185	+ /*
	186	+ * The instructions have now been copied to the ROX region from
	187	+ * where they will execute.
	188	+ * Write any modified data cache blocks out to memory and
	189	+ * invalidate the corresponding blocks in the instruction cache.
	190	+ */
	191	+ bpf_flush_icache(jit_data->ro_header, ctx->ro_insns + ctx->ninsns);
174	192	for (i = 0; i < prog->len; i++)
175	193	ctx->offset[i] = ninsns_rvoff(ctx->offset[i]);
176	194	bpf_prog_fill_jited_linfo(prog, ctx->offset);
@@ -185,6 +203,14 @@ out: · 8 unmodified lines
185	203	bpf_jit_prog_release_other(prog, prog == orig_prog ?
186	204	tmp : orig_prog);
187	205	return prog;
	206	+
	207	+out_free_hdr:
	208	+ if (jit_data->header) {
	209	+ bpf_arch_text_copy(&jit_data->ro_header->size, &jit_data->header->size,
	210	+ sizeof(jit_data->header->size));
	211	+ bpf_jit_binary_pack_free(jit_data->ro_header, jit_data->header);
	212	+ }
	213	+ goto out_offset;
188	214	}
189	215
190	216	u64 bpf_jit_alloc_exec_limit(void)
@@ -204,3 +230,51 @@ void bpf_jit_free_exec(void *addr) · 13 unmodified lines
204	230	{
205	231	return vfree(addr);
206	232	}
	233	+
	234	+void bpf_arch_text_copy(void dst, void *src, size_t len)
	235	+{
	236	+ int ret;
	237	+
	238	+ mutex_lock(&text_mutex);
	239	+ ret = patch_text_nosync(dst, src, len);
	240	+ mutex_unlock(&text_mutex);
	241	+
	242	+ if (ret)
	243	+ return ERR_PTR(-EINVAL);
	244	+
	245	+ return dst;
	246	+}
	247	+
	248	+int bpf_arch_text_invalidate(void *dst, size_t len)
	249	+{
	250	+ int ret;
	251	+
	252	+ mutex_lock(&text_mutex);
	253	+ ret = patch_text_set_nosync(dst, 0, len);
	254	+ mutex_unlock(&text_mutex);
	255	+
	256	+ return ret;
	257	+}
	258	+
	259	+void bpf_jit_free(struct bpf_prog *prog)
	260	+{
	261	+ if (prog->jited) {
	262	+ struct rv_jit_data *jit_data = prog->aux->jit_data;
	263	+ struct bpf_binary_header *hdr;
	264	+
	265	+ /*
	266	+ * If we fail the final pass of JIT (from jit_subprogs),
	267	+ * the program may not be finalized yet. Call finalize here
	268	+ * before freeing it.
	269	+ */
	270	+ if (jit_data) {
	271	+ bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, jit_data->header);
	272	+ kfree(jit_data);
	273	+ }
	274	+ hdr = bpf_jit_binary_pack_hdr(prog);
	275	+ bpf_jit_binary_pack_free(hdr, NULL);
	276	+ WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(prog));
	277	+ }
	278	+
	279	+ bpf_prog_unlock_free(prog);
	280	+}

kernel/bpf/core.c+4−4

@@ -870,7 +870,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins · 869 unmodified lines
870	870	GFP_KERNEL);
871	871	if (!pack)
872	872	return NULL;
873		− pack->ptr = module_alloc(BPF_PROG_PACK_SIZE);
	873	+ pack->ptr = bpf_jit_alloc_exec(BPF_PROG_PACK_SIZE);
874	874	if (!pack->ptr) {
875	875	kfree(pack);
876	876	return NULL;
@@ -894,7 +894,7 @@ void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns) · 17 unmodified lines
894	894	mutex_lock(&pack_mutex);
895	895	if (size > BPF_PROG_PACK_SIZE) {
896	896	size = round_up(size, PAGE_SIZE);
897		− ptr = module_alloc(size);
	897	+ ptr = bpf_jit_alloc_exec(size);
898	898	if (ptr) {
899	899	bpf_fill_ill_insns(ptr, size);
900	900	set_vm_flush_reset_perms(ptr);
@@ -932,7 +932,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 31 unmodified lines
932	932
933	933	mutex_lock(&pack_mutex);
934	934	if (hdr->size > BPF_PROG_PACK_SIZE) {
935		− module_memfree(hdr);
	935	+ bpf_jit_free_exec(hdr);
936	936	goto out;
937	937	}
938	938
@@ -956,7 +956,7 @@ void bpf_prog_pack_free(struct bpf_binary_header *hdr) · 17 unmodified lines
956	956	if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
957	957	BPF_PROG_CHUNK_COUNT, 0) == 0) {
958	958	list_del(&pack->list);
959		− module_memfree(pack->ptr);
	959	+ bpf_jit_free_exec(pack->ptr);
960	960	kfree(pack);
961	961	}
962	962	out:

Merge patch series "bpf, riscv: use BPF prog pack allocator in BPF JIT"

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code