bpf: Optimize program stats

Move bpf_prog_stats from prog->aux into prog to avoid one extra load in critical path of program execution.

Alexei Starovoitov committed 5 years ago

Commit 700d4791 parent 6df8fb86 files · +18 −18Browse files

Unified · Split

6 files

include/linux/bpf.h +0 −8
include/linux/filter.h +11 −3
kernel/bpf/core.c +4 −4
kernel/bpf/syscall.c +1 −1
kernel/bpf/trampoline.c +1 −1
kernel/bpf/verifier.c +1 −1

@@ -14,7 +14,6 @@ · 13 unmodified lines
14	14	#include <linux/numa.h>
15	15	#include <linux/mm_types.h>
16	16	#include <linux/wait.h>
17		−#include <linux/u64_stats_sync.h>
18	17	#include <linux/refcount.h>
19	18	#include <linux/mutex.h>
20	19	#include <linux/module.h>
@@ -507,12 +506,6 @@ enum bpf_cgroup_storage_type { · 486 unmodified lines
507	506	*/
508	507	#define MAX_BPF_FUNC_ARGS 12
509	508
510		−struct bpf_prog_stats {
511		− u64 cnt;
512		− u64 nsecs;
513		− struct u64_stats_sync syncp;
514		−} __aligned(2 * sizeof(u64));
515		−
516	509	struct btf_func_model {
517	510	u8 ret_size;
518	511	u8 nr_args;
@@ -845,7 +838,6 @@ struct bpf_prog_aux { · 326 unmodified lines
845	838	u32 linfo_idx;
846	839	u32 num_exentries;
847	840	struct exception_table_entry *extable;
848		− struct bpf_prog_stats __percpu *stats;
849	841	union {
850	842	struct work_struct work;
851	843	struct rcu_head rcu;

include/linux/filter.h+11−3

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	#include <linux/vmalloc.h>
23	23	#include <linux/sockptr.h>
24	24	#include <crypto/sha1.h>
	25	+#include <linux/u64_stats_sync.h>
25	26
26	27	#include <net/sch_generic.h>
27	28
@@ -539,6 +540,12 @@ struct bpf_binary_header { · 511 unmodified lines
539	540	u8 image[] __aligned(BPF_IMAGE_ALIGNMENT);
540	541	};
541	542
	543	+struct bpf_prog_stats {
	544	+ u64 cnt;
	545	+ u64 nsecs;
	546	+ struct u64_stats_sync syncp;
	547	+} __aligned(2 * sizeof(u64));
	548	+
542	549	struct bpf_prog {
543	550	u16 pages; /* Number of allocated pages */
544	551	u16 jited:1, /* Is our filter JIT'ed? */
@@ -557,10 +564,11 @@ struct bpf_prog { · 12 unmodified lines
557	564	u32 len; /* Number of filter blocks */
558	565	u32 jited_len; /* Size of jited insns in bytes */
559	566	u8 tag[BPF_TAG_SIZE];
560		− struct bpf_prog_aux aux; / Auxiliary fields */
561		− struct sock_fprog_kern orig_prog; / Original BPF program */
	567	+ struct bpf_prog_stats __percpu *stats;
562	568	unsigned int (bpf_func)(const void ctx,
563	569	const struct bpf_insn *insn);
	570	+ struct bpf_prog_aux aux; / Auxiliary fields */
	571	+ struct sock_fprog_kern orig_prog; / Original BPF program */
564	572	/* Instructions for interpreter */
565	573	struct sock_filter insns[0];
566	574	struct bpf_insn insnsi[];
@@ -581,7 +589,7 @@ DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key); · 14 unmodified lines
581	589	struct bpf_prog_stats *__stats; \
582	590	u64 __start = sched_clock(); \
583	591	__ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \
584		− __stats = this_cpu_ptr(prog->aux->stats); \
	592	+ __stats = this_cpu_ptr(prog->stats); \
585	593	u64_stats_update_begin(&__stats->syncp); \
586	594	__stats->cnt++; \
587	595	__stats->nsecs += sched_clock() - __start; \

kernel/bpf/core.c+4−4

@@ -114,8 +114,8 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 113 unmodified lines
114	114	if (!prog)
115	115	return NULL;
116	116
117		− prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
118		− if (!prog->aux->stats) {
	117	+ prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
	118	+ if (!prog->stats) {
119	119	kfree(prog->aux);
120	120	vfree(prog);
121	121	return NULL;
@@ -124,7 +124,7 @@ struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) · 2 unmodified lines
124	124	for_each_possible_cpu(cpu) {
125	125	struct bpf_prog_stats *pstats;
126	126
127		− pstats = per_cpu_ptr(prog->aux->stats, cpu);
	127	+ pstats = per_cpu_ptr(prog->stats, cpu);
128	128	u64_stats_init(&pstats->syncp);
129	129	}
130	130	return prog;
@@ -249,10 +249,10 @@ void __bpf_prog_free(struct bpf_prog *fp) · 118 unmodified lines
249	249	if (fp->aux) {
250	250	mutex_destroy(&fp->aux->used_maps_mutex);
251	251	mutex_destroy(&fp->aux->dst_mutex);
252		− free_percpu(fp->aux->stats);
253	252	kfree(fp->aux->poke_tab);
254	253	kfree(fp->aux);
255	254	}
	255	+ free_percpu(fp->stats);
256	256	vfree(fp);
257	257	}
258	258

kernel/bpf/syscall.c+1−1

@@ -1739,7 +1739,7 @@ static void bpf_prog_get_stats(const struct bpf_prog *prog, · 1738 unmodified lines
1739	1739	unsigned int start;
1740	1740	u64 tnsecs, tcnt;
1741	1741
1742		− st = per_cpu_ptr(prog->aux->stats, cpu);
	1742	+ st = per_cpu_ptr(prog->stats, cpu);
1743	1743	do {
1744	1744	start = u64_stats_fetch_begin_irq(&st->syncp);
1745	1745	tnsecs = st->nsecs;

kernel/bpf/trampoline.c+1−1

@@ -412,7 +412,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 411 unmodified lines
412	412	* Hence check that 'start' is not zero.
413	413	*/
414	414	start) {
415		− stats = this_cpu_ptr(prog->aux->stats);
	415	+ stats = this_cpu_ptr(prog->stats);
416	416	u64_stats_update_begin(&stats->syncp);
417	417	stats->cnt++;
418	418	stats->nsecs += sched_clock() - start;

kernel/bpf/verifier.c+1−1

@@ -11253,7 +11253,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 11252 unmodified lines
11253	11253	/* BPF_PROG_RUN doesn't call subprogs directly,
11254	11254	* hence main prog stats include the runtime of subprogs.
11255	11255	* subprogs don't have IDs and not reachable via prog_get_next_id
11256		− * func[i]->aux->stats will never be accessed and stays NULL
	11256	+ * func[i]->stats will never be accessed and stays NULL
11257	11257	*/
11258	11258	func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
11259	11259	if (!func[i])

bpf: Optimize program stats

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code