bpf: Optimize program stats

Move bpf_prog_stats from prog->aux into prog to avoid one extra load in critical path of program execution.

Alexei Starovoitov committed 5 years ago

Commit 700d4791 parent 6df8fb86 files · +18 −18Browse files

Unified · Split

6 files

 #include <linux/numa.h>
 #include <linux/mm_types.h>
 #include <linux/wait.h>
-#include <linux/u64_stats_sync.h>
 #include <linux/refcount.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
  */
 #define MAX_BPF_FUNC_ARGS 12
-struct bpf_prog_stats {
-	u64 cnt;
-	u64 nsecs;
-	struct u64_stats_sync syncp;
-} __aligned(2 * sizeof(u64));
 struct btf_func_model {
 	u8 ret_size;
 	u8 nr_args;
 	u32 linfo_idx;
 	u32 num_exentries;
 	struct exception_table_entry *extable;
-	struct bpf_prog_stats __percpu *stats;
 	union {
 		struct work_struct work;
 		struct rcu_head	rcu;

 #include <linux/vmalloc.h>
 #include <linux/sockptr.h>
 #include <crypto/sha1.h>
+#include <linux/u64_stats_sync.h>
 #include <net/sch_generic.h>
 	u8 image[] __aligned(BPF_IMAGE_ALIGNMENT);
 };
+struct bpf_prog_stats {
+	u64 cnt;
+	u64 nsecs;
+	struct u64_stats_sync syncp;
+} __aligned(2 * sizeof(u64));
 struct bpf_prog {
 	u16			pages;		/* Number of allocated pages */
 	u16			jited:1,	/* Is our filter JIT'ed? */
 	u32			len;		/* Number of filter blocks */
 	u32			jited_len;	/* Size of jited insns in bytes */
 	u8			tag[BPF_TAG_SIZE];
-	struct bpf_prog_aux	*aux;		/* Auxiliary fields */
+	struct bpf_prog_stats __percpu *stats;
-	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */
 	unsigned int		(*bpf_func)(const void *ctx,
 					    const struct bpf_insn *insn);
+	struct bpf_prog_aux	*aux;		/* Auxiliary fields */
+	struct sock_fprog_kern	*orig_prog;	/* Original BPF program */
 	/* Instructions for interpreter */
 	struct sock_filter	insns[0];
 	struct bpf_insn		insnsi[];
 		struct bpf_prog_stats *__stats;				\
 		u64 __start = sched_clock();				\
 		__ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func);	\
-		__stats = this_cpu_ptr(prog->aux->stats);		\
+		__stats = this_cpu_ptr(prog->stats);			\
 		u64_stats_update_begin(&__stats->syncp);		\
 		__stats->cnt++;						\
 		__stats->nsecs += sched_clock() - __start;		\

 	if (!prog)
 		return NULL;
-	prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
+	prog->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags);
-	if (!prog->aux->stats) {
+	if (!prog->stats) {
 		kfree(prog->aux);
 		vfree(prog);
 		return NULL;
 	for_each_possible_cpu(cpu) {
 		struct bpf_prog_stats *pstats;
-		pstats = per_cpu_ptr(prog->aux->stats, cpu);
+		pstats = per_cpu_ptr(prog->stats, cpu);
 		u64_stats_init(&pstats->syncp);
 	}
 	return prog;
 	if (fp->aux) {
 		mutex_destroy(&fp->aux->used_maps_mutex);
 		mutex_destroy(&fp->aux->dst_mutex);
-		free_percpu(fp->aux->stats);
 		kfree(fp->aux->poke_tab);
 		kfree(fp->aux);
 	}
+	free_percpu(fp->stats);
 	vfree(fp);
 }

@@ -1739,7 +1739,7 @@ static void bpf_prog_get_stats(const struct bpf_prog *prog, · 1738 unmodified lines
1739	unsigned int start;	1739	unsigned int start;
1740	u64 tnsecs, tcnt;	1740	u64 tnsecs, tcnt;
1741		1741
1742	st = per_cpu_ptr(prog->aux->stats, cpu);	1742	st = per_cpu_ptr(prog->stats, cpu);
1743	do {	1743	do {
1744	start = u64_stats_fetch_begin_irq(&st->syncp);	1744	start = u64_stats_fetch_begin_irq(&st->syncp);
1745	tnsecs = st->nsecs;	1745	tnsecs = st->nsecs;

@@ -412,7 +412,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 411 unmodified lines
412	* Hence check that 'start' is not zero.	412	* Hence check that 'start' is not zero.
413	*/	413	*/
414	start) {	414	start) {
415	stats = this_cpu_ptr(prog->aux->stats);	415	stats = this_cpu_ptr(prog->stats);
416	u64_stats_update_begin(&stats->syncp);	416	u64_stats_update_begin(&stats->syncp);
417	stats->cnt++;	417	stats->cnt++;
418	stats->nsecs += sched_clock() - start;	418	stats->nsecs += sched_clock() - start;

@@ -11253,7 +11253,7 @@ static int jit_subprogs(struct bpf_verifier_env *env) · 11252 unmodified lines
11253	/* BPF_PROG_RUN doesn't call subprogs directly,	11253	/* BPF_PROG_RUN doesn't call subprogs directly,
11254	* hence main prog stats include the runtime of subprogs.	11254	* hence main prog stats include the runtime of subprogs.
11255	* subprogs don't have IDs and not reachable via prog_get_next_id	11255	* subprogs don't have IDs and not reachable via prog_get_next_id
11256	* func[i]->aux->stats will never be accessed and stays NULL	11256	* func[i]->stats will never be accessed and stays NULL
11257	*/	11257	*/
11258	func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);	11258	func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
11259	if (!func[i])	11259	if (!func[i])

Keyboard shortcuts