History

bpf: enable program stats

JITed BPF programs are indistinguishable from kernel functions, but unlike kernel code BPF code can be changed often. Typical approach of "perf record" + "perf report" profiling and tuning of kernel code works just as well for BPF programs, but kernel code doesn't need to be monitored whereas BPF programs do. Users load and run large amount of BPF programs. These BPF stats allow tools monitor the usage of BPF on the server. The monitoring tools will turn sysctl kernel.bpf_stats_enabled on and off for few seconds to sample average cost of the programs. Aggregated data over hours and days will provide an insight into cost of BPF and alarms can trigger in case given program suddenly gets more expensive. The cost of two sched_clock() per program invocation adds ~20 nsec. Fast BPF progs (like selftests/bpf/progs/test_pkt_access.c) will slow down from ~10 nsec to ~30 nsec. static_key minimizes the cost of the stats collection. There is no measurable difference before/after this patch with kernel.bpf_stats_enabled=0

Alexei Starovoitov committed 7 years ago

Commit 492ecee1 parent 143bdc26 files · +129 −6Browse files

Showing the changes to include/linux (2 of 6 files). Show all

Unified · Split

2 files
  1. include/linux/bpf.h +9 −0
  2. include/linux/filter.h +19 −1
include/linux/bpf.h+9−0
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <linux/rbtree_latch.h>
1717 #include <linux/numa.h>
1818 #include <linux/wait.h>
19+#include <linux/u64_stats_sync.h>
1920
2021 struct bpf_verifier_env;
2122 struct perf_event;
@@ -340,6 +341,12 @@ enum bpf_cgroup_storage_type { · 318 unmodified lines
340341
341342 #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
342343
344+struct bpf_prog_stats {
345+ u64 cnt;
346+ u64 nsecs;
347+ struct u64_stats_sync syncp;
348+};
349+
343350 struct bpf_prog_aux {
344351 atomic_t refcnt;
345352 u32 used_map_cnt;
@@ -389,6 +396,7 @@ struct bpf_prog_aux { · 43 unmodified lines
389396 * main prog always has linfo_idx == 0
390397 */
391398 u32 linfo_idx;
399+ struct bpf_prog_stats __percpu *stats;
392400 union {
393401 struct work_struct work;
394402 struct rcu_head rcu;
@@ -559,6 +567,7 @@ void bpf_map_area_free(void *base); · 164 unmodified lines
559567 void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
560568
561569 extern int sysctl_unprivileged_bpf_disabled;
570+extern int sysctl_bpf_stats_enabled;
562571
563572 int bpf_map_new_fd(struct bpf_map *map, int flags);
564573 int bpf_prog_new_fd(struct bpf_prog *prog);
include/linux/filter.h+19−1
@@ -533,7 +533,24 @@ struct sk_filter { · 532 unmodified lines
533533 struct bpf_prog *prog;
534534 };
535535
536−#define BPF_PROG_RUN(filter, ctx) ({ cant_sleep(); (*(filter)->bpf_func)(ctx, (filter)->insnsi); })
536+DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
537+
538+#define BPF_PROG_RUN(prog, ctx) ({ \
539+ u32 ret; \
540+ cant_sleep(); \
541+ if (static_branch_unlikely(&bpf_stats_enabled_key)) { \
542+ struct bpf_prog_stats *stats; \
543+ u64 start = sched_clock(); \
544+ ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi); \
545+ stats = this_cpu_ptr(prog->aux->stats); \
546+ u64_stats_update_begin(&stats->syncp); \
547+ stats->cnt++; \
548+ stats->nsecs += sched_clock() - start; \
549+ u64_stats_update_end(&stats->syncp); \
550+ } else { \
551+ ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi); \
552+ } \
553+ ret; })
537554
538555 #define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
539556
@@ -764,6 +781,7 @@ void bpf_prog_free_jited_linfo(struct bpf_prog *prog); · 224 unmodified lines
764781 void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
765782
766783 struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
784+struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags);
767785 struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
768786 gfp_t gfp_extra_flags);
769787 void __bpf_prog_free(struct bpf_prog *fp);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t