History

bpf: enable program stats

JITed BPF programs are indistinguishable from kernel functions, but unlike kernel code BPF code can be changed often. Typical approach of "perf record" + "perf report" profiling and tuning of kernel code works just as well for BPF programs, but kernel code doesn't need to be monitored whereas BPF programs do. Users load and run large amount of BPF programs. These BPF stats allow tools monitor the usage of BPF on the server. The monitoring tools will turn sysctl kernel.bpf_stats_enabled on and off for few seconds to sample average cost of the programs. Aggregated data over hours and days will provide an insight into cost of BPF and alarms can trigger in case given program suddenly gets more expensive. The cost of two sched_clock() per program invocation adds ~20 nsec. Fast BPF progs (like selftests/bpf/progs/test_pkt_access.c) will slow down from ~10 nsec to ~30 nsec. static_key minimizes the cost of the stats collection. There is no measurable difference before/after this patch with kernel.bpf_stats_enabled=0

Alexei Starovoitov committed 7 years ago

Commit 492ecee1 parent 143bdc26 files · +129 −6Browse files

Showing the changes to include/linux (2 of 6 files). Show all

Unified · Split

2 files
  1. include/linux/bpf.h +9 −0
  2. include/linux/filter.h +19 −1
include/linux/bpf.h+9−0
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include <linux/rbtree_latch.h>16#include <linux/rbtree_latch.h>
17#include <linux/numa.h>17#include <linux/numa.h>
18#include <linux/wait.h>18#include <linux/wait.h>
19#include <linux/u64_stats_sync.h>
19 20
20struct bpf_verifier_env;21struct bpf_verifier_env;
21struct perf_event;22struct perf_event;
@@ -340,6 +341,12 @@ enum bpf_cgroup_storage_type { · 318 unmodified lines
340 341
341#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX342#define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX
342 343
344struct bpf_prog_stats {
345 u64 cnt;
346 u64 nsecs;
347 struct u64_stats_sync syncp;
348};
349
343struct bpf_prog_aux {350struct bpf_prog_aux {
344 atomic_t refcnt;351 atomic_t refcnt;
345 u32 used_map_cnt;352 u32 used_map_cnt;
@@ -389,6 +396,7 @@ struct bpf_prog_aux { · 43 unmodified lines
389 * main prog always has linfo_idx == 0396 * main prog always has linfo_idx == 0
390 */397 */
391 u32 linfo_idx;398 u32 linfo_idx;
399 struct bpf_prog_stats __percpu *stats;
392 union {400 union {
393 struct work_struct work;401 struct work_struct work;
394 struct rcu_head rcu;402 struct rcu_head rcu;
@@ -559,6 +567,7 @@ void bpf_map_area_free(void *base); · 164 unmodified lines
559void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);567void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
560 568
561extern int sysctl_unprivileged_bpf_disabled;569extern int sysctl_unprivileged_bpf_disabled;
570extern int sysctl_bpf_stats_enabled;
562 571
563int bpf_map_new_fd(struct bpf_map *map, int flags);572int bpf_map_new_fd(struct bpf_map *map, int flags);
564int bpf_prog_new_fd(struct bpf_prog *prog);573int bpf_prog_new_fd(struct bpf_prog *prog);
include/linux/filter.h+19−1
@@ -533,7 +533,24 @@ struct sk_filter { · 532 unmodified lines
533 struct bpf_prog *prog;533 struct bpf_prog *prog;
534};534};
535 535
536#define BPF_PROG_RUN(filter, ctx) ({ cant_sleep(); (*(filter)->bpf_func)(ctx, (filter)->insnsi); })536DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
537
538#define BPF_PROG_RUN(prog, ctx) ({ \
539 u32 ret; \
540 cant_sleep(); \
541 if (static_branch_unlikely(&bpf_stats_enabled_key)) { \
542 struct bpf_prog_stats *stats; \
543 u64 start = sched_clock(); \
544 ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi); \
545 stats = this_cpu_ptr(prog->aux->stats); \
546 u64_stats_update_begin(&stats->syncp); \
547 stats->cnt++; \
548 stats->nsecs += sched_clock() - start; \
549 u64_stats_update_end(&stats->syncp); \
550 } else { \
551 ret = (*(prog)->bpf_func)(ctx, (prog)->insnsi); \
552 } \
553 ret; })
537 554
538#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN555#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
539 556
@@ -764,6 +781,7 @@ void bpf_prog_free_jited_linfo(struct bpf_prog *prog); · 224 unmodified lines
764void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);781void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog);
765 782
766struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);783struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
784struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags);
767struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,785struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
768 gfp_t gfp_extra_flags);786 gfp_t gfp_extra_flags);
769void __bpf_prog_free(struct bpf_prog *fp);787void __bpf_prog_free(struct bpf_prog *fp);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t