// SPDX-License-Identifier: GPL-2.0-only#define _GNU_SOURCE #include <arpa/inet.h>#include <bpf/bpf.h>#include <bpf/libbpf.h>#include <errno.h>#include <fcntl.h>#include <getopt.h>#include <linux/ethtool.h>#include <linux/hashtable.h>#include <linux/if_link.h>#include <linux/jhash.h>#include <linux/limits.h>#include <linux/list.h>#include <linux/sockios.h>#include <locale.h>#include <math.h>#include <net/if.h>#include <poll.h>#include <signal.h>#include <stdbool.h>#include <stdio.h>#include <stdlib.h>#include <string.h>#include <sys/ioctl.h>#include <sys/mman.h>#include <sys/signalfd.h>#include <sys/sysinfo.h>#include <sys/timerfd.h>#include <sys/utsname.h>#include <time.h>#include <unistd.h> #include "bpf_util.h"#include "xdp_sample_user.h" #define __sample_print(fmt, cond, ...) \ ({ \ if (cond) \ printf(fmt, ##__VA_ARGS__); \ }) #define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__)#define print_default(fmt, ...) \ __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__)#define __print_err(err, fmt, ...) \ ({ \ __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \ ##__VA_ARGS__); \ sample_err_exp = sample_err_exp ? true : err > 0; \ })#define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__) #define __COLUMN(x) "%'10" x " %-13s"#define FMT_COLUMNf __COLUMN(".0f")#define FMT_COLUMNd __COLUMN("d")#define FMT_COLUMNl __COLUMN("llu")#define RX(rx) rx, "rx/s"#define PPS(pps) pps, "pkt/s"#define DROP(drop) drop, "drop/s"#define ERR(err) err, "error/s"#define HITS(hits) hits, "hit/s"#define XMIT(xmit) xmit, "xmit/s"#define PASS(pass) pass, "pass/s"#define REDIR(redir) redir, "redir/s"#define NANOSEC_PER_SEC 1000000000 /* 10^9 */ #define XDP_UNKNOWN (XDP_REDIRECT + 1)#define XDP_ACTION_MAX (XDP_UNKNOWN + 1)#define XDP_REDIRECT_ERR_MAX 7 enum map_type { MAP_RX, MAP_REDIRECT_ERR, MAP_CPUMAP_ENQUEUE, MAP_CPUMAP_KTHREAD, MAP_EXCEPTION, MAP_DEVMAP_XMIT, MAP_DEVMAP_XMIT_MULTI, NUM_MAP,}; enum log_level { LL_DEFAULT = 1U << 0, LL_SIMPLE = 1U << 1, LL_DEBUG = 1U << 2,}; struct record { __u64 timestamp; struct datarec total; struct datarec *cpu;}; struct map_entry { struct hlist_node node; __u64 pair; struct record val;}; struct stats_record { struct record rx_cnt; struct record redir_err[XDP_REDIRECT_ERR_MAX]; struct record kthread; struct record exception[XDP_ACTION_MAX]; struct record devmap_xmit; DECLARE_HASHTABLE(xmit_map, 5); struct record enq[];}; struct sample_output { struct { __u64 rx; __u64 redir; __u64 drop; __u64 drop_xmit; __u64 err; __u64 xmit; } totals; struct { union { __u64 pps; __u64 num; }; __u64 drop; __u64 err; } rx_cnt; struct { __u64 suc; __u64 err; } redir_cnt; struct { __u64 hits; } except_cnt; struct { __u64 pps; __u64 drop; __u64 err; double bavg; } xmit_cnt;}; struct xdp_desc { int ifindex; __u32 prog_id; int flags;} sample_xdp_progs[32]; struct datarec *sample_mmap[NUM_MAP];struct bpf_map *sample_map[NUM_MAP];size_t sample_map_count[NUM_MAP];enum log_level sample_log_level;struct sample_output sample_out;unsigned long sample_interval;bool sample_err_exp;int sample_xdp_cnt;int sample_n_cpus;int sample_sig_fd;int sample_mask; static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = { /* Key=1 keeps unknown errors */ "Success", "Unknown", "EINVAL", "ENETDOWN", "EMSGSIZE", "EOPNOTSUPP", "ENOSPC",}; /* Keyed from Unknown */static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = { "Unknown error", "Invalid redirection", "Device being redirected to is down", "Packet length too large for device", "Operation not supported", "No space in ptr_ring of cpumap kthread",}; static const char *xdp_action_names[XDP_ACTION_MAX] = { [XDP_ABORTED] = "XDP_ABORTED", [XDP_DROP] = "XDP_DROP", [XDP_PASS] = "XDP_PASS", [XDP_TX] = "XDP_TX", [XDP_REDIRECT] = "XDP_REDIRECT", [XDP_UNKNOWN] = "XDP_UNKNOWN",}; static __u64 gettime(void){ struct timespec t; int res; res = clock_gettime(CLOCK_MONOTONIC, &t); if (res < 0) { fprintf(stderr, "Error with gettimeofday! (%i)\n", res); return UINT64_MAX; } return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;} static const char *action2str(int action){ if (action < XDP_ACTION_MAX) return xdp_action_names[action]; return NULL;} static void sample_print_help(int mask){ printf("Output format description\n\n" "By default, redirect success statistics are disabled, use -s to enable.\n" "The terse output mode is default, verbose mode can be activated using -v\n" "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n" "Terse mode displays at most the following fields:\n" " rx/s Number of packets received per second\n" " redir/s Number of packets successfully redirected per second\n" " err,drop/s Aggregated count of errors per second (including dropped packets)\n" " xmit/s Number of packets transmitted on the output device per second\n\n" "Output description for verbose mode:\n" " FIELD DESCRIPTION\n"); if (mask & SAMPLE_RX_CNT) { printf(" receive\t\tDisplays the number of packets received & errors encountered\n" " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n" " \t\t\tand drop statistics will be expanded inline in terse mode.\n" " \t\t\t\tpkt/s - Packets received per second\n" " \t\t\t\tdrop/s - Packets dropped per second\n" " \t\t\t\terror/s - Errors encountered per second\n\n"); } if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { printf(" redirect\t\tDisplays the number of packets successfully redirected\n" " \t\t\tErrors encountered are expanded under redirect_err field\n" " \t\t\tNote that passing -s to enable it has a per packet overhead\n" " \t\t\t\tredir/s - Packets redirected successfully per second\n\n" " redirect_err\t\tDisplays the number of packets that failed redirection\n" " \t\t\tThe errno is expanded under this field with per CPU count\n" " \t\t\tThe recognized errors are:\n"); for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++) printf("\t\t\t %s: %s\n", xdp_redirect_err_names[i], xdp_redirect_err_help[i - 1]); printf(" \n\t\t\t\terror/s - Packets that failed redirection per second\n\n"); } if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) { printf(" enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n" " \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n" " \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n" " \t\t\tpackets to.\n" " \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n" " \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n" " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); } if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) { printf(" kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n" " \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n" " \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n" " \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n" " \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n" " \t\t\t\tdrop/s - Packets dropped per second in kthread\n" " \t\t\t\tsched - Number of times kthread called schedule()\n\n" " \t\t\txdp_stats (also expands to per-CPU counts)\n" " \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n" " \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n" " \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n"); } if (mask & SAMPLE_EXCEPTION_CNT) { printf(" xdp_exception\t\tDisplays xdp_exception tracepoint events\n" " \t\t\tThis can occur due to internal driver errors, unrecognized\n" " \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n" " \t\t\tEach action is expanded below this field with its count\n" " \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n"); } if (mask & SAMPLE_DEVMAP_XMIT_CNT) { printf(" devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n" " \t\t\tThis tracepoint is invoked for successful transmissions on output\n" " \t\t\tdevice but these statistics are not available for generic XDP mode,\n" " \t\t\thence they will be omitted from the output when using SKB mode\n" " \t\t\t\txmit/s - Number of packets that were transmitted per second\n" " \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n" " \t\t\t\tdrv_err/s - Number of internal driver errors per second\n" " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); }} void sample_usage(char *argv[], const struct option *long_options, const char *doc, int mask, bool error){ int i; if (!error) sample_print_help(mask); printf("\n%s\nOption for %s:\n", doc, argv[0]); for (i = 0; long_options[i].name != 0; i++) { printf(" --%-15s", long_options[i].name); if (long_options[i].flag != NULL) printf(" flag (internal value: %d)", *long_options[i].flag); else printf("\t short-option: -%c", long_options[i].val); printf("\n"); } printf("\n");} static struct datarec *alloc_record_per_cpu(void){ unsigned int nr_cpus = libbpf_num_possible_cpus(); struct datarec *array; array = calloc(nr_cpus, sizeof(*array)); if (!array) { fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n", nr_cpus); return NULL; } return array;} static int map_entry_init(struct map_entry *e, __u64 pair){ e->pair = pair; INIT_HLIST_NODE(&e->node); e->val.timestamp = gettime(); e->val.cpu = alloc_record_per_cpu(); if (!e->val.cpu) return -ENOMEM; return 0;} static void map_collect_percpu(struct datarec *values, struct record *rec){ /* For percpu maps, userspace gets a value per possible CPU */ unsigned int nr_cpus = libbpf_num_possible_cpus(); __u64 sum_xdp_redirect = 0; __u64 sum_processed = 0; __u64 sum_xdp_pass = 0; __u64 sum_xdp_drop = 0; __u64 sum_dropped = 0; __u64 sum_issue = 0; int i; /* Get time as close as possible to reading map contents */ rec->timestamp = gettime(); /* Record and sum values from each CPU */ for (i = 0; i < nr_cpus; i++) { rec->cpu[i].processed = READ_ONCE(values[i].processed); rec->cpu[i].dropped = READ_ONCE(values[i].dropped); rec->cpu[i].issue = READ_ONCE(values[i].issue); rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass); rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop); rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect); sum_processed += rec->cpu[i].processed; sum_dropped += rec->cpu[i].dropped; sum_issue += rec->cpu[i].issue; sum_xdp_pass += rec->cpu[i].xdp_pass; sum_xdp_drop += rec->cpu[i].xdp_drop; sum_xdp_redirect += rec->cpu[i].xdp_redirect; } rec->total.processed = sum_processed; rec->total.dropped = sum_dropped; rec->total.issue = sum_issue; rec->total.xdp_pass = sum_xdp_pass; rec->total.xdp_drop = sum_xdp_drop; rec->total.xdp_redirect = sum_xdp_redirect;} static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec){ unsigned int nr_cpus = bpf_num_possible_cpus(); __u32 batch, count = 32; struct datarec *values; bool init = false; __u64 *keys; int i, ret; keys = calloc(count, sizeof(__u64)); if (!keys) return -ENOMEM; values = calloc(count * nr_cpus, sizeof(struct datarec)); if (!values) { free(keys); return -ENOMEM; } for (;;) { bool exit = false; ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch, keys, values, &count, NULL); if (ret < 0 && errno != ENOENT) break; if (errno == ENOENT) exit = true; init = true; for (i = 0; i < count; i++) { struct map_entry *e, *x = NULL; __u64 pair = keys[i]; struct datarec *arr; arr = &values[i * nr_cpus]; hash_for_each_possible(rec->xmit_map, e, node, pair) { if (e->pair == pair) { x = e; break; } } if (!x) { x = calloc(1, sizeof(*x)); if (!x) goto cleanup; if (map_entry_init(x, pair) < 0) { free(x); goto cleanup; } hash_add(rec->xmit_map, &x->node, pair); } map_collect_percpu(arr, &x->val); } if (exit) break; count = 32; } free(values); free(keys); return 0;cleanup: free(values); free(keys); return -ENOMEM;} static struct stats_record *alloc_stats_record(void){ struct stats_record *rec; int i; rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record)); if (!rec) { fprintf(stderr, "Failed to allocate memory\n"); return NULL; } if (sample_mask & SAMPLE_RX_CNT) { rec->rx_cnt.cpu = alloc_record_per_cpu(); if (!rec->rx_cnt.cpu) { fprintf(stderr, "Failed to allocate rx_cnt per-CPU array\n"); goto end_rec; } } if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) { rec->redir_err[i].cpu = alloc_record_per_cpu(); if (!rec->redir_err[i].cpu) { fprintf(stderr, "Failed to allocate redir_err per-CPU array for " "\"%s\" case\n", xdp_redirect_err_names[i]); while (i--) free(rec->redir_err[i].cpu); goto end_rx_cnt; } } } if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) { rec->kthread.cpu = alloc_record_per_cpu(); if (!rec->kthread.cpu) { fprintf(stderr, "Failed to allocate kthread per-CPU array\n"); goto end_redir; } } if (sample_mask & SAMPLE_EXCEPTION_CNT) { for (i = 0; i < XDP_ACTION_MAX; i++) { rec->exception[i].cpu = alloc_record_per_cpu(); if (!rec->exception[i].cpu) { fprintf(stderr, "Failed to allocate exception per-CPU array for " "\"%s\" case\n", action2str(i)); while (i--) free(rec->exception[i].cpu); goto end_kthread; }Lines 1–500 of 1673. Load more
// SPDX-License-Identifier: GPL-2.0-only #define _GNU_SOURCE #include <arpa/inet.h> #include <bpf/bpf.h> #include <bpf/libbpf.h> #include <errno.h> #include <fcntl.h> #include <getopt.h> #include <linux/ethtool.h> #include <linux/hashtable.h> #include <linux/if_link.h> #include <linux/jhash.h> #include <linux/limits.h> #include <linux/list.h> #include <linux/sockios.h> #include <locale.h> #include <math.h> #include <net/if.h> #include <poll.h> #include <signal.h> #include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/ioctl.h> #include <sys/mman.h> #include <sys/signalfd.h> #include <sys/sysinfo.h> #include <sys/timerfd.h> #include <sys/utsname.h> #include <time.h> #include <unistd.h> #include "bpf_util.h" #include "xdp_sample_user.h" #define __sample_print(fmt, cond, ...) \ ({ \ if (cond) \ printf(fmt, ##__VA_ARGS__); \ }) #define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__) #define print_default(fmt, ...) \ __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__) #define __print_err(err, fmt, ...) \ ({ \ __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \ ##__VA_ARGS__); \ sample_err_exp = sample_err_exp ? true : err > 0; \ }) #define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__) #define __COLUMN(x) "%'10" x " %-13s" #define FMT_COLUMNf __COLUMN(".0f") #define FMT_COLUMNd __COLUMN("d") #define FMT_COLUMNl __COLUMN("llu") #define RX(rx) rx, "rx/s" #define PPS(pps) pps, "pkt/s" #define DROP(drop) drop, "drop/s" #define ERR(err) err, "error/s" #define HITS(hits) hits, "hit/s" #define XMIT(xmit) xmit, "xmit/s" #define PASS(pass) pass, "pass/s" #define REDIR(redir) redir, "redir/s" #define NANOSEC_PER_SEC 1000000000 /* 10^9 */ #define XDP_UNKNOWN (XDP_REDIRECT + 1) #define XDP_ACTION_MAX (XDP_UNKNOWN + 1) #define XDP_REDIRECT_ERR_MAX 7 enum map_type { MAP_RX, MAP_REDIRECT_ERR, MAP_CPUMAP_ENQUEUE, MAP_CPUMAP_KTHREAD, MAP_EXCEPTION, MAP_DEVMAP_XMIT, MAP_DEVMAP_XMIT_MULTI, NUM_MAP, }; enum log_level { LL_DEFAULT = 1U << 0, LL_SIMPLE = 1U << 1, LL_DEBUG = 1U << 2, }; struct record { __u64 timestamp; struct datarec total; struct datarec *cpu; }; struct map_entry { struct hlist_node node; __u64 pair; struct record val; }; struct stats_record { struct record rx_cnt; struct record redir_err[XDP_REDIRECT_ERR_MAX]; struct record kthread; struct record exception[XDP_ACTION_MAX]; struct record devmap_xmit; DECLARE_HASHTABLE(xmit_map, 5); struct record enq[]; }; struct sample_output { struct { __u64 rx; __u64 redir; __u64 drop; __u64 drop_xmit; __u64 err; __u64 xmit; } totals; struct { union { __u64 pps; __u64 num; }; __u64 drop; __u64 err; } rx_cnt; struct { __u64 suc; __u64 err; } redir_cnt; struct { __u64 hits; } except_cnt; struct { __u64 pps; __u64 drop; __u64 err; double bavg; } xmit_cnt; }; struct xdp_desc { int ifindex; __u32 prog_id; int flags; } sample_xdp_progs[32]; struct datarec *sample_mmap[NUM_MAP]; struct bpf_map *sample_map[NUM_MAP]; size_t sample_map_count[NUM_MAP]; enum log_level sample_log_level; struct sample_output sample_out; unsigned long sample_interval; bool sample_err_exp; int sample_xdp_cnt; int sample_n_cpus; int sample_sig_fd; int sample_mask; static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = { /* Key=1 keeps unknown errors */ "Success", "Unknown", "EINVAL", "ENETDOWN", "EMSGSIZE", "EOPNOTSUPP", "ENOSPC", }; /* Keyed from Unknown */ static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = { "Unknown error", "Invalid redirection", "Device being redirected to is down", "Packet length too large for device", "Operation not supported", "No space in ptr_ring of cpumap kthread", }; static const char *xdp_action_names[XDP_ACTION_MAX] = { [XDP_ABORTED] = "XDP_ABORTED", [XDP_DROP] = "XDP_DROP", [XDP_PASS] = "XDP_PASS", [XDP_TX] = "XDP_TX", [XDP_REDIRECT] = "XDP_REDIRECT", [XDP_UNKNOWN] = "XDP_UNKNOWN", }; static __u64 gettime(void) { struct timespec t; int res; res = clock_gettime(CLOCK_MONOTONIC, &t); if (res < 0) { fprintf(stderr, "Error with gettimeofday! (%i)\n", res); return UINT64_MAX; } return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec; } static const char *action2str(int action) { if (action < XDP_ACTION_MAX) return xdp_action_names[action]; return NULL; } static void sample_print_help(int mask) { printf("Output format description\n\n" "By default, redirect success statistics are disabled, use -s to enable.\n" "The terse output mode is default, verbose mode can be activated using -v\n" "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n" "Terse mode displays at most the following fields:\n" " rx/s Number of packets received per second\n" " redir/s Number of packets successfully redirected per second\n" " err,drop/s Aggregated count of errors per second (including dropped packets)\n" " xmit/s Number of packets transmitted on the output device per second\n\n" "Output description for verbose mode:\n" " FIELD DESCRIPTION\n"); if (mask & SAMPLE_RX_CNT) { printf(" receive\t\tDisplays the number of packets received & errors encountered\n" " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n" " \t\t\tand drop statistics will be expanded inline in terse mode.\n" " \t\t\t\tpkt/s - Packets received per second\n" " \t\t\t\tdrop/s - Packets dropped per second\n" " \t\t\t\terror/s - Errors encountered per second\n\n"); } if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { printf(" redirect\t\tDisplays the number of packets successfully redirected\n" " \t\t\tErrors encountered are expanded under redirect_err field\n" " \t\t\tNote that passing -s to enable it has a per packet overhead\n" " \t\t\t\tredir/s - Packets redirected successfully per second\n\n" " redirect_err\t\tDisplays the number of packets that failed redirection\n" " \t\t\tThe errno is expanded under this field with per CPU count\n" " \t\t\tThe recognized errors are:\n"); for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++) printf("\t\t\t %s: %s\n", xdp_redirect_err_names[i], xdp_redirect_err_help[i - 1]); printf(" \n\t\t\t\terror/s - Packets that failed redirection per second\n\n"); } if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) { printf(" enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n" " \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n" " \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n" " \t\t\tpackets to.\n" " \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n" " \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n" " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); } if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) { printf(" kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n" " \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n" " \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n" " \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n" " \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n" " \t\t\t\tdrop/s - Packets dropped per second in kthread\n" " \t\t\t\tsched - Number of times kthread called schedule()\n\n" " \t\t\txdp_stats (also expands to per-CPU counts)\n" " \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n" " \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n" " \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n"); } if (mask & SAMPLE_EXCEPTION_CNT) { printf(" xdp_exception\t\tDisplays xdp_exception tracepoint events\n" " \t\t\tThis can occur due to internal driver errors, unrecognized\n" " \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n" " \t\t\tEach action is expanded below this field with its count\n" " \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n"); } if (mask & SAMPLE_DEVMAP_XMIT_CNT) { printf(" devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n" " \t\t\tThis tracepoint is invoked for successful transmissions on output\n" " \t\t\tdevice but these statistics are not available for generic XDP mode,\n" " \t\t\thence they will be omitted from the output when using SKB mode\n" " \t\t\t\txmit/s - Number of packets that were transmitted per second\n" " \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n" " \t\t\t\tdrv_err/s - Number of internal driver errors per second\n" " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n"); } } void sample_usage(char *argv[], const struct option *long_options, const char *doc, int mask, bool error) { int i; if (!error) sample_print_help(mask); printf("\n%s\nOption for %s:\n", doc, argv[0]); for (i = 0; long_options[i].name != 0; i++) { printf(" --%-15s", long_options[i].name); if (long_options[i].flag != NULL) printf(" flag (internal value: %d)", *long_options[i].flag); else printf("\t short-option: -%c", long_options[i].val); printf("\n"); } printf("\n"); } static struct datarec *alloc_record_per_cpu(void) { unsigned int nr_cpus = libbpf_num_possible_cpus(); struct datarec *array; array = calloc(nr_cpus, sizeof(*array)); if (!array) { fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n", nr_cpus); return NULL; } return array; } static int map_entry_init(struct map_entry *e, __u64 pair) { e->pair = pair; INIT_HLIST_NODE(&e->node); e->val.timestamp = gettime(); e->val.cpu = alloc_record_per_cpu(); if (!e->val.cpu) return -ENOMEM; return 0; } static void map_collect_percpu(struct datarec *values, struct record *rec) { /* For percpu maps, userspace gets a value per possible CPU */ unsigned int nr_cpus = libbpf_num_possible_cpus(); __u64 sum_xdp_redirect = 0; __u64 sum_processed = 0; __u64 sum_xdp_pass = 0; __u64 sum_xdp_drop = 0; __u64 sum_dropped = 0; __u64 sum_issue = 0; int i; /* Get time as close as possible to reading map contents */ rec->timestamp = gettime(); /* Record and sum values from each CPU */ for (i = 0; i < nr_cpus; i++) { rec->cpu[i].processed = READ_ONCE(values[i].processed); rec->cpu[i].dropped = READ_ONCE(values[i].dropped); rec->cpu[i].issue = READ_ONCE(values[i].issue); rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass); rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop); rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect); sum_processed += rec->cpu[i].processed; sum_dropped += rec->cpu[i].dropped; sum_issue += rec->cpu[i].issue; sum_xdp_pass += rec->cpu[i].xdp_pass; sum_xdp_drop += rec->cpu[i].xdp_drop; sum_xdp_redirect += rec->cpu[i].xdp_redirect; } rec->total.processed = sum_processed; rec->total.dropped = sum_dropped; rec->total.issue = sum_issue; rec->total.xdp_pass = sum_xdp_pass; rec->total.xdp_drop = sum_xdp_drop; rec->total.xdp_redirect = sum_xdp_redirect; } static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec) { unsigned int nr_cpus = bpf_num_possible_cpus(); __u32 batch, count = 32; struct datarec *values; bool init = false; __u64 *keys; int i, ret; keys = calloc(count, sizeof(__u64)); if (!keys) return -ENOMEM; values = calloc(count * nr_cpus, sizeof(struct datarec)); if (!values) { free(keys); return -ENOMEM; } for (;;) { bool exit = false; ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch, keys, values, &count, NULL); if (ret < 0 && errno != ENOENT) break; if (errno == ENOENT) exit = true; init = true; for (i = 0; i < count; i++) { struct map_entry *e, *x = NULL; __u64 pair = keys[i]; struct datarec *arr; arr = &values[i * nr_cpus]; hash_for_each_possible(rec->xmit_map, e, node, pair) { if (e->pair == pair) { x = e; break; } } if (!x) { x = calloc(1, sizeof(*x)); if (!x) goto cleanup; if (map_entry_init(x, pair) < 0) { free(x); goto cleanup; } hash_add(rec->xmit_map, &x->node, pair); } map_collect_percpu(arr, &x->val); } if (exit) break; count = 32; } free(values); free(keys); return 0; cleanup: free(values); free(keys); return -ENOMEM; } static struct stats_record *alloc_stats_record(void) { struct stats_record *rec; int i; rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record)); if (!rec) { fprintf(stderr, "Failed to allocate memory\n"); return NULL; } if (sample_mask & SAMPLE_RX_CNT) { rec->rx_cnt.cpu = alloc_record_per_cpu(); if (!rec->rx_cnt.cpu) { fprintf(stderr, "Failed to allocate rx_cnt per-CPU array\n"); goto end_rec; } } if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) { for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) { rec->redir_err[i].cpu = alloc_record_per_cpu(); if (!rec->redir_err[i].cpu) { fprintf(stderr, "Failed to allocate redir_err per-CPU array for " "\"%s\" case\n", xdp_redirect_err_names[i]); while (i--) free(rec->redir_err[i].cpu); goto end_rx_cnt; } } } if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) { rec->kthread.cpu = alloc_record_per_cpu(); if (!rec->kthread.cpu) { fprintf(stderr, "Failed to allocate kthread per-CPU array\n"); goto end_redir; } } if (sample_mask & SAMPLE_EXCEPTION_CNT) { for (i = 0; i < XDP_ACTION_MAX; i++) { rec->exception[i].cpu = alloc_record_per_cpu(); if (!rec->exception[i].cpu) { fprintf(stderr, "Failed to allocate exception per-CPU array for " "\"%s\" case\n", action2str(i)); while (i--) free(rec->exception[i].cpu); goto end_kthread; }