History

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf

Alexei Starovoitov says: ==================== pull-request: bpf 2020-11-06 1) Pre-allocated per-cpu hashmap needs to zero-fill reused element, from David. 2) Tighten bpf_lsm function check, from KP. 3) Fix bpftool attaching to flow dissector, from Lorenz. 4) Use -fno-gcse for the whole kernel/bpf/core.c instead of function attribute, from Ard. * git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf: bpf: Update verification logic for LSM programs bpf: Zero-fill re-used per-cpu map element bpf: BPF_PRELOAD depends on BPF_SYSCALL tools/bpftool: Fix attaching flow dissector libbpf: Fix possible use after free in xsk_socket__delete libbpf: Fix null dereference in xsk_socket__delete libbpf, hashmap: Fix undefined behavior in hash_bits bpf: Don't rely on GCC __attribute__((optimize)) to disable GCSE tools, bpftool: Remove two unused variables. tools, bpftool: Avoid array index warnings. xsk: Fix possible memory leak at socket close bpf: Add struct bpf_redir_neigh forward declaration to BPF helper defs samples/bpf: Set rlimit for memlock to infinity in all samples bpf: Fix -Wshadow warnings selftest/bpf: Fix profiler test using CO-RE relocation for enums ====================

Jakub Kicinski committed 6 years ago

Commit 86bbf012 parents bf3e762 6f64e4725 files · +346 −49Browse files

Unified · Split

25 files
  1. include/linux/compiler-gcc.h +0 −2
  2. include/linux/compiler_types.h +0 −4
  3. include/linux/filter.h +11 −11
  4. include/net/xsk_buff_pool.h +1 −1
  5. kernel/bpf/Makefile +5 −1
  6. kernel/bpf/bpf_lsm.c +7 −3
  7. kernel/bpf/core.c +1 −1
  8. kernel/bpf/hashtab.c +28 −2
  9. kernel/bpf/preload/Kconfig +1 −0
  10. net/xdp/xsk.c +2 −1
  11. net/xdp/xsk_buff_pool.c +5 −2
  12. samples/bpf/task_fd_query_user.c +1 −1
  13. samples/bpf/tracex2_user.c +1 −1
  14. samples/bpf/tracex3_user.c +1 −1
  15. samples/bpf/xdp_redirect_cpu_user.c +1 −1
  16. samples/bpf/xdp_rxq_info_user.c +1 −1
  17. scripts/bpf_helpers_doc.py +1 −0
  18. tools/bpf/bpftool/feature.c +6 −1
  19. tools/bpf/bpftool/prog.c +1 −1
  20. tools/bpf/bpftool/skeleton/profiler.bpf.c +2 −2
  21. tools/lib/bpf/hashmap.h +9 −6
  22. tools/lib/bpf/xsk.c +6 −3
  23. tools/testing/selftests/bpf/prog_tests/map_init.c +214 −0
  24. tools/testing/selftests/bpf/progs/profiler.inc.h +8 −3
  25. tools/testing/selftests/bpf/progs/test_map_init.c +33 −0
include/linux/compiler-gcc.h+0−2
@@ -175,5 +175,3 @@ · 174 unmodified lines
175#else175#else
176#define __diag_GCC_8(s)176#define __diag_GCC_8(s)
177#endif177#endif
178
179#define __no_fgcse __attribute__((optimize("-fno-gcse")))
include/linux/compiler_types.h+0−4
@@ -247,10 +247,6 @@ struct ftrace_likely_data { · 246 unmodified lines
247#define asm_inline asm247#define asm_inline asm
248#endif248#endif
249 249
250#ifndef __no_fgcse
251# define __no_fgcse
252#endif
253
254/* Are two types/vars the same type (ignoring qualifiers)? */250/* Are two types/vars the same type (ignoring qualifiers)? */
255#define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))251#define __same_type(a, b) __builtin_types_compatible_p(typeof(a), typeof(b))
256 252
include/linux/filter.h+11−11
@@ -558,21 +558,21 @@ struct sk_filter { · 557 unmodified lines
558DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);558DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
559 559
560#define __BPF_PROG_RUN(prog, ctx, dfunc) ({ \560#define __BPF_PROG_RUN(prog, ctx, dfunc) ({ \
561 u32 ret; \561 u32 __ret; \
562 cant_migrate(); \562 cant_migrate(); \
563 if (static_branch_unlikely(&bpf_stats_enabled_key)) { \563 if (static_branch_unlikely(&bpf_stats_enabled_key)) { \
564 struct bpf_prog_stats *stats; \564 struct bpf_prog_stats *__stats; \
565 u64 start = sched_clock(); \565 u64 __start = sched_clock(); \
566 ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \566 __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \
567 stats = this_cpu_ptr(prog->aux->stats); \567 __stats = this_cpu_ptr(prog->aux->stats); \
568 u64_stats_update_begin(&stats->syncp); \568 u64_stats_update_begin(&__stats->syncp); \
569 stats->cnt++; \569 __stats->cnt++; \
570 stats->nsecs += sched_clock() - start; \570 __stats->nsecs += sched_clock() - __start; \
571 u64_stats_update_end(&stats->syncp); \571 u64_stats_update_end(&__stats->syncp); \
572 } else { \572 } else { \
573 ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \573 __ret = dfunc(ctx, (prog)->insnsi, (prog)->bpf_func); \
574 } \574 } \
575 ret; })575 __ret; })
576 576
577#define BPF_PROG_RUN(prog, ctx) \577#define BPF_PROG_RUN(prog, ctx) \
578 __BPF_PROG_RUN(prog, ctx, bpf_dispatcher_nop_func)578 __BPF_PROG_RUN(prog, ctx, bpf_dispatcher_nop_func)
include/net/xsk_buff_pool.h+1−1
@@ -86,7 +86,7 @@ int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem, · 85 unmodified lines
86void xp_destroy(struct xsk_buff_pool *pool);86void xp_destroy(struct xsk_buff_pool *pool);
87void xp_release(struct xdp_buff_xsk *xskb);87void xp_release(struct xdp_buff_xsk *xskb);
88void xp_get_pool(struct xsk_buff_pool *pool);88void xp_get_pool(struct xsk_buff_pool *pool);
89void xp_put_pool(struct xsk_buff_pool *pool);89bool xp_put_pool(struct xsk_buff_pool *pool);
90void xp_clear_dev(struct xsk_buff_pool *pool);90void xp_clear_dev(struct xsk_buff_pool *pool);
91void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);91void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
92void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);92void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs);
kernel/bpf/Makefile+5−1
@@ -1,6 +1,10 @@
1# SPDX-License-Identifier: GPL-2.01# SPDX-License-Identifier: GPL-2.0
2obj-y := core.o2obj-y := core.o
3CFLAGS_core.o += $(call cc-disable-warning, override-init)3ifneq ($(CONFIG_BPF_JIT_ALWAYS_ON),y)
4# ___bpf_prog_run() needs GCSE disabled on x86; see 3193c0836f203 for details
5cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse
6endif
7CFLAGS_core.o += $(call cc-disable-warning, override-init) $(cflags-nogcse-yy)
4 8
5obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o9obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_iter.o map_iter.o task_iter.o prog_iter.o
6obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o10obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
kernel/bpf/bpf_lsm.c+7−3
@@ -13,6 +13,7 @@ · 12 unmodified lines
13#include <linux/bpf_verifier.h>13#include <linux/bpf_verifier.h>
14#include <net/bpf_sk_storage.h>14#include <net/bpf_sk_storage.h>
15#include <linux/bpf_local_storage.h>15#include <linux/bpf_local_storage.h>
16#include <linux/btf_ids.h>
16 17
17/* For every LSM hook that allows attachment of BPF programs, declare a nop18/* For every LSM hook that allows attachment of BPF programs, declare a nop
18 * function where a BPF program can be attached.19 * function where a BPF program can be attached.
@@ -26,7 +27,11 @@ noinline RET bpf_lsm_##NAME(__VA_ARGS__) \ · 7 unmodified lines
26#include <linux/lsm_hook_defs.h>27#include <linux/lsm_hook_defs.h>
27#undef LSM_HOOK28#undef LSM_HOOK
28 29
29#define BPF_LSM_SYM_PREFX "bpf_lsm_"30#define LSM_HOOK(RET, DEFAULT, NAME, ...) BTF_ID(func, bpf_lsm_##NAME)
31BTF_SET_START(bpf_lsm_hooks)
32#include <linux/lsm_hook_defs.h>
33#undef LSM_HOOK
34BTF_SET_END(bpf_lsm_hooks)
30 35
31int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,36int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
32 const struct bpf_prog *prog)37 const struct bpf_prog *prog)
@@ -37,8 +42,7 @@ int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, · 4 unmodified lines
37 return -EINVAL;42 return -EINVAL;
38 }43 }
39 44
40 if (strncmp(BPF_LSM_SYM_PREFX, prog->aux->attach_func_name,45 if (!btf_id_set_contains(&bpf_lsm_hooks, prog->aux->attach_btf_id)) {
41 sizeof(BPF_LSM_SYM_PREFX) - 1)) {
42 bpf_log(vlog, "attach_btf_id %u points to wrong type name %s\n",46 bpf_log(vlog, "attach_btf_id %u points to wrong type name %s\n",
43 prog->aux->attach_btf_id, prog->aux->attach_func_name);47 prog->aux->attach_btf_id, prog->aux->attach_func_name);
44 return -EINVAL;48 return -EINVAL;
kernel/bpf/core.c+1−1
@@ -1369,7 +1369,7 @@ u64 __weak bpf_probe_read_kernel(void *dst, u32 size, const void *unsafe_ptr) · 1368 unmodified lines
1369 *1369 *
1370 * Decode and execute eBPF instructions.1370 * Decode and execute eBPF instructions.
1371 */1371 */
1372static u64 __no_fgcse ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack)1372static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack)
1373{1373{
1374#define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y1374#define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y
1375#define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z1375#define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z
kernel/bpf/hashtab.c+28−2
@@ -821,6 +821,32 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr, · 820 unmodified lines
821 }821 }
822}822}
823 823
824static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
825 void *value, bool onallcpus)
826{
827 /* When using prealloc and not setting the initial value on all cpus,
828 * zero-fill element values for other cpus (just as what happens when
829 * not using prealloc). Otherwise, bpf program has no way to ensure
830 * known initial values for cpus other than current one
831 * (onallcpus=false always when coming from bpf prog).
832 */
833 if (htab_is_prealloc(htab) && !onallcpus) {
834 u32 size = round_up(htab->map.value_size, 8);
835 int current_cpu = raw_smp_processor_id();
836 int cpu;
837
838 for_each_possible_cpu(cpu) {
839 if (cpu == current_cpu)
840 bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value,
841 size);
842 else
843 memset(per_cpu_ptr(pptr, cpu), 0, size);
844 }
845 } else {
846 pcpu_copy_value(htab, pptr, value, onallcpus);
847 }
848}
849
824static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab)850static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab)
825{851{
826 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS &&852 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS &&
@@ -891,7 +917,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key, · 64 unmodified lines
891 }917 }
892 }918 }
893 919
894 pcpu_copy_value(htab, pptr, value, onallcpus);920 pcpu_init_value(htab, pptr, value, onallcpus);
895 921
896 if (!prealloc)922 if (!prealloc)
897 htab_elem_set_ptr(l_new, key_size, pptr);923 htab_elem_set_ptr(l_new, key_size, pptr);
@@ -1183,7 +1209,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, · 285 unmodified lines
1183 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),1209 pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),
1184 value, onallcpus);1210 value, onallcpus);
1185 } else {1211 } else {
1186 pcpu_copy_value(htab, htab_elem_get_ptr(l_new, key_size),1212 pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),
1187 value, onallcpus);1213 value, onallcpus);
1188 hlist_nulls_add_head_rcu(&l_new->hash_node, head);1214 hlist_nulls_add_head_rcu(&l_new->hash_node, head);
1189 l_new = NULL;1215 l_new = NULL;
kernel/bpf/preload/Kconfig+1−0
@@ -6,6 +6,7 @@ config USERMODE_DRIVER · 5 unmodified lines
6menuconfig BPF_PRELOAD6menuconfig BPF_PRELOAD
7 bool "Preload BPF file system with kernel specific program and map iterators"7 bool "Preload BPF file system with kernel specific program and map iterators"
8 depends on BPF8 depends on BPF
9 depends on BPF_SYSCALL
9 # The dependency on !COMPILE_TEST prevents it from being enabled10 # The dependency on !COMPILE_TEST prevents it from being enabled
10 # in allmodconfig or allyesconfig configurations11 # in allmodconfig or allyesconfig configurations
11 depends on !COMPILE_TEST12 depends on !COMPILE_TEST
net/xdp/xsk.c+2−1
@@ -1146,7 +1146,8 @@ static void xsk_destruct(struct sock *sk) · 1145 unmodified lines
1146 if (!sock_flag(sk, SOCK_DEAD))1146 if (!sock_flag(sk, SOCK_DEAD))
1147 return;1147 return;
1148 1148
1149 xp_put_pool(xs->pool);1149 if (!xp_put_pool(xs->pool))
1150 xdp_put_umem(xs->umem);
1150 1151
1151 sk_refcnt_debug_dec(sk);1152 sk_refcnt_debug_dec(sk);
1152}1153}
net/xdp/xsk_buff_pool.c+5−2
@@ -251,15 +251,18 @@ void xp_get_pool(struct xsk_buff_pool *pool) · 250 unmodified lines
251 refcount_inc(&pool->users);251 refcount_inc(&pool->users);
252}252}
253 253
254void xp_put_pool(struct xsk_buff_pool *pool)254bool xp_put_pool(struct xsk_buff_pool *pool)
255{255{
256 if (!pool)256 if (!pool)
257 return;257 return false;
258 258
259 if (refcount_dec_and_test(&pool->users)) {259 if (refcount_dec_and_test(&pool->users)) {
260 INIT_WORK(&pool->work, xp_release_deferred);260 INIT_WORK(&pool->work, xp_release_deferred);
261 schedule_work(&pool->work);261 schedule_work(&pool->work);
262 return true;
262 }263 }
264
265 return false;
263}266}
264 267
265static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool)268static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool)
samples/bpf/task_fd_query_user.c+1−1
@@ -290,7 +290,7 @@ static int test_debug_fs_uprobe(char *binary_path, long offset, bool is_return) · 289 unmodified lines
290 290
291int main(int argc, char **argv)291int main(int argc, char **argv)
292{292{
293 struct rlimit r = {1024*1024, RLIM_INFINITY};293 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
294 extern char __executable_start;294 extern char __executable_start;
295 char filename[256], buf[256];295 char filename[256], buf[256];
296 __u64 uprobe_file_offset;296 __u64 uprobe_file_offset;
samples/bpf/tracex2_user.c+1−1
@@ -116,7 +116,7 @@ static void int_exit(int sig) · 115 unmodified lines
116 116
117int main(int ac, char **argv)117int main(int ac, char **argv)
118{118{
119 struct rlimit r = {1024*1024, RLIM_INFINITY};119 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
120 long key, next_key, value;120 long key, next_key, value;
121 struct bpf_link *links[2];121 struct bpf_link *links[2];
122 struct bpf_program *prog;122 struct bpf_program *prog;
samples/bpf/tracex3_user.c+1−1
@@ -107,7 +107,7 @@ static void print_hist(int fd) · 106 unmodified lines
107 107
108int main(int ac, char **argv)108int main(int ac, char **argv)
109{109{
110 struct rlimit r = {1024*1024, RLIM_INFINITY};110 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
111 struct bpf_link *links[2];111 struct bpf_link *links[2];
112 struct bpf_program *prog;112 struct bpf_program *prog;
113 struct bpf_object *obj;113 struct bpf_object *obj;
samples/bpf/xdp_redirect_cpu_user.c+1−1
@@ -765,7 +765,7 @@ static int load_cpumap_prog(char *file_name, char *prog_name, · 764 unmodified lines
765 765
766int main(int argc, char **argv)766int main(int argc, char **argv)
767{767{
768 struct rlimit r = {10 * 1024 * 1024, RLIM_INFINITY};768 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
769 char *prog_name = "xdp_cpu_map5_lb_hash_ip_pairs";769 char *prog_name = "xdp_cpu_map5_lb_hash_ip_pairs";
770 char *mprog_filename = "xdp_redirect_kern.o";770 char *mprog_filename = "xdp_redirect_kern.o";
771 char *redir_interface = NULL, *redir_map = NULL;771 char *redir_interface = NULL, *redir_map = NULL;
samples/bpf/xdp_rxq_info_user.c+1−1
@@ -450,7 +450,7 @@ static void stats_poll(int interval, int action, __u32 cfg_opt) · 449 unmodified lines
450int main(int argc, char **argv)450int main(int argc, char **argv)
451{451{
452 __u32 cfg_options= NO_TOUCH ; /* Default: Don't touch packet memory */452 __u32 cfg_options= NO_TOUCH ; /* Default: Don't touch packet memory */
453 struct rlimit r = {10 * 1024 * 1024, RLIM_INFINITY};453 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
454 struct bpf_prog_load_attr prog_load_attr = {454 struct bpf_prog_load_attr prog_load_attr = {
455 .prog_type = BPF_PROG_TYPE_XDP,455 .prog_type = BPF_PROG_TYPE_XDP,
456 };456 };
scripts/bpf_helpers_doc.py+1−0
@@ -408,6 +408,7 @@ class PrinterHelpers(Printer): · 407 unmodified lines
408 'struct bpf_perf_event_data',408 'struct bpf_perf_event_data',
409 'struct bpf_perf_event_value',409 'struct bpf_perf_event_value',
410 'struct bpf_pidns_info',410 'struct bpf_pidns_info',
411 'struct bpf_redir_neigh',
411 'struct bpf_sock',412 'struct bpf_sock',
412 'struct bpf_sock_addr',413 'struct bpf_sock_addr',
413 'struct bpf_sock_ops',414 'struct bpf_sock_ops',
tools/bpf/bpftool/feature.c+6−1
@@ -843,9 +843,14 @@ static int handle_perms(void) · 842 unmodified lines
843 else843 else
844 p_err("missing %s%s%s%s%s%s%s%srequired for full feature probing; run as root or use 'unprivileged'",844 p_err("missing %s%s%s%s%s%s%s%srequired for full feature probing; run as root or use 'unprivileged'",
845 capability_msg(bpf_caps, 0),845 capability_msg(bpf_caps, 0),
846#ifdef CAP_BPF
846 capability_msg(bpf_caps, 1),847 capability_msg(bpf_caps, 1),
847 capability_msg(bpf_caps, 2),848 capability_msg(bpf_caps, 2),
848 capability_msg(bpf_caps, 3));849 capability_msg(bpf_caps, 3)
850#else
851 "", "", "", "", "", ""
852#endif /* CAP_BPF */
853 );
849 goto exit_free;854 goto exit_free;
850 }855 }
851 856
tools/bpf/bpftool/prog.c+1−1
@@ -940,7 +940,7 @@ static int parse_attach_detach_args(int argc, char **argv, int *progfd, · 939 unmodified lines
940 }940 }
941 941
942 if (*attach_type == BPF_FLOW_DISSECTOR) {942 if (*attach_type == BPF_FLOW_DISSECTOR) {
943 *mapfd = -1;943 *mapfd = 0;
944 return 0;944 return 0;
945 }945 }
946 946
tools/bpf/bpftool/skeleton/profiler.bpf.c+2−2
@@ -70,7 +70,7 @@ int BPF_PROG(fentry_XXX) · 69 unmodified lines
70static inline void70static inline void
71fexit_update_maps(u32 id, struct bpf_perf_event_value *after)71fexit_update_maps(u32 id, struct bpf_perf_event_value *after)
72{72{
73 struct bpf_perf_event_value *before, diff, *accum;73 struct bpf_perf_event_value *before, diff;
74 74
75 before = bpf_map_lookup_elem(&fentry_readings, &id);75 before = bpf_map_lookup_elem(&fentry_readings, &id);
76 /* only account samples with a valid fentry_reading */76 /* only account samples with a valid fentry_reading */
@@ -95,7 +95,7 @@ int BPF_PROG(fexit_XXX) · 18 unmodified lines
95{95{
96 struct bpf_perf_event_value readings[MAX_NUM_MATRICS];96 struct bpf_perf_event_value readings[MAX_NUM_MATRICS];
97 u32 cpu = bpf_get_smp_processor_id();97 u32 cpu = bpf_get_smp_processor_id();
98 u32 i, one = 1, zero = 0;98 u32 i, zero = 0;
99 int err;99 int err;
100 u64 *count;100 u64 *count;
101 101
tools/lib/bpf/hashmap.h+9−6
@@ -15,6 +15,9 @@ · 14 unmodified lines
15static inline size_t hash_bits(size_t h, int bits)15static inline size_t hash_bits(size_t h, int bits)
16{16{
17 /* shuffle bits and return requested number of upper bits */17 /* shuffle bits and return requested number of upper bits */
18 if (bits == 0)
19 return 0;
20
18#if (__SIZEOF_SIZE_T__ == __SIZEOF_LONG_LONG__)21#if (__SIZEOF_SIZE_T__ == __SIZEOF_LONG_LONG__)
19 /* LP64 case */22 /* LP64 case */
20 return (h * 11400714819323198485llu) >> (__SIZEOF_LONG_LONG__ * 8 - bits);23 return (h * 11400714819323198485llu) >> (__SIZEOF_LONG_LONG__ * 8 - bits);
@@ -174,17 +177,17 @@ bool hashmap__find(const struct hashmap *map, const void *key, void **value); · 153 unmodified lines
174 * @key: key to iterate entries for177 * @key: key to iterate entries for
175 */178 */
176#define hashmap__for_each_key_entry(map, cur, _key) \179#define hashmap__for_each_key_entry(map, cur, _key) \
177 for (cur = ({ size_t bkt = hash_bits(map->hash_fn((_key), map->ctx),\180 for (cur = map->buckets \
178 map->cap_bits); \181 ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \
179 map->buckets ? map->buckets[bkt] : NULL; }); \182 : NULL; \
180 cur; \183 cur; \
181 cur = cur->next) \184 cur = cur->next) \
182 if (map->equal_fn(cur->key, (_key), map->ctx))185 if (map->equal_fn(cur->key, (_key), map->ctx))
183 186
184#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \187#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \
185 for (cur = ({ size_t bkt = hash_bits(map->hash_fn((_key), map->ctx),\188 for (cur = map->buckets \
186 map->cap_bits); \189 ? map->buckets[hash_bits(map->hash_fn((_key), map->ctx), map->cap_bits)] \
187 cur = map->buckets ? map->buckets[bkt] : NULL; }); \190 : NULL; \
188 cur && ({ tmp = cur->next; true; }); \191 cur && ({ tmp = cur->next; true; }); \
189 cur = tmp) \192 cur = tmp) \
190 if (map->equal_fn(cur->key, (_key), map->ctx))193 if (map->equal_fn(cur->key, (_key), map->ctx))
tools/lib/bpf/xsk.c+6−3
@@ -891,13 +891,16 @@ int xsk_umem__delete(struct xsk_umem *umem) · 890 unmodified lines
891void xsk_socket__delete(struct xsk_socket *xsk)891void xsk_socket__delete(struct xsk_socket *xsk)
892{892{
893 size_t desc_sz = sizeof(struct xdp_desc);893 size_t desc_sz = sizeof(struct xdp_desc);
894 struct xsk_ctx *ctx = xsk->ctx;
895 struct xdp_mmap_offsets off;894 struct xdp_mmap_offsets off;
895 struct xsk_umem *umem;
896 struct xsk_ctx *ctx;
896 int err;897 int err;
897 898
898 if (!xsk)899 if (!xsk)
899 return;900 return;
900 901
902 ctx = xsk->ctx;
903 umem = ctx->umem;
901 if (ctx->prog_fd != -1) {904 if (ctx->prog_fd != -1) {
902 xsk_delete_bpf_maps(xsk);905 xsk_delete_bpf_maps(xsk);
903 close(ctx->prog_fd);906 close(ctx->prog_fd);
@@ -917,11 +920,11 @@ void xsk_socket__delete(struct xsk_socket *xsk) · 13 unmodified lines
917 920
918 xsk_put_ctx(ctx);921 xsk_put_ctx(ctx);
919 922
920 ctx->umem->refcount--;923 umem->refcount--;
921 /* Do not close an fd that also has an associated umem connected924 /* Do not close an fd that also has an associated umem connected
922 * to it.925 * to it.
923 */926 */
924 if (xsk->fd != ctx->umem->fd)927 if (xsk->fd != umem->fd)
925 close(xsk->fd);928 close(xsk->fd);
926 free(xsk);929 free(xsk);
927}930}
tools/testing/selftests/bpf/prog_tests/map_init.c+214−0
@@ -0,0 +1,214 @@
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2020 Tessares SA <http://www.tessares.net> */
3
4#include <test_progs.h>
5#include "test_map_init.skel.h"
6
7#define TEST_VALUE 0x1234
8#define FILL_VALUE 0xdeadbeef
9
10static int nr_cpus;
11static int duration;
12
13typedef unsigned long long map_key_t;
14typedef unsigned long long map_value_t;
15typedef struct {
16 map_value_t v; /* padding */
17} __bpf_percpu_val_align pcpu_map_value_t;
18
19
20static int map_populate(int map_fd, int num)
21{
22 pcpu_map_value_t value[nr_cpus];
23 int i, err;
24 map_key_t key;
25
26 for (i = 0; i < nr_cpus; i++)
27 bpf_percpu(value, i) = FILL_VALUE;
28
29 for (key = 1; key <= num; key++) {
30 err = bpf_map_update_elem(map_fd, &key, value, BPF_NOEXIST);
31 if (!ASSERT_OK(err, "bpf_map_update_elem"))
32 return -1;
33 }
34
35 return 0;
36}
37
38static struct test_map_init *setup(enum bpf_map_type map_type, int map_sz,
39 int *map_fd, int populate)
40{
41 struct test_map_init *skel;
42 int err;
43
44 skel = test_map_init__open();
45 if (!ASSERT_OK_PTR(skel, "skel_open"))
46 return NULL;
47
48 err = bpf_map__set_type(skel->maps.hashmap1, map_type);
49 if (!ASSERT_OK(err, "bpf_map__set_type"))
50 goto error;
51
52 err = bpf_map__set_max_entries(skel->maps.hashmap1, map_sz);
53 if (!ASSERT_OK(err, "bpf_map__set_max_entries"))
54 goto error;
55
56 err = test_map_init__load(skel);
57 if (!ASSERT_OK(err, "skel_load"))
58 goto error;
59
60 *map_fd = bpf_map__fd(skel->maps.hashmap1);
61 if (CHECK(*map_fd < 0, "bpf_map__fd", "failed\n"))
62 goto error;
63
64 err = map_populate(*map_fd, populate);
65 if (!ASSERT_OK(err, "map_populate"))
66 goto error_map;
67
68 return skel;
69
70error_map:
71 close(*map_fd);
72error:
73 test_map_init__destroy(skel);
74 return NULL;
75}
76
77/* executes bpf program that updates map with key, value */
78static int prog_run_insert_elem(struct test_map_init *skel, map_key_t key,
79 map_value_t value)
80{
81 struct test_map_init__bss *bss;
82
83 bss = skel->bss;
84
85 bss->inKey = key;
86 bss->inValue = value;
87 bss->inPid = getpid();
88
89 if (!ASSERT_OK(test_map_init__attach(skel), "skel_attach"))
90 return -1;
91
92 /* Let tracepoint trigger */
93 syscall(__NR_getpgid);
94
95 test_map_init__detach(skel);
96
97 return 0;
98}
99
100static int check_values_one_cpu(pcpu_map_value_t *value, map_value_t expected)
101{
102 int i, nzCnt = 0;
103 map_value_t val;
104
105 for (i = 0; i < nr_cpus; i++) {
106 val = bpf_percpu(value, i);
107 if (val) {
108 if (CHECK(val != expected, "map value",
109 "unexpected for cpu %d: 0x%llx\n", i, val))
110 return -1;
111 nzCnt++;
112 }
113 }
114
115 if (CHECK(nzCnt != 1, "map value", "set for %d CPUs instead of 1!\n",
116 nzCnt))
117 return -1;
118
119 return 0;
120}
121
122/* Add key=1 elem with values set for all CPUs
123 * Delete elem key=1
124 * Run bpf prog that inserts new key=1 elem with value=0x1234
125 * (bpf prog can only set value for current CPU)
126 * Lookup Key=1 and check value is as expected for all CPUs:
127 * value set by bpf prog for one CPU, 0 for all others
128 */
129static void test_pcpu_map_init(void)
130{
131 pcpu_map_value_t value[nr_cpus];
132 struct test_map_init *skel;
133 int map_fd, err;
134 map_key_t key;
135
136 /* max 1 elem in map so insertion is forced to reuse freed entry */
137 skel = setup(BPF_MAP_TYPE_PERCPU_HASH, 1, &map_fd, 1);
138 if (!ASSERT_OK_PTR(skel, "prog_setup"))
139 return;
140
141 /* delete element so the entry can be re-used*/
142 key = 1;
143 err = bpf_map_delete_elem(map_fd, &key);
144 if (!ASSERT_OK(err, "bpf_map_delete_elem"))
145 goto cleanup;
146
147 /* run bpf prog that inserts new elem, re-using the slot just freed */
148 err = prog_run_insert_elem(skel, key, TEST_VALUE);
149 if (!ASSERT_OK(err, "prog_run_insert_elem"))
150 goto cleanup;
151
152 /* check that key=1 was re-created by bpf prog */
153 err = bpf_map_lookup_elem(map_fd, &key, value);
154 if (!ASSERT_OK(err, "bpf_map_lookup_elem"))
155 goto cleanup;
156
157 /* and has expected values */
158 check_values_one_cpu(value, TEST_VALUE);
159
160cleanup:
161 test_map_init__destroy(skel);
162}
163
164/* Add key=1 and key=2 elems with values set for all CPUs
165 * Run bpf prog that inserts new key=3 elem
166 * (only for current cpu; other cpus should have initial value = 0)
167 * Lookup Key=1 and check value is as expected for all CPUs
168 */
169static void test_pcpu_lru_map_init(void)
170{
171 pcpu_map_value_t value[nr_cpus];
172 struct test_map_init *skel;
173 int map_fd, err;
174 map_key_t key;
175
176 /* Set up LRU map with 2 elements, values filled for all CPUs.
177 * With these 2 elements, the LRU map is full
178 */
179 skel = setup(BPF_MAP_TYPE_LRU_PERCPU_HASH, 2, &map_fd, 2);
180 if (!ASSERT_OK_PTR(skel, "prog_setup"))
181 return;
182
183 /* run bpf prog that inserts new key=3 element, re-using LRU slot */
184 key = 3;
185 err = prog_run_insert_elem(skel, key, TEST_VALUE);
186 if (!ASSERT_OK(err, "prog_run_insert_elem"))
187 goto cleanup;
188
189 /* check that key=3 replaced one of earlier elements */
190 err = bpf_map_lookup_elem(map_fd, &key, value);
191 if (!ASSERT_OK(err, "bpf_map_lookup_elem"))
192 goto cleanup;
193
194 /* and has expected values */
195 check_values_one_cpu(value, TEST_VALUE);
196
197cleanup:
198 test_map_init__destroy(skel);
199}
200
201void test_map_init(void)
202{
203 nr_cpus = bpf_num_possible_cpus();
204 if (nr_cpus <= 1) {
205 printf("%s:SKIP: >1 cpu needed for this test\n", __func__);
206 test__skip();
207 return;
208 }
209
210 if (test__start_subtest("pcpu_map_init"))
211 test_pcpu_map_init();
212 if (test__start_subtest("pcpu_lru_map_init"))
213 test_pcpu_lru_map_init();
214}
tools/testing/selftests/bpf/progs/profiler.inc.h+8−3
@@ -243,7 +243,10 @@ static ino_t get_inode_from_kernfs(struct kernfs_node* node) · 242 unmodified lines
243 }243 }
244}244}
245 245
246int pids_cgrp_id = 1;246extern bool CONFIG_CGROUP_PIDS __kconfig __weak;
247enum cgroup_subsys_id___local {
248 pids_cgrp_id___local = 123, /* value doesn't matter */
249};
247 250
248static INLINE void* populate_cgroup_info(struct cgroup_data_t* cgroup_data,251static INLINE void* populate_cgroup_info(struct cgroup_data_t* cgroup_data,
249 struct task_struct* task,252 struct task_struct* task,
@@ -253,7 +256,9 @@ static INLINE void* populate_cgroup_info(struct cgroup_data_t* cgroup_data, · 3 unmodified lines
253 BPF_CORE_READ(task, nsproxy, cgroup_ns, root_cset, dfl_cgrp, kn);256 BPF_CORE_READ(task, nsproxy, cgroup_ns, root_cset, dfl_cgrp, kn);
254 struct kernfs_node* proc_kernfs = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn);257 struct kernfs_node* proc_kernfs = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn);
255 258
256 if (ENABLE_CGROUP_V1_RESOLVER) {259 if (ENABLE_CGROUP_V1_RESOLVER && CONFIG_CGROUP_PIDS) {
260 int cgrp_id = bpf_core_enum_value(enum cgroup_subsys_id___local,
261 pids_cgrp_id___local);
257#ifdef UNROLL262#ifdef UNROLL
258#pragma unroll263#pragma unroll
259#endif264#endif
@@ -262,7 +267,7 @@ static INLINE void* populate_cgroup_info(struct cgroup_data_t* cgroup_data, · 2 unmodified lines
262 BPF_CORE_READ(task, cgroups, subsys[i]);267 BPF_CORE_READ(task, cgroups, subsys[i]);
263 if (subsys != NULL) {268 if (subsys != NULL) {
264 int subsys_id = BPF_CORE_READ(subsys, ss, id);269 int subsys_id = BPF_CORE_READ(subsys, ss, id);
265 if (subsys_id == pids_cgrp_id) {270 if (subsys_id == cgrp_id) {
266 proc_kernfs = BPF_CORE_READ(subsys, cgroup, kn);271 proc_kernfs = BPF_CORE_READ(subsys, cgroup, kn);
267 root_kernfs = BPF_CORE_READ(subsys, ss, root, kf_root, kn);272 root_kernfs = BPF_CORE_READ(subsys, ss, root, kf_root, kn);
268 break;273 break;
tools/testing/selftests/bpf/progs/test_map_init.c+33−0
@@ -0,0 +1,33 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Tessares SA <http://www.tessares.net> */
3
4#include "vmlinux.h"
5#include <bpf/bpf_helpers.h>
6
7__u64 inKey = 0;
8__u64 inValue = 0;
9__u32 inPid = 0;
10
11struct {
12 __uint(type, BPF_MAP_TYPE_PERCPU_HASH);
13 __uint(max_entries, 2);
14 __type(key, __u64);
15 __type(value, __u64);
16} hashmap1 SEC(".maps");
17
18
19SEC("tp/syscalls/sys_enter_getpgid")
20int sysenter_getpgid(const void *ctx)
21{
22 /* Just do it for once, when called from our own test prog. This
23 * ensures the map value is only updated for a single CPU.
24 */
25 int cur_pid = bpf_get_current_pid_tgid() >> 32;
26
27 if (cur_pid == inPid)
28 bpf_map_update_elem(&hashmap1, &inKey, &inValue, BPF_NOEXIST);
29
30 return 0;
31}
32
33char _license[] SEC("license") = "GPL";