History

bpf: Convert bpf_preload.ko to use light skeleton.

The main change is a move of the single line #include "iterators.lskel.h" from iterators/iterators.c to bpf_preload_kern.c. Which means that generated light skeleton can be used from user space or user mode driver like iterators.c or from the kernel module or the kernel itself. The direct use of light skeleton from the kernel module simplifies the code, since UMD is no longer necessary. The libbpf.a required user space and UMD. The CO-RE in the kernel and generated "loader bpf program" used by the light skeleton are capable to perform complex loading operations traditionally provided by libbpf. In addition UMD approach was launching UMD process every time bpffs has to be mounted. With light skeleton in the kernel the bpf_preload kernel module loads bpf iterators once and pins them multiple times into different bpffs mounts.

Alexei Starovoitov committed 4 years ago

Commit cb80ddc1 parent d7beb3d9 files · +70 −247Browse files

Showing the changes to kernel/bpf (9 of 9 files). Show all

Unified · Split

9 files
  1. kernel/bpf/inode.c +10 −29
  2. kernel/bpf/preload/Kconfig +3 −4
  3. kernel/bpf/preload/Makefile +2 −12
  4. kernel/bpf/preload/bpf_preload.h +4 −4
  5. kernel/bpf/preload/bpf_preload_kern.c +49 −70
  6. kernel/bpf/preload/bpf_preload_umd_blob.S +0 −7
  7. kernel/bpf/preload/iterators/bpf_preload_common.h +0 −13
  8. kernel/bpf/preload/iterators/iterators.c +0 −108
  9. kernel/bpf/syscall.c +2 −0
kernel/bpf/inode.c+10−29
@@ -710,11 +710,10 @@ static DEFINE_MUTEX(bpf_preload_lock); · 709 unmodified lines
710static int populate_bpffs(struct dentry *parent)710static int populate_bpffs(struct dentry *parent)
711{711{
712 struct bpf_preload_info objs[BPF_PRELOAD_LINKS] = {};712 struct bpf_preload_info objs[BPF_PRELOAD_LINKS] = {};
713 struct bpf_link *links[BPF_PRELOAD_LINKS] = {};
714 int err = 0, i;713 int err = 0, i;
715 714
716 /* grab the mutex to make sure the kernel interactions with bpf_preload715 /* grab the mutex to make sure the kernel interactions with bpf_preload
717 * UMD are serialized716 * are serialized
718 */717 */
719 mutex_lock(&bpf_preload_lock);718 mutex_lock(&bpf_preload_lock);
720 719
@@ -722,40 +721,22 @@ static int populate_bpffs(struct dentry *parent) · 1 unmodified line
722 if (!bpf_preload_mod_get())721 if (!bpf_preload_mod_get())
723 goto out;722 goto out;
724 723
725 if (!bpf_preload_ops->info.tgid) {724 err = bpf_preload_ops->preload(objs);
726 /* preload() will start UMD that will load BPF iterator programs */725 if (err)
727 err = bpf_preload_ops->preload(objs);726 goto out_put;
728 if (err)727 for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
728 bpf_link_inc(objs[i].link);
729 err = bpf_iter_link_pin_kernel(parent,
730 objs[i].link_name, objs[i].link);
731 if (err) {
732 bpf_link_put(objs[i].link);
729 goto out_put;733 goto out_put;
730 for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
731 links[i] = bpf_link_by_id(objs[i].link_id);
732 if (IS_ERR(links[i])) {
733 err = PTR_ERR(links[i]);
734 goto out_put;
735 }
736 }734 }
737 for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
738 err = bpf_iter_link_pin_kernel(parent,
739 objs[i].link_name, links[i]);
740 if (err)
741 goto out_put;
742 /* do not unlink successfully pinned links even
743 * if later link fails to pin
744 */
745 links[i] = NULL;
746 }
747 /* finish() will tell UMD process to exit */
748 err = bpf_preload_ops->finish();
749 if (err)
750 goto out_put;
751 }735 }
752out_put:736out_put:
753 bpf_preload_mod_put();737 bpf_preload_mod_put();
754out:738out:
755 mutex_unlock(&bpf_preload_lock);739 mutex_unlock(&bpf_preload_lock);
756 for (i = 0; i < BPF_PRELOAD_LINKS && err; i++)
757 if (!IS_ERR_OR_NULL(links[i]))
758 bpf_link_put(links[i]);
759 return err;740 return err;
760}741}
761 742
kernel/bpf/preload/Kconfig+3−4
@@ -18,10 +18,9 @@ menuconfig BPF_PRELOAD · 17 unmodified lines
18 18
19if BPF_PRELOAD19if BPF_PRELOAD
20config BPF_PRELOAD_UMD20config BPF_PRELOAD_UMD
21 tristate "bpf_preload kernel module with user mode driver"21 tristate "bpf_preload kernel module"
22 depends on CC_CAN_LINK
23 depends on m || CC_CAN_LINK_STATIC
24 default m22 default m
25 help23 help
26 This builds bpf_preload kernel module with embedded user mode driver.24 This builds bpf_preload kernel module with embedded BPF programs for
25 introspection in bpffs.
27endif26endif
kernel/bpf/preload/Makefile+2−12
@@ -3,16 +3,6 @@ · 2 unmodified lines
3LIBBPF_SRCS = $(srctree)/tools/lib/bpf/3LIBBPF_SRCS = $(srctree)/tools/lib/bpf/
4LIBBPF_INCLUDE = $(LIBBPF_SRCS)/..4LIBBPF_INCLUDE = $(LIBBPF_SRCS)/..
5 5
6userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi \
7 -I $(LIBBPF_INCLUDE) -Wno-unused-result
8
9userprogs := bpf_preload_umd
10
11bpf_preload_umd-objs := iterators/iterators.o
12
13$(obj)/bpf_preload_umd:
14
15$(obj)/bpf_preload_umd_blob.o: $(obj)/bpf_preload_umd
16
17obj-$(CONFIG_BPF_PRELOAD_UMD) += bpf_preload.o6obj-$(CONFIG_BPF_PRELOAD_UMD) += bpf_preload.o
18bpf_preload-objs += bpf_preload_kern.o bpf_preload_umd_blob.o7CFLAGS_bpf_preload_kern.o += -I $(LIBBPF_INCLUDE)
8bpf_preload-objs += bpf_preload_kern.o
kernel/bpf/preload/bpf_preload.h+4−4
@@ -2,13 +2,13 @@ · 1 unmodified line
2#ifndef _BPF_PRELOAD_H2#ifndef _BPF_PRELOAD_H
3#define _BPF_PRELOAD_H3#define _BPF_PRELOAD_H
4 4
5#include <linux/usermode_driver.h>5struct bpf_preload_info {
6#include "iterators/bpf_preload_common.h"6 char link_name[16];
7 struct bpf_link *link;
8};
7 9
8struct bpf_preload_ops {10struct bpf_preload_ops {
9 struct umd_info info;
10 int (*preload)(struct bpf_preload_info *);11 int (*preload)(struct bpf_preload_info *);
11 int (*finish)(void);
12 struct module *owner;12 struct module *owner;
13};13};
14extern struct bpf_preload_ops *bpf_preload_ops;14extern struct bpf_preload_ops *bpf_preload_ops;
kernel/bpf/preload/bpf_preload_kern.c+49−70
@@ -2,101 +2,80 @@ · 1 unmodified line
2#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt2#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3#include <linux/init.h>3#include <linux/init.h>
4#include <linux/module.h>4#include <linux/module.h>
5#include <linux/pid.h>
6#include <linux/fs.h>
7#include <linux/sched/signal.h>
8#include "bpf_preload.h"5#include "bpf_preload.h"
6#include "iterators/iterators.lskel.h"
9 7
10extern char bpf_preload_umd_start;8static struct bpf_link *maps_link, *progs_link;
11extern char bpf_preload_umd_end;9static struct iterators_bpf *skel;
12 10
13static int preload(struct bpf_preload_info *obj);11static void free_links_and_skel(void)
14static int finish(void);12{
13 if (!IS_ERR_OR_NULL(maps_link))
14 bpf_link_put(maps_link);
15 if (!IS_ERR_OR_NULL(progs_link))
16 bpf_link_put(progs_link);
17 iterators_bpf__destroy(skel);
18}
19
20static int preload(struct bpf_preload_info *obj)
21{
22 strlcpy(obj[0].link_name, "maps.debug", sizeof(obj[0].link_name));
23 obj[0].link = maps_link;
24 strlcpy(obj[1].link_name, "progs.debug", sizeof(obj[1].link_name));
25 obj[1].link = progs_link;
26 return 0;
27}
15 28
16static struct bpf_preload_ops umd_ops = {29static struct bpf_preload_ops ops = {
17 .info.driver_name = "bpf_preload",
18 .preload = preload,30 .preload = preload,
19 .finish = finish,
20 .owner = THIS_MODULE,31 .owner = THIS_MODULE,
21};32};
22 33
23static int preload(struct bpf_preload_info *obj)34static int load_skel(void)
24{35{
25 int magic = BPF_PRELOAD_START;36 int err;
26 loff_t pos = 0;
27 int i, err;
28 ssize_t n;
29 37
30 err = fork_usermode_driver(&umd_ops.info);38 skel = iterators_bpf__open();
39 if (!skel)
40 return -ENOMEM;
41 err = iterators_bpf__load(skel);
31 if (err)42 if (err)
32 return err;43 goto out;
33 44 err = iterators_bpf__attach(skel);
34 /* send the start magic to let UMD proceed with loading BPF progs */45 if (err)
35 n = kernel_write(umd_ops.info.pipe_to_umh,46 goto out;
36 &magic, sizeof(magic), &pos);47 maps_link = bpf_link_get_from_fd(skel->links.dump_bpf_map_fd);
37 if (n != sizeof(magic))48 if (IS_ERR(maps_link)) {
38 return -EPIPE;49 err = PTR_ERR(maps_link);
39 50 goto out;
40 /* receive bpf_link IDs and names from UMD */
41 pos = 0;
42 for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
43 n = kernel_read(umd_ops.info.pipe_from_umh,
44 &obj[i], sizeof(*obj), &pos);
45 if (n != sizeof(*obj))
46 return -EPIPE;
47 }51 }
48 return 0;52 progs_link = bpf_link_get_from_fd(skel->links.dump_bpf_prog_fd);
49}53 if (IS_ERR(progs_link)) {
50 54 err = PTR_ERR(progs_link);
51static int finish(void)55 goto out;
52{
53 int magic = BPF_PRELOAD_END;
54 struct pid *tgid;
55 loff_t pos = 0;
56 ssize_t n;
57
58 /* send the last magic to UMD. It will do a normal exit. */
59 n = kernel_write(umd_ops.info.pipe_to_umh,
60 &magic, sizeof(magic), &pos);
61 if (n != sizeof(magic))
62 return -EPIPE;
63
64 tgid = umd_ops.info.tgid;
65 if (tgid) {
66 wait_event(tgid->wait_pidfd, thread_group_exited(tgid));
67 umd_cleanup_helper(&umd_ops.info);
68 }56 }
69 return 0;57 return 0;
58out:
59 free_links_and_skel();
60 return err;
70}61}
71 62
72static int __init load_umd(void)63static int __init load(void)
73{64{
74 int err;65 int err;
75 66
76 err = umd_load_blob(&umd_ops.info, &bpf_preload_umd_start,67 err = load_skel();
77 &bpf_preload_umd_end - &bpf_preload_umd_start);
78 if (err)68 if (err)
79 return err;69 return err;
80 bpf_preload_ops = &umd_ops;70 bpf_preload_ops = &ops;
81 return err;71 return err;
82}72}
83 73
84static void __exit fini_umd(void)74static void __exit fini(void)
85{75{
86 struct pid *tgid;
87
88 bpf_preload_ops = NULL;76 bpf_preload_ops = NULL;
89 77 free_links_and_skel();
90 /* kill UMD in case it's still there due to earlier error */
91 tgid = umd_ops.info.tgid;
92 if (tgid) {
93 kill_pid(tgid, SIGKILL, 1);
94
95 wait_event(tgid->wait_pidfd, thread_group_exited(tgid));
96 umd_cleanup_helper(&umd_ops.info);
97 }
98 umd_unload_blob(&umd_ops.info);
99}78}
100late_initcall(load_umd);79late_initcall(load);
101module_exit(fini_umd);80module_exit(fini);
102MODULE_LICENSE("GPL");81MODULE_LICENSE("GPL");
kernel/bpf/preload/bpf_preload_umd_blob.S+0−7
@@ -1,7 +0,0 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2 .section .init.rodata, "a"
3 .global bpf_preload_umd_start
4bpf_preload_umd_start:
5 .incbin "kernel/bpf/preload/bpf_preload_umd"
6 .global bpf_preload_umd_end
7bpf_preload_umd_end:
kernel/bpf/preload/iterators/bpf_preload_common.h+0−13
@@ -1,13 +0,0 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _BPF_PRELOAD_COMMON_H
3#define _BPF_PRELOAD_COMMON_H
4
5#define BPF_PRELOAD_START 0x5555
6#define BPF_PRELOAD_END 0xAAAA
7
8struct bpf_preload_info {
9 char link_name[16];
10 int link_id;
11};
12
13#endif
kernel/bpf/preload/iterators/iterators.c+0−108
@@ -1,108 +0,0 @@
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright (c) 2020 Facebook */
3#include <errno.h>
4#include <stdio.h>
5#include <stdlib.h>
6#include <string.h>
7#include <unistd.h>
8#include <fcntl.h>
9#include <sys/resource.h>
10#include <bpf/libbpf.h>
11#include <bpf/bpf.h>
12#include <sys/mount.h>
13#include "iterators.lskel.h"
14#include "bpf_preload_common.h"
15
16int to_kernel = -1;
17int from_kernel = 0;
18
19static int __bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len)
20{
21 union bpf_attr attr;
22 int err;
23
24 memset(&attr, 0, sizeof(attr));
25 attr.info.bpf_fd = bpf_fd;
26 attr.info.info_len = *info_len;
27 attr.info.info = (long) info;
28
29 err = skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr));
30 if (!err)
31 *info_len = attr.info.info_len;
32 return err;
33}
34
35static int send_link_to_kernel(int link_fd, const char *link_name)
36{
37 struct bpf_preload_info obj = {};
38 struct bpf_link_info info = {};
39 __u32 info_len = sizeof(info);
40 int err;
41
42 err = __bpf_obj_get_info_by_fd(link_fd, &info, &info_len);
43 if (err)
44 return err;
45 obj.link_id = info.id;
46 if (strlen(link_name) >= sizeof(obj.link_name))
47 return -E2BIG;
48 strcpy(obj.link_name, link_name);
49 if (write(to_kernel, &obj, sizeof(obj)) != sizeof(obj))
50 return -EPIPE;
51 return 0;
52}
53
54int main(int argc, char **argv)
55{
56 struct iterators_bpf *skel;
57 int err, magic;
58 int debug_fd;
59
60 debug_fd = open("/dev/console", O_WRONLY | O_NOCTTY | O_CLOEXEC);
61 if (debug_fd < 0)
62 return 1;
63 to_kernel = dup(1);
64 close(1);
65 dup(debug_fd);
66 /* now stdin and stderr point to /dev/console */
67
68 read(from_kernel, &magic, sizeof(magic));
69 if (magic != BPF_PRELOAD_START) {
70 printf("bad start magic %d\n", magic);
71 return 1;
72 }
73 /* libbpf opens BPF object and loads it into the kernel */
74 skel = iterators_bpf__open_and_load();
75 if (!skel) {
76 /* iterators.skel.h is little endian.
77 * libbpf doesn't support automatic little->big conversion
78 * of BPF bytecode yet.
79 * The program load will fail in such case.
80 */
81 printf("Failed load could be due to wrong endianness\n");
82 return 1;
83 }
84 err = iterators_bpf__attach(skel);
85 if (err)
86 goto cleanup;
87
88 /* send two bpf_link IDs with names to the kernel */
89 err = send_link_to_kernel(skel->links.dump_bpf_map_fd, "maps.debug");
90 if (err)
91 goto cleanup;
92 err = send_link_to_kernel(skel->links.dump_bpf_prog_fd, "progs.debug");
93 if (err)
94 goto cleanup;
95
96 /* The kernel will proceed with pinnging the links in bpffs.
97 * UMD will wait on read from pipe.
98 */
99 read(from_kernel, &magic, sizeof(magic));
100 if (magic != BPF_PRELOAD_END) {
101 printf("bad final magic %d\n", magic);
102 err = -EINVAL;
103 }
104cleanup:
105 iterators_bpf__destroy(skel);
106
107 return err != 0;
108}
kernel/bpf/syscall.c+2−0
@@ -2490,6 +2490,7 @@ void bpf_link_put(struct bpf_link *link) · 2489 unmodified lines
2490 bpf_link_free(link);2490 bpf_link_free(link);
2491 }2491 }
2492}2492}
2493EXPORT_SYMBOL(bpf_link_put);
2493 2494
2494static int bpf_link_release(struct inode *inode, struct file *filp)2495static int bpf_link_release(struct inode *inode, struct file *filp)
2495{2496{
@@ -2632,6 +2633,7 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd) · 136 unmodified lines
2632 2633
2633 return link;2634 return link;
2634}2635}
2636EXPORT_SYMBOL(bpf_link_get_from_fd);
2635 2637
2636struct bpf_tracing_link {2638struct bpf_tracing_link {
2637 struct bpf_link link;2639 struct bpf_link link;