bpf: Convert bpf_preload.ko to use light skeleton.

The main change is a move of the single line #include "iterators.lskel.h" from iterators/iterators.c to bpf_preload_kern.c. Which means that generated light skeleton can be used from user space or user mode driver like iterators.c or from the kernel module or the kernel itself. The direct use of light skeleton from the kernel module simplifies the code, since UMD is no longer necessary. The libbpf.a required user space and UMD. The CO-RE in the kernel and generated "loader bpf program" used by the light skeleton are capable to perform complex loading operations traditionally provided by libbpf. In addition UMD approach was launching UMD process every time bpffs has to be mounted. With light skeleton in the kernel the bpf_preload kernel module loads bpf iterators once and pins them multiple times into different bpffs mounts.

Alexei Starovoitov committed 4 years ago

Commit cb80ddc1 parent d7beb3d9 files · +70 −247Browse files

Showing the changes to kernel (9 of 9 files). Show all

Unified · Split

9 files

kernel/bpf/inode.c +10 −29
kernel/bpf/preload/Kconfig +3 −4
kernel/bpf/preload/Makefile +2 −12
kernel/bpf/preload/bpf_preload.h +4 −4
kernel/bpf/preload/bpf_preload_kern.c +49 −70
kernel/bpf/preload/bpf_preload_umd_blob.S +0 −7
kernel/bpf/preload/iterators/bpf_preload_common.h +0 −13
kernel/bpf/preload/iterators/iterators.c +0 −108
kernel/bpf/syscall.c +2 −0

kernel/bpf/inode.c+10−29

@@ -710,11 +710,10 @@ static DEFINE_MUTEX(bpf_preload_lock); · 709 unmodified lines
710	710	static int populate_bpffs(struct dentry *parent)
711	711	{
712	712	struct bpf_preload_info objs[BPF_PRELOAD_LINKS] = {};
713		− struct bpf_link *links[BPF_PRELOAD_LINKS] = {};
714	713	int err = 0, i;
715	714
716	715	/* grab the mutex to make sure the kernel interactions with bpf_preload
717		− * UMD are serialized
	716	+ * are serialized
718	717	*/
719	718	mutex_lock(&bpf_preload_lock);
720	719
@@ -722,40 +721,22 @@ static int populate_bpffs(struct dentry *parent) · 1 unmodified line
722	721	if (!bpf_preload_mod_get())
723	722	goto out;
724	723
725		− if (!bpf_preload_ops->info.tgid) {
726		− /* preload() will start UMD that will load BPF iterator programs */
727		− err = bpf_preload_ops->preload(objs);
728		− if (err)
	724	+ err = bpf_preload_ops->preload(objs);
	725	+ if (err)
	726	+ goto out_put;
	727	+ for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
	728	+ bpf_link_inc(objs[i].link);
	729	+ err = bpf_iter_link_pin_kernel(parent,
	730	+ objs[i].link_name, objs[i].link);
	731	+ if (err) {
	732	+ bpf_link_put(objs[i].link);
729	733	goto out_put;
730		− for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
731		− links[i] = bpf_link_by_id(objs[i].link_id);
732		− if (IS_ERR(links[i])) {
733		− err = PTR_ERR(links[i]);
734		− goto out_put;
735		− }
736	734	}
737		− for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
738		− err = bpf_iter_link_pin_kernel(parent,
739		− objs[i].link_name, links[i]);
740		− if (err)
741		− goto out_put;
742		− /* do not unlink successfully pinned links even
743		− * if later link fails to pin
744		− */
745		− links[i] = NULL;
746		− }
747		− /* finish() will tell UMD process to exit */
748		− err = bpf_preload_ops->finish();
749		− if (err)
750		− goto out_put;
751	735	}
752	736	out_put:
753	737	bpf_preload_mod_put();
754	738	out:
755	739	mutex_unlock(&bpf_preload_lock);
756		− for (i = 0; i < BPF_PRELOAD_LINKS && err; i++)
757		− if (!IS_ERR_OR_NULL(links[i]))
758		− bpf_link_put(links[i]);
759	740	return err;
760	741	}
761	742

kernel/bpf/preload/Kconfig+3−4

@@ -18,10 +18,9 @@ menuconfig BPF_PRELOAD · 17 unmodified lines
18	18
19	19	if BPF_PRELOAD
20	20	config BPF_PRELOAD_UMD
21		− tristate "bpf_preload kernel module with user mode driver"
22		− depends on CC_CAN_LINK
23		− depends on m \|\| CC_CAN_LINK_STATIC
	21	+ tristate "bpf_preload kernel module"
24	22	default m
25	23	help
26		− This builds bpf_preload kernel module with embedded user mode driver.
	24	+ This builds bpf_preload kernel module with embedded BPF programs for
	25	+ introspection in bpffs.
27	26	endif

kernel/bpf/preload/Makefile+2−12

@@ -3,16 +3,6 @@ · 2 unmodified lines
3	3	LIBBPF_SRCS = $(srctree)/tools/lib/bpf/
4	4	LIBBPF_INCLUDE = $(LIBBPF_SRCS)/..
5	5
6		−userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi \
7		− -I $(LIBBPF_INCLUDE) -Wno-unused-result
8		−
9		−userprogs := bpf_preload_umd
10		−
11		−bpf_preload_umd-objs := iterators/iterators.o
12		−
13		−$(obj)/bpf_preload_umd:
14		−
15		−$(obj)/bpf_preload_umd_blob.o: $(obj)/bpf_preload_umd
16		−
17	6	obj-$(CONFIG_BPF_PRELOAD_UMD) += bpf_preload.o
18		−bpf_preload-objs += bpf_preload_kern.o bpf_preload_umd_blob.o
	7	+CFLAGS_bpf_preload_kern.o += -I $(LIBBPF_INCLUDE)
	8	+bpf_preload-objs += bpf_preload_kern.o

kernel/bpf/preload/bpf_preload.h+4−4

@@ -2,13 +2,13 @@ · 1 unmodified line
2	2	#ifndef _BPF_PRELOAD_H
3	3	#define _BPF_PRELOAD_H
4	4
5		−#include <linux/usermode_driver.h>
6		−#include "iterators/bpf_preload_common.h"
	5	+struct bpf_preload_info {
	6	+ char link_name[16];
	7	+ struct bpf_link *link;
	8	+};
7	9
8	10	struct bpf_preload_ops {
9		− struct umd_info info;
10	11	int (preload)(struct bpf_preload_info );
11		− int (*finish)(void);
12	12	struct module *owner;
13	13	};
14	14	extern struct bpf_preload_ops *bpf_preload_ops;

kernel/bpf/preload/bpf_preload_kern.c+49−70

@@ -2,101 +2,80 @@ · 1 unmodified line
2	2	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
3	3	#include <linux/init.h>
4	4	#include <linux/module.h>
5		−#include <linux/pid.h>
6		−#include <linux/fs.h>
7		−#include <linux/sched/signal.h>
8	5	#include "bpf_preload.h"
	6	+#include "iterators/iterators.lskel.h"
9	7
10		−extern char bpf_preload_umd_start;
11		−extern char bpf_preload_umd_end;
	8	+static struct bpf_link maps_link, progs_link;
	9	+static struct iterators_bpf *skel;
12	10
13		−static int preload(struct bpf_preload_info *obj);
14		−static int finish(void);
	11	+static void free_links_and_skel(void)
	12	+{
	13	+ if (!IS_ERR_OR_NULL(maps_link))
	14	+ bpf_link_put(maps_link);
	15	+ if (!IS_ERR_OR_NULL(progs_link))
	16	+ bpf_link_put(progs_link);
	17	+ iterators_bpf__destroy(skel);
	18	+}
	19	+
	20	+static int preload(struct bpf_preload_info *obj)
	21	+{
	22	+ strlcpy(obj[0].link_name, "maps.debug", sizeof(obj[0].link_name));
	23	+ obj[0].link = maps_link;
	24	+ strlcpy(obj[1].link_name, "progs.debug", sizeof(obj[1].link_name));
	25	+ obj[1].link = progs_link;
	26	+ return 0;
	27	+}
15	28
16		−static struct bpf_preload_ops umd_ops = {
17		− .info.driver_name = "bpf_preload",
	29	+static struct bpf_preload_ops ops = {
18	30	.preload = preload,
19		− .finish = finish,
20	31	.owner = THIS_MODULE,
21	32	};
22	33
23		−static int preload(struct bpf_preload_info *obj)
	34	+static int load_skel(void)
24	35	{
25		− int magic = BPF_PRELOAD_START;
26		− loff_t pos = 0;
27		− int i, err;
28		− ssize_t n;
	36	+ int err;
29	37
30		− err = fork_usermode_driver(&umd_ops.info);
	38	+ skel = iterators_bpf__open();
	39	+ if (!skel)
	40	+ return -ENOMEM;
	41	+ err = iterators_bpf__load(skel);
31	42	if (err)
32		− return err;
33		−
34		− /* send the start magic to let UMD proceed with loading BPF progs */
35		− n = kernel_write(umd_ops.info.pipe_to_umh,
36		− &magic, sizeof(magic), &pos);
37		− if (n != sizeof(magic))
38		− return -EPIPE;
39		−
40		− /* receive bpf_link IDs and names from UMD */
41		− pos = 0;
42		− for (i = 0; i < BPF_PRELOAD_LINKS; i++) {
43		− n = kernel_read(umd_ops.info.pipe_from_umh,
44		− &obj[i], sizeof(*obj), &pos);
45		− if (n != sizeof(*obj))
46		− return -EPIPE;
	43	+ goto out;
	44	+ err = iterators_bpf__attach(skel);
	45	+ if (err)
	46	+ goto out;
	47	+ maps_link = bpf_link_get_from_fd(skel->links.dump_bpf_map_fd);
	48	+ if (IS_ERR(maps_link)) {
	49	+ err = PTR_ERR(maps_link);
	50	+ goto out;
47	51	}
48		− return 0;
49		−}
50		−
51		−static int finish(void)
52		−{
53		− int magic = BPF_PRELOAD_END;
54		− struct pid *tgid;
55		− loff_t pos = 0;
56		− ssize_t n;
57		−
58		− /* send the last magic to UMD. It will do a normal exit. */
59		− n = kernel_write(umd_ops.info.pipe_to_umh,
60		− &magic, sizeof(magic), &pos);
61		− if (n != sizeof(magic))
62		− return -EPIPE;
63		−
64		− tgid = umd_ops.info.tgid;
65		− if (tgid) {
66		− wait_event(tgid->wait_pidfd, thread_group_exited(tgid));
67		− umd_cleanup_helper(&umd_ops.info);
	52	+ progs_link = bpf_link_get_from_fd(skel->links.dump_bpf_prog_fd);
	53	+ if (IS_ERR(progs_link)) {
	54	+ err = PTR_ERR(progs_link);
	55	+ goto out;
68	56	}
69	57	return 0;
	58	+out:
	59	+ free_links_and_skel();
	60	+ return err;
70	61	}
71	62
72		−static int __init load_umd(void)
	63	+static int __init load(void)
73	64	{
74	65	int err;
75	66
76		− err = umd_load_blob(&umd_ops.info, &bpf_preload_umd_start,
77		− &bpf_preload_umd_end - &bpf_preload_umd_start);
	67	+ err = load_skel();
78	68	if (err)
79	69	return err;
80		− bpf_preload_ops = &umd_ops;
	70	+ bpf_preload_ops = &ops;
81	71	return err;
82	72	}
83	73
84		−static void __exit fini_umd(void)
	74	+static void __exit fini(void)
85	75	{
86		− struct pid *tgid;
87		−
88	76	bpf_preload_ops = NULL;
89		−
90		− /* kill UMD in case it's still there due to earlier error */
91		− tgid = umd_ops.info.tgid;
92		− if (tgid) {
93		− kill_pid(tgid, SIGKILL, 1);
94		−
95		− wait_event(tgid->wait_pidfd, thread_group_exited(tgid));
96		− umd_cleanup_helper(&umd_ops.info);
97		− }
98		− umd_unload_blob(&umd_ops.info);
	77	+ free_links_and_skel();
99	78	}
100		−late_initcall(load_umd);
101		−module_exit(fini_umd);
	79	+late_initcall(load);
	80	+module_exit(fini);
102	81	MODULE_LICENSE("GPL");

kernel/bpf/preload/bpf_preload_umd_blob.S+0−7

@@ -1,7 +0,0 @@
1		−/* SPDX-License-Identifier: GPL-2.0 */
2		− .section .init.rodata, "a"
3		− .global bpf_preload_umd_start
4		−bpf_preload_umd_start:
5		− .incbin "kernel/bpf/preload/bpf_preload_umd"
6		− .global bpf_preload_umd_end
7		−bpf_preload_umd_end:

kernel/bpf/preload/iterators/bpf_preload_common.h+0−13

@@ -1,13 +0,0 @@
1		−/* SPDX-License-Identifier: GPL-2.0 */
2		−#ifndef _BPF_PRELOAD_COMMON_H
3		−#define _BPF_PRELOAD_COMMON_H
4		−
5		−#define BPF_PRELOAD_START 0x5555
6		−#define BPF_PRELOAD_END 0xAAAA
7		−
8		−struct bpf_preload_info {
9		− char link_name[16];
10		− int link_id;
11		−};
12		−
13		−#endif

kernel/bpf/preload/iterators/iterators.c+0−108

@@ -1,108 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0
2		−/* Copyright (c) 2020 Facebook */
3		−#include <errno.h>
4		−#include <stdio.h>
5		−#include <stdlib.h>
6		−#include <string.h>
7		−#include <unistd.h>
8		−#include <fcntl.h>
9		−#include <sys/resource.h>
10		−#include <bpf/libbpf.h>
11		−#include <bpf/bpf.h>
12		−#include <sys/mount.h>
13		−#include "iterators.lskel.h"
14		−#include "bpf_preload_common.h"
15		−
16		−int to_kernel = -1;
17		−int from_kernel = 0;
18		−
19		−static int __bpf_obj_get_info_by_fd(int bpf_fd, void info, __u32 info_len)
20		−{
21		− union bpf_attr attr;
22		− int err;
23		−
24		− memset(&attr, 0, sizeof(attr));
25		− attr.info.bpf_fd = bpf_fd;
26		− attr.info.info_len = *info_len;
27		− attr.info.info = (long) info;
28		−
29		− err = skel_sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr));
30		− if (!err)
31		− *info_len = attr.info.info_len;
32		− return err;
33		−}
34		−
35		−static int send_link_to_kernel(int link_fd, const char *link_name)
36		−{
37		− struct bpf_preload_info obj = {};
38		− struct bpf_link_info info = {};
39		− __u32 info_len = sizeof(info);
40		− int err;
41		−
42		− err = __bpf_obj_get_info_by_fd(link_fd, &info, &info_len);
43		− if (err)
44		− return err;
45		− obj.link_id = info.id;
46		− if (strlen(link_name) >= sizeof(obj.link_name))
47		− return -E2BIG;
48		− strcpy(obj.link_name, link_name);
49		− if (write(to_kernel, &obj, sizeof(obj)) != sizeof(obj))
50		− return -EPIPE;
51		− return 0;
52		−}
53		−
54		−int main(int argc, char **argv)
55		−{
56		− struct iterators_bpf *skel;
57		− int err, magic;
58		− int debug_fd;
59		−
60		− debug_fd = open("/dev/console", O_WRONLY \| O_NOCTTY \| O_CLOEXEC);
61		− if (debug_fd < 0)
62		− return 1;
63		− to_kernel = dup(1);
64		− close(1);
65		− dup(debug_fd);
66		− /* now stdin and stderr point to /dev/console */
67		−
68		− read(from_kernel, &magic, sizeof(magic));
69		− if (magic != BPF_PRELOAD_START) {
70		− printf("bad start magic %d\n", magic);
71		− return 1;
72		− }
73		− /* libbpf opens BPF object and loads it into the kernel */
74		− skel = iterators_bpf__open_and_load();
75		− if (!skel) {
76		− /* iterators.skel.h is little endian.
77		− * libbpf doesn't support automatic little->big conversion
78		− * of BPF bytecode yet.
79		− * The program load will fail in such case.
80		− */
81		− printf("Failed load could be due to wrong endianness\n");
82		− return 1;
83		− }
84		− err = iterators_bpf__attach(skel);
85		− if (err)
86		− goto cleanup;
87		−
88		− /* send two bpf_link IDs with names to the kernel */
89		− err = send_link_to_kernel(skel->links.dump_bpf_map_fd, "maps.debug");
90		− if (err)
91		− goto cleanup;
92		− err = send_link_to_kernel(skel->links.dump_bpf_prog_fd, "progs.debug");
93		− if (err)
94		− goto cleanup;
95		−
96		− /* The kernel will proceed with pinnging the links in bpffs.
97		− * UMD will wait on read from pipe.
98		− */
99		− read(from_kernel, &magic, sizeof(magic));
100		− if (magic != BPF_PRELOAD_END) {
101		− printf("bad final magic %d\n", magic);
102		− err = -EINVAL;
103		− }
104		−cleanup:
105		− iterators_bpf__destroy(skel);
106		−
107		− return err != 0;
108		−}

kernel/bpf/syscall.c+2−0

@@ -2490,6 +2490,7 @@ void bpf_link_put(struct bpf_link *link) · 2489 unmodified lines
2490	2490	bpf_link_free(link);
2491	2491	}
2492	2492	}
	2493	+EXPORT_SYMBOL(bpf_link_put);
2493	2494
2494	2495	static int bpf_link_release(struct inode inode, struct file filp)
2495	2496	{
@@ -2632,6 +2633,7 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd) · 136 unmodified lines
2632	2633
2633	2634	return link;
2634	2635	}
	2636	+EXPORT_SYMBOL(bpf_link_get_from_fd);
2635	2637
2636	2638	struct bpf_tracing_link {
2637	2639	struct bpf_link link;