Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Daniel Borkmann says: ==================== pull-request: bpf-next 2019-10-27 The following pull-request contains BPF updates for your *net-next* tree. We've added 52 non-merge commits during the last 11 day(s) which contain a total of 65 files changed, 2604 insertions(+), 1100 deletions(-). The main changes are: 1) Revolutionize BPF tracing by using in-kernel BTF to type check BPF assembly code. The work here teaches BPF verifier to recognize kfree_skb()'s first argument as 'struct sk_buff *' in tracepoints such that verifier allows direct use of bpf_skb_event_output() helper used in tc BPF et al (w/o probing memory access) that dumps skb data into perf ring buffer. Also add direct loads to probe memory in order to speed up/replace bpf_probe_read() calls, from Alexei Starovoitov. 2) Big batch of changes to improve libbpf and BPF kselftests. Besides others: generalization of libbpf's CO-RE relocation support to now also include field existence relocations, revamp the BPF kselftest Makefile to add test runner concept allowing to exercise various ways to build BPF programs, and teach bpf_object__open() and friends to automatically derive BPF program type/expected attach type from section names to ease their use, from Andrii Nakryiko. 3) Fix deadlock in stackmap's build-id lookup on rq_lock(), from Song Liu. 4) Allow to read BTF as raw data from bpftool. Most notable use case is to dump /sys/kernel/btf/vmlinux through this, from Jiri Olsa. 5) Use bpf_redirect_map() helper in libbpf's AF_XDP helper prog which manages to improve "rx_drop" performance by ~4%., from Björn Töpel. 6) Fix to restore the flow dissector after reattach BPF test and also fix error handling in bpf_helper_defs.h generation, from Jakub Sitnicki. 7) Improve verifier's BTF ctx access for use outside of raw_tp, from Martin KaFai Lau. 8) Improve documentation for AF_XDP with new sections and to reflect latest features, from Magnus Karlsson. 9) Add back 'version' section parsing to libbpf for old kernels, from John Fastabend. 10) Fix strncat bounds error in libbpf's libbpf_prog_type_by_name(), from KP Singh. 11) Turn on -mattr=+alu32 in LLVM by default for BPF kselftests in order to improve insn coverage for built BPF progs, from Yonghong Song. 12) Misc minor cleanups and fixes, from various others. ====================

David S. Miller committed 7 years ago

Commit 5b7fe932 parents b951248 027cbaa65 files · +2604 −1100Browse files

Unified · Split

65 files

Documentation/networking/af_xdp.rst +231 −28
arch/x86/net/bpf_jit_comp.c +93 −4
include/linux/bpf.h +38 −6
include/linux/bpf_verifier.h +7 −1
include/linux/btf.h +32 −0
include/linux/extable.h +10 −0
include/linux/filter.h +5 −1
include/trace/bpf_probe.h +2 −1
include/trace/events/xdp.h +1 −1
include/uapi/linux/bpf.h +27 −1
kernel/bpf/btf.c +301 −21
kernel/bpf/core.c +38 −1
kernel/bpf/stackmap.c +4 −3
kernel/bpf/syscall.c +64 −27
kernel/bpf/verifier.c +193 −18
kernel/extable.c +2 −0
kernel/trace/bpf_trace.c +9 −1
net/bpf/test_run.c +9 −0
net/core/filter.c +14 −1
scripts/bpf_helpers_doc.py +2 −2
tools/bpf/bpftool/btf.c +56 −1
tools/bpf/bpftool/prog.c +4 −4
tools/include/uapi/linux/bpf.h +27 −1
tools/include/uapi/linux/if_link.h +2 −0
tools/lib/bpf/Makefile +3 −0
tools/lib/bpf/bpf.c +3 −0
tools/lib/bpf/bpf_core_read.h +23 −1
tools/lib/bpf/btf.c +36 −36
tools/lib/bpf/btf.h +2 −2
tools/lib/bpf/btf_dump.c +9 −9
tools/lib/bpf/libbpf.c +541 −440
tools/lib/bpf/libbpf.h +20 −8
tools/lib/bpf/libbpf.map +2 −0
tools/lib/bpf/libbpf_internal.h +22 −11
tools/lib/bpf/xsk.c +34 −12
tools/testing/selftests/bpf/.gitignore +3 −3
tools/testing/selftests/bpf/Makefile +190 −161
tools/testing/selftests/bpf/prog_tests/attach_probe.c +1 −6
tools/testing/selftests/bpf/prog_tests/core_reloc.c +76 −5
tools/testing/selftests/bpf/prog_tests/flow_dissector_reattach.c +17 −4
tools/testing/selftests/bpf/prog_tests/kfree_skb.c +89 −0
tools/testing/selftests/bpf/prog_tests/rdonly_maps.c +0 −4
tools/testing/selftests/bpf/prog_tests/reference_tracking.c +3 −2
tools/testing/selftests/bpf/prog_tests/section_names.c +30 −60
tools/testing/selftests/bpf/prog_tests/skb_ctx.c +5 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence.c +3 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_kind.c +3 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_value_type.c +3 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_kind.c +3 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_sz.c +3 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_type.c +3 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_struct_type.c +3 −0
tools/testing/selftests/bpf/progs/btf__core_reloc_existence___minimal.c +3 −0
tools/testing/selftests/bpf/progs/core_reloc_types.h +57 −1
tools/testing/selftests/bpf/progs/kfree_skb.c +103 −0
tools/testing/selftests/bpf/progs/test_core_reloc_existence.c +79 −0
tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c +2 −1
tools/testing/selftests/bpf/progs/test_queue_stack_map.h +0 −0
tools/testing/selftests/bpf/progs/test_sk_lookup_kern.c +9 −9
tools/testing/selftests/bpf/progs/test_skb_ctx.c +1 −0
tools/testing/selftests/bpf/test_libbpf.sh +0 −43
tools/testing/selftests/bpf/test_libbpf_open.c +0 −144
tools/testing/selftests/bpf/test_maps.c +4 −8
tools/testing/selftests/bpf/test_offload.py +1 −1
tools/testing/selftests/bpf/test_progs.c +44 −6

Documentation/networking/af_xdp.rst+231−28

@@ -40,13 +40,13 @@ allocates memory for this UMEM using whatever means it feels is most · 39 unmodified lines
40	appropriate (malloc, mmap, huge pages, etc). This memory area is then	40	appropriate (malloc, mmap, huge pages, etc). This memory area is then
41	registered with the kernel using the new setsockopt XDP_UMEM_REG. The	41	registered with the kernel using the new setsockopt XDP_UMEM_REG. The
42	UMEM also has two rings: the FILL ring and the COMPLETION ring. The	42	UMEM also has two rings: the FILL ring and the COMPLETION ring. The
43	fill ring is used by the application to send down addr for the kernel	43	FILL ring is used by the application to send down addr for the kernel
44	to fill in with RX packet data. References to these frames will then	44	to fill in with RX packet data. References to these frames will then
45	appear in the RX ring once each packet has been received. The	45	appear in the RX ring once each packet has been received. The
46	completion ring, on the other hand, contains frame addr that the	46	COMPLETION ring, on the other hand, contains frame addr that the
47	kernel has transmitted completely and can now be used again by user	47	kernel has transmitted completely and can now be used again by user
48	space, for either TX or RX. Thus, the frame addrs appearing in the	48	space, for either TX or RX. Thus, the frame addrs appearing in the
49	completion ring are addrs that were previously transmitted using the	49	COMPLETION ring are addrs that were previously transmitted using the
50	TX ring. In summary, the RX and FILL rings are used for the RX path	50	TX ring. In summary, the RX and FILL rings are used for the RX path
51	and the TX and COMPLETION rings are used for the TX path.	51	and the TX and COMPLETION rings are used for the TX path.
52		52
@@ -91,11 +91,16 @@ Concepts · 38 unmodified lines
91	========	91	========
92		92
93	In order to use an AF_XDP socket, a number of associated objects need	93	In order to use an AF_XDP socket, a number of associated objects need
94	to be setup.	94	to be setup. These objects and their options are explained in the
		95	following sections.
95		96
96	Jonathan Corbet has also written an excellent article on LWN,	97	For an overview on how AF_XDP works, you can also take a look at the
97	"Accelerating networking with AF_XDP". It can be found at	98	Linux Plumbers paper from 2018 on the subject:
98	https://lwn.net/Articles/750845/.	99	http://vger.kernel.org/lpc_net2018_talks/lpc18_paper_af_xdp_perf-v2.pdf. Do
		100	NOT consult the paper from 2017 on "AF_PACKET v4", the first attempt
		101	at AF_XDP. Nearly everything changed since then. Jonathan Corbet has
		102	also written an excellent article on LWN, "Accelerating networking
		103	with AF_XDP". It can be found at https://lwn.net/Articles/750845/.
99		104
100	UMEM	105	UMEM
101	----	106	----
@@ -113,22 +118,22 @@ the next socket B can do this by setting the XDP_SHARED_UMEM flag in · 11 unmodified lines
113	struct sockaddr_xdp member sxdp_flags, and passing the file descriptor	118	struct sockaddr_xdp member sxdp_flags, and passing the file descriptor
114	of A to struct sockaddr_xdp member sxdp_shared_umem_fd.	119	of A to struct sockaddr_xdp member sxdp_shared_umem_fd.
115		120
116	The UMEM has two single-producer/single-consumer rings, that are used	121	The UMEM has two single-producer/single-consumer rings that are used
117	to transfer ownership of UMEM frames between the kernel and the	122	to transfer ownership of UMEM frames between the kernel and the
118	user-space application.	123	user-space application.
119		124
120	Rings	125	Rings
121	-----	126	-----
122		127
123	There are a four different kind of rings: Fill, Completion, RX and	128	There are a four different kind of rings: FILL, COMPLETION, RX and
124	TX. All rings are single-producer/single-consumer, so the user-space	129	TX. All rings are single-producer/single-consumer, so the user-space
125	application need explicit synchronization of multiple	130	application need explicit synchronization of multiple
126	processes/threads are reading/writing to them.	131	processes/threads are reading/writing to them.
127		132
128	The UMEM uses two rings: Fill and Completion. Each socket associated	133	The UMEM uses two rings: FILL and COMPLETION. Each socket associated
129	with the UMEM must have an RX queue, TX queue or both. Say, that there	134	with the UMEM must have an RX queue, TX queue or both. Say, that there
130	is a setup with four sockets (all doing TX and RX). Then there will be	135	is a setup with four sockets (all doing TX and RX). Then there will be
131	one Fill ring, one Completion ring, four TX rings and four RX rings.	136	one FILL ring, one COMPLETION ring, four TX rings and four RX rings.
132		137
133	The rings are head(producer)/tail(consumer) based rings. A producer	138	The rings are head(producer)/tail(consumer) based rings. A producer
134	writes the data ring at the index pointed out by struct xdp_ring	139	writes the data ring at the index pointed out by struct xdp_ring
@@ -146,7 +151,7 @@ The size of the rings need to be of size power of two. · 11 unmodified lines
146	UMEM Fill Ring	151	UMEM Fill Ring
147	~~~~~~~~~~~~~~	152	~~~~~~~~~~~~~~
148		153
149	The Fill ring is used to transfer ownership of UMEM frames from	154	The FILL ring is used to transfer ownership of UMEM frames from
150	user-space to kernel-space. The UMEM addrs are passed in the ring. As	155	user-space to kernel-space. The UMEM addrs are passed in the ring. As
151	an example, if the UMEM is 64k and each chunk is 4k, then the UMEM has	156	an example, if the UMEM is 64k and each chunk is 4k, then the UMEM has
152	16 chunks and can pass addrs between 0 and 64k.	157	16 chunks and can pass addrs between 0 and 64k.
@@ -164,8 +169,8 @@ chunks mode, then the incoming addr will be left untouched. · 11 unmodified lines
164	UMEM Completion Ring	169	UMEM Completion Ring
165	~~~~~~~~~~~~~~~~~~~~	170	~~~~~~~~~~~~~~~~~~~~
166		171
167	The Completion Ring is used transfer ownership of UMEM frames from	172	The COMPLETION Ring is used transfer ownership of UMEM frames from
168	kernel-space to user-space. Just like the Fill ring, UMEM indicies are	173	kernel-space to user-space. Just like the FILL ring, UMEM indices are
169	used.	174	used.
170		175
171	Frames passed from the kernel to user-space are frames that has been	176	Frames passed from the kernel to user-space are frames that has been
@@ -181,7 +186,7 @@ The RX ring is the receiving side of a socket. Each entry in the ring · 9 unmodified lines
181	is a struct xdp_desc descriptor. The descriptor contains UMEM offset	186	is a struct xdp_desc descriptor. The descriptor contains UMEM offset
182	(addr) and the length of the data (len).	187	(addr) and the length of the data (len).
183		188
184	If no frames have been passed to kernel via the Fill ring, no	189	If no frames have been passed to kernel via the FILL ring, no
185	descriptors will (or can) appear on the RX ring.	190	descriptors will (or can) appear on the RX ring.
186		191
187	The user application consumes struct xdp_desc descriptors from this	192	The user application consumes struct xdp_desc descriptors from this
@@ -199,8 +204,24 @@ be relaxed in the future. · 11 unmodified lines
199	The user application produces struct xdp_desc descriptors to this	204	The user application produces struct xdp_desc descriptors to this
200	ring.	205	ring.
201		206
		207	Libbpf
		208	======
		209
		210	Libbpf is a helper library for eBPF and XDP that makes using these
		211	technologies a lot simpler. It also contains specific helper functions
		212	in tools/lib/bpf/xsk.h for facilitating the use of AF_XDP. It
		213	contains two types of functions: those that can be used to make the
		214	setup of AF_XDP socket easier and ones that can be used in the data
		215	plane to access the rings safely and quickly. To see an example on how
		216	to use this API, please take a look at the sample application in
		217	samples/bpf/xdpsock_usr.c which uses libbpf for both setup and data
		218	plane operations.
		219
		220	We recommend that you use this library unless you have become a power
		221	user. It will make your program a lot simpler.
		222
202	XSKMAP / BPF_MAP_TYPE_XSKMAP	223	XSKMAP / BPF_MAP_TYPE_XSKMAP
203	----------------------------	224	============================
204		225
205	On XDP side there is a BPF map type BPF_MAP_TYPE_XSKMAP (XSKMAP) that	226	On XDP side there is a BPF map type BPF_MAP_TYPE_XSKMAP (XSKMAP) that
206	is used in conjunction with bpf_redirect_map() to pass the ingress	227	is used in conjunction with bpf_redirect_map() to pass the ingress
@@ -216,21 +237,184 @@ queue 17. Only the XDP program executing for eth0 and queue 17 will · 9 unmodified lines
216	successfully pass data to the socket. Please refer to the sample	237	successfully pass data to the socket. Please refer to the sample
217	application (samples/bpf/) in for an example.	238	application (samples/bpf/) in for an example.
218		239
		240	Configuration Flags and Socket Options
		241	======================================
		242
		243	These are the various configuration flags that can be used to control
		244	and monitor the behavior of AF_XDP sockets.
		245
		246	XDP_COPY and XDP_ZERO_COPY bind flags
		247	-------------------------------------
		248
		249	When you bind to a socket, the kernel will first try to use zero-copy
		250	copy. If zero-copy is not supported, it will fall back on using copy
		251	mode, i.e. copying all packets out to user space. But if you would
		252	like to force a certain mode, you can use the following flags. If you
		253	pass the XDP_COPY flag to the bind call, the kernel will force the
		254	socket into copy mode. If it cannot use copy mode, the bind call will
		255	fail with an error. Conversely, the XDP_ZERO_COPY flag will force the
		256	socket into zero-copy mode or fail.
		257
		258	XDP_SHARED_UMEM bind flag
		259	-------------------------
		260
		261	This flag enables you to bind multiple sockets to the same UMEM, but
		262	only if they share the same queue id. In this mode, each socket has
		263	their own RX and TX rings, but the UMEM (tied to the fist socket
		264	created) only has a single FILL ring and a single COMPLETION
		265	ring. To use this mode, create the first socket and bind it in the normal
		266	way. Create a second socket and create an RX and a TX ring, or at
		267	least one of them, but no FILL or COMPLETION rings as the ones from
		268	the first socket will be used. In the bind call, set he
		269	XDP_SHARED_UMEM option and provide the initial socket's fd in the
		270	sxdp_shared_umem_fd field. You can attach an arbitrary number of extra
		271	sockets this way.
		272
		273	What socket will then a packet arrive on? This is decided by the XDP
		274	program. Put all the sockets in the XSK_MAP and just indicate which
		275	index in the array you would like to send each packet to. A simple
		276	round-robin example of distributing packets is shown below:
		277
		278	.. code-block:: c
		279
		280	#include <linux/bpf.h>
		281	#include "bpf_helpers.h"
		282
		283	#define MAX_SOCKS 16
		284
		285	struct {
		286	__uint(type, BPF_MAP_TYPE_XSKMAP);
		287	__uint(max_entries, MAX_SOCKS);
		288	__uint(key_size, sizeof(int));
		289	__uint(value_size, sizeof(int));
		290	} xsks_map SEC(".maps");
		291
		292	static unsigned int rr;
		293
		294	SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
		295	{
		296	rr = (rr + 1) & (MAX_SOCKS - 1);
		297
		298	return bpf_redirect_map(&xsks_map, rr, 0);
		299	}
		300
		301	Note, that since there is only a single set of FILL and COMPLETION
		302	rings, and they are single producer, single consumer rings, you need
		303	to make sure that multiple processes or threads do not use these rings
		304	concurrently. There are no synchronization primitives in the
		305	libbpf code that protects multiple users at this point in time.
		306
		307	XDP_USE_NEED_WAKEUP bind flag
		308	-----------------------------
		309
		310	This option adds support for a new flag called need_wakeup that is
		311	present in the FILL ring and the TX ring, the rings for which user
		312	space is a producer. When this option is set in the bind call, the
		313	need_wakeup flag will be set if the kernel needs to be explicitly
		314	woken up by a syscall to continue processing packets. If the flag is
		315	zero, no syscall is needed.
		316
		317	If the flag is set on the FILL ring, the application needs to call
		318	poll() to be able to continue to receive packets on the RX ring. This
		319	can happen, for example, when the kernel has detected that there are no
		320	more buffers on the FILL ring and no buffers left on the RX HW ring of
		321	the NIC. In this case, interrupts are turned off as the NIC cannot
		322	receive any packets (as there are no buffers to put them in), and the
		323	need_wakeup flag is set so that user space can put buffers on the
		324	FILL ring and then call poll() so that the kernel driver can put these
		325	buffers on the HW ring and start to receive packets.
		326
		327	If the flag is set for the TX ring, it means that the application
		328	needs to explicitly notify the kernel to send any packets put on the
		329	TX ring. This can be accomplished either by a poll() call, as in the
		330	RX path, or by calling sendto().
		331
		332	An example of how to use this flag can be found in
		333	samples/bpf/xdpsock_user.c. An example with the use of libbpf helpers
		334	would look like this for the TX path:
		335
		336	.. code-block:: c
		337
		338	if (xsk_ring_prod__needs_wakeup(&my_tx_ring))
		339	sendto(xsk_socket__fd(xsk_handle), NULL, 0, MSG_DONTWAIT, NULL, 0);
		340
		341	I.e., only use the syscall if the flag is set.
		342
		343	We recommend that you always enable this mode as it usually leads to
		344	better performance especially if you run the application and the
		345	driver on the same core, but also if you use different cores for the
		346	application and the kernel driver, as it reduces the number of
		347	syscalls needed for the TX path.
		348
		349	XDP_{RX\|TX\|UMEM_FILL\|UMEM_COMPLETION}_RING setsockopts
		350	------------------------------------------------------
		351
		352	These setsockopts sets the number of descriptors that the RX, TX,
		353	FILL, and COMPLETION rings respectively should have. It is mandatory
		354	to set the size of at least one of the RX and TX rings. If you set
		355	both, you will be able to both receive and send traffic from your
		356	application, but if you only want to do one of them, you can save
		357	resources by only setting up one of them. Both the FILL ring and the
		358	COMPLETION ring are mandatory if you have a UMEM tied to your socket,
		359	which is the normal case. But if the XDP_SHARED_UMEM flag is used, any
		360	socket after the first one does not have a UMEM and should in that
		361	case not have any FILL or COMPLETION rings created.
		362
		363	XDP_UMEM_REG setsockopt
		364	-----------------------
		365
		366	This setsockopt registers a UMEM to a socket. This is the area that
		367	contain all the buffers that packet can recide in. The call takes a
		368	pointer to the beginning of this area and the size of it. Moreover, it
		369	also has parameter called chunk_size that is the size that the UMEM is
		370	divided into. It can only be 2K or 4K at the moment. If you have an
		371	UMEM area that is 128K and a chunk size of 2K, this means that you
		372	will be able to hold a maximum of 128K / 2K = 64 packets in your UMEM
		373	area and that your largest packet size can be 2K.
		374
		375	There is also an option to set the headroom of each single buffer in
		376	the UMEM. If you set this to N bytes, it means that the packet will
		377	start N bytes into the buffer leaving the first N bytes for the
		378	application to use. The final option is the flags field, but it will
		379	be dealt with in separate sections for each UMEM flag.
		380
		381	XDP_STATISTICS getsockopt
		382	-------------------------
		383
		384	Gets drop statistics of a socket that can be useful for debug
		385	purposes. The supported statistics are shown below:
		386
		387	.. code-block:: c
		388
		389	struct xdp_statistics {
		390	__u64 rx_dropped; /* Dropped for reasons other than invalid desc */
		391	__u64 rx_invalid_descs; /* Dropped due to invalid descriptor */
		392	__u64 tx_invalid_descs; /* Dropped due to invalid descriptor */
		393	};
		394
		395	XDP_OPTIONS getsockopt
		396	----------------------
		397
		398	Gets options from an XDP socket. The only one supported so far is
		399	XDP_OPTIONS_ZEROCOPY which tells you if zero-copy is on or not.
		400
219	Usage	401	Usage
220	=====	402	=====
221		403
222	In order to use AF_XDP sockets there are two parts needed. The	404	In order to use AF_XDP sockets two parts are needed. The
223	user-space application and the XDP program. For a complete setup and	405	user-space application and the XDP program. For a complete setup and
224	usage example, please refer to the sample application. The user-space	406	usage example, please refer to the sample application. The user-space
225	side is xdpsock_user.c and the XDP side is part of libbpf.	407	side is xdpsock_user.c and the XDP side is part of libbpf.
226		408
227	The XDP code sample included in tools/lib/bpf/xsk.c is the following::	409	The XDP code sample included in tools/lib/bpf/xsk.c is the following:
		410
		411	.. code-block:: c
228		412
229	SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)	413	SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
230	{	414	{
231	int index = ctx->rx_queue_index;	415	int index = ctx->rx_queue_index;
232		416
233	// A set entry here means that the correspnding queue_id	417	// A set entry here means that the corresponding queue_id
234	// has an active AF_XDP socket bound to it.	418	// has an active AF_XDP socket bound to it.
235	if (bpf_map_lookup_elem(&xsks_map, &index))	419	if (bpf_map_lookup_elem(&xsks_map, &index))
236	return bpf_redirect_map(&xsks_map, index, 0);	420	return bpf_redirect_map(&xsks_map, index, 0);
@@ -238,7 +422,10 @@ The XDP code sample included in tools/lib/bpf/xsk.c is the following:: · 1 unmodified line
238	return XDP_PASS;	422	return XDP_PASS;
239	}	423	}
240		424
241	Naive ring dequeue and enqueue could look like this::	425	A simple but not so performance ring dequeue and enqueue could look
		426	like this:
		427
		428	.. code-block:: c
242		429
243	// struct xdp_rxtx_ring {	430	// struct xdp_rxtx_ring {
244	// __u32 *producer;	431	// __u32 *producer;
@@ -287,17 +474,16 @@ Naive ring dequeue and enqueue could look like this:: · 42 unmodified lines
287	return 0;	474	return 0;
288	}	475	}
289		476
290		477	But please use the libbpf functions as they are optimized and ready to
291	For a more optimized version, please refer to the sample application.	478	use. Will make your life easier.
292		479
293	Sample application	480	Sample application
294	==================	481	==================
295		482
296	There is a xdpsock benchmarking/test application included that	483	There is a xdpsock benchmarking/test application included that
297	demonstrates how to use AF_XDP sockets with both private and shared	484	demonstrates how to use AF_XDP sockets with private UMEMs. Say that
298	UMEMs. Say that you would like your UDP traffic from port 4242 to end	485	you would like your UDP traffic from port 4242 to end up in queue 16,
299	up in queue 16, that we will enable AF_XDP on. Here, we use ethtool	486	that we will enable AF_XDP on. Here, we use ethtool for this::
300	for this::
301		487
302	ethtool -N p3p2 rx-flow-hash udp4 fn	488	ethtool -N p3p2 rx-flow-hash udp4 fn
303	ethtool -N p3p2 flow-type udp4 src-port 4242 dst-port 4242 \	489	ethtool -N p3p2 flow-type udp4 src-port 4242 dst-port 4242 \
@@ -311,13 +497,18 @@ using:: · 7 unmodified lines
311	For XDP_SKB mode, use the switch "-S" instead of "-N" and all options	497	For XDP_SKB mode, use the switch "-S" instead of "-N" and all options
312	can be displayed with "-h", as usual.	498	can be displayed with "-h", as usual.
313		499
		500	This sample application uses libbpf to make the setup and usage of
		501	AF_XDP simpler. If you want to know how the raw uapi of AF_XDP is
		502	really used to make something more advanced, take a look at the libbpf
		503	code in tools/lib/bpf/xsk.[ch].
		504
314	FAQ	505	FAQ
315	=======	506	=======
316		507
317	Q: I am not seeing any traffic on the socket. What am I doing wrong?	508	Q: I am not seeing any traffic on the socket. What am I doing wrong?
318		509
319	A: When a netdev of a physical NIC is initialized, Linux usually	510	A: When a netdev of a physical NIC is initialized, Linux usually
320	allocates one Rx and Tx queue pair per core. So on a 8 core system,	511	allocates one RX and TX queue pair per core. So on a 8 core system,
321	queue ids 0 to 7 will be allocated, one per core. In the AF_XDP	512	queue ids 0 to 7 will be allocated, one per core. In the AF_XDP
322	bind call or the xsk_socket__create libbpf function call, you	513	bind call or the xsk_socket__create libbpf function call, you
323	specify a specific queue id to bind to and it is only the traffic	514	specify a specific queue id to bind to and it is only the traffic
@@ -343,9 +534,21 @@ A: When a netdev of a physical NIC is initialized, Linux usually · 19 unmodified lines
343	sudo ethtool -N <interface> flow-type udp4 src-port 4242 dst-port \	534	sudo ethtool -N <interface> flow-type udp4 src-port 4242 dst-port \
344	4242 action 2	535	4242 action 2
345		536
346	A number of other ways are possible all up to the capabilitites of	537	A number of other ways are possible all up to the capabilities of
347	the NIC you have.	538	the NIC you have.
348		539
		540	Q: Can I use the XSKMAP to implement a switch betwen different umems
		541	in copy mode?
		542
		543	A: The short answer is no, that is not supported at the moment. The
		544	XSKMAP can only be used to switch traffic coming in on queue id X
		545	to sockets bound to the same queue id X. The XSKMAP can contain
		546	sockets bound to different queue ids, for example X and Y, but only
		547	traffic goming in from queue id Y can be directed to sockets bound
		548	to the same queue id Y. In zero-copy mode, you should use the
		549	switch, or other distribution mechanism, in your NIC to direct
		550	traffic to the correct queue id and socket.
		551
349	Credits	552	Credits
350	=======	553	=======
351		554

arch/x86/net/bpf_jit_comp.c+93−4

@@ -9,7 +9,7 @@ · 8 unmodified lines
9	#include <linux/filter.h>	9	#include <linux/filter.h>
10	#include <linux/if_vlan.h>	10	#include <linux/if_vlan.h>
11	#include <linux/bpf.h>	11	#include <linux/bpf.h>
12		12	#include <asm/extable.h>
13	#include <asm/set_memory.h>	13	#include <asm/set_memory.h>
14	#include <asm/nospec-branch.h>	14	#include <asm/nospec-branch.h>
15		15
@@ -123,6 +123,19 @@ static const int reg2hex[] = { · 107 unmodified lines
123	[AUX_REG] = 3, /* R11 temp register */	123	[AUX_REG] = 3, /* R11 temp register */
124	};	124	};
125		125
		126	static const int reg2pt_regs[] = {
		127	[BPF_REG_0] = offsetof(struct pt_regs, ax),
		128	[BPF_REG_1] = offsetof(struct pt_regs, di),
		129	[BPF_REG_2] = offsetof(struct pt_regs, si),
		130	[BPF_REG_3] = offsetof(struct pt_regs, dx),
		131	[BPF_REG_4] = offsetof(struct pt_regs, cx),
		132	[BPF_REG_5] = offsetof(struct pt_regs, r8),
		133	[BPF_REG_6] = offsetof(struct pt_regs, bx),
		134	[BPF_REG_7] = offsetof(struct pt_regs, r13),
		135	[BPF_REG_8] = offsetof(struct pt_regs, r14),
		136	[BPF_REG_9] = offsetof(struct pt_regs, r15),
		137	};
		138
126	/*	139	/*
127	* is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15	140	* is_ereg() == true if BPF register 'reg' maps to x86-64 r8..r15
128	* which need extra byte of encoding.	141	* which need extra byte of encoding.
@@ -377,6 +390,19 @@ static void emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg) · 248 unmodified lines
377	*pprog = prog;	390	*pprog = prog;
378	}	391	}
379		392
		393
		394	static bool ex_handler_bpf(const struct exception_table_entry *x,
		395	struct pt_regs *regs, int trapnr,
		396	unsigned long error_code, unsigned long fault_addr)
		397	{
		398	u32 reg = x->fixup >> 8;
		399
		400	/* jump over faulting load and clear dest register */
		401	(unsigned long )((void *)regs + reg) = 0;
		402	regs->ip += x->fixup & 0xff;
		403	return true;
		404	}
		405
380	static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 *image,	406	static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 *image,
381	int oldproglen, struct jit_context *ctx)	407	int oldproglen, struct jit_context *ctx)
382	{	408	{
@@ -384,7 +410,7 @@ static int do_jit(struct bpf_prog bpf_prog, int addrs, u8 *image, · 1 unmodified line
384	int insn_cnt = bpf_prog->len;	410	int insn_cnt = bpf_prog->len;
385	bool seen_exit = false;	411	bool seen_exit = false;
386	u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];	412	u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
387	int i, cnt = 0;	413	int i, cnt = 0, excnt = 0;
388	int proglen = 0;	414	int proglen = 0;
389	u8 *prog = temp;	415	u8 *prog = temp;
390		416
@@ -778,14 +804,17 @@ stx: if (is_imm8(insn->off)) · 387 unmodified lines
778		804
779	/* LDX: dst_reg = (u8)(src_reg + off) */	805	/* LDX: dst_reg = (u8)(src_reg + off) */
780	case BPF_LDX \| BPF_MEM \| BPF_B:	806	case BPF_LDX \| BPF_MEM \| BPF_B:
		807	case BPF_LDX \| BPF_PROBE_MEM \| BPF_B:
781	/* Emit 'movzx rax, byte ptr [rax + off]' */	808	/* Emit 'movzx rax, byte ptr [rax + off]' */
782	EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);	809	EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
783	goto ldx;	810	goto ldx;
784	case BPF_LDX \| BPF_MEM \| BPF_H:	811	case BPF_LDX \| BPF_MEM \| BPF_H:
		812	case BPF_LDX \| BPF_PROBE_MEM \| BPF_H:
785	/* Emit 'movzx rax, word ptr [rax + off]' */	813	/* Emit 'movzx rax, word ptr [rax + off]' */
786	EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);	814	EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
787	goto ldx;	815	goto ldx;
788	case BPF_LDX \| BPF_MEM \| BPF_W:	816	case BPF_LDX \| BPF_MEM \| BPF_W:
		817	case BPF_LDX \| BPF_PROBE_MEM \| BPF_W:
789	/* Emit 'mov eax, dword ptr [rax+0x14]' */	818	/* Emit 'mov eax, dword ptr [rax+0x14]' */
790	if (is_ereg(dst_reg) \|\| is_ereg(src_reg))	819	if (is_ereg(dst_reg) \|\| is_ereg(src_reg))
791	EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);	820	EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
@@ -793,6 +822,7 @@ stx: if (is_imm8(insn->off)) · 1 unmodified line
793	EMIT1(0x8B);	822	EMIT1(0x8B);
794	goto ldx;	823	goto ldx;
795	case BPF_LDX \| BPF_MEM \| BPF_DW:	824	case BPF_LDX \| BPF_MEM \| BPF_DW:
		825	case BPF_LDX \| BPF_PROBE_MEM \| BPF_DW:
796	/* Emit 'mov rax, qword ptr [rax+0x14]' */	826	/* Emit 'mov rax, qword ptr [rax+0x14]' */
797	EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);	827	EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
798	ldx: /*	828	ldx: /*
@@ -805,6 +835,48 @@ ldx: /* · 6 unmodified lines
805	else	835	else
806	EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),	836	EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),
807	insn->off);	837	insn->off);
		838	if (BPF_MODE(insn->code) == BPF_PROBE_MEM) {
		839	struct exception_table_entry *ex;
		840	u8 *_insn = image + proglen;
		841	s64 delta;
		842
		843	if (!bpf_prog->aux->extable)
		844	break;
		845
		846	if (excnt >= bpf_prog->aux->num_exentries) {
		847	pr_err("ex gen bug\n");
		848	return -EFAULT;
		849	}
		850	ex = &bpf_prog->aux->extable[excnt++];
		851
		852	delta = _insn - (u8 *)&ex->insn;
		853	if (!is_simm32(delta)) {
		854	pr_err("extable->insn doesn't fit into 32-bit\n");
		855	return -EFAULT;
		856	}
		857	ex->insn = delta;
		858
		859	delta = (u8 )ex_handler_bpf - (u8 )&ex->handler;
		860	if (!is_simm32(delta)) {
		861	pr_err("extable->handler doesn't fit into 32-bit\n");
		862	return -EFAULT;
		863	}
		864	ex->handler = delta;
		865
		866	if (dst_reg > BPF_REG_9) {
		867	pr_err("verifier error\n");
		868	return -EFAULT;
		869	}
		870	/*
		871	* Compute size of x86 insn and its target dest x86 register.
		872	* ex_handler_bpf() will use lower 8 bits to adjust
		873	* pt_regs->ip to jump over this x86 instruction
		874	* and upper bits to figure out which pt_regs to zero out.
		875	* End result: x86 insn "mov rbx, qword ptr [rax+0x14]"
		876	* of 4 bytes will be ignored and rbx will be zero inited.
		877	*/
		878	ex->fixup = (prog - temp) \| (reg2pt_regs[dst_reg] << 8);
		879	}
808	break;	880	break;
809		881
810	/* STX XADD: lock (u32)(dst_reg + off) += src_reg */	882	/* STX XADD: lock (u32)(dst_reg + off) += src_reg */
@@ -1058,6 +1130,11 @@ emit_jmp: · 247 unmodified lines
1058	addrs[i] = proglen;	1130	addrs[i] = proglen;
1059	prog = temp;	1131	prog = temp;
1060	}	1132	}
		1133
		1134	if (image && excnt != bpf_prog->aux->num_exentries) {
		1135	pr_err("extable is not populated\n");
		1136	return -EFAULT;
		1137	}
1061	return proglen;	1138	return proglen;
1062	}	1139	}
1063		1140
@@ -1158,12 +1235,24 @@ out_image: · 94 unmodified lines
1158	break;	1235	break;
1159	}	1236	}
1160	if (proglen == oldproglen) {	1237	if (proglen == oldproglen) {
1161	header = bpf_jit_binary_alloc(proglen, &image,	1238	/*
1162	1, jit_fill_hole);	1239	* The number of entries in extable is the number of BPF_LDX
		1240	* insns that access kernel memory via "pointer to BTF type".
		1241	* The verifier changed their opcode from LDX\|MEM\|size
		1242	* to LDX\|PROBE_MEM\|size to make JITing easier.
		1243	*/
		1244	u32 align = __alignof__(struct exception_table_entry);
		1245	u32 extable_size = prog->aux->num_exentries *
		1246	sizeof(struct exception_table_entry);
		1247
		1248	/* allocate module memory for x86 insns and extable */
		1249	header = bpf_jit_binary_alloc(roundup(proglen, align) + extable_size,
		1250	&image, align, jit_fill_hole);
1163	if (!header) {	1251	if (!header) {
1164	prog = orig_prog;	1252	prog = orig_prog;
1165	goto out_addrs;	1253	goto out_addrs;
1166	}	1254	}
		1255	prog->aux->extable = (void *) image + roundup(proglen, align);
1167	}	1256	}
1168	oldproglen = proglen;	1257	oldproglen = proglen;
1169	cond_resched();	1258	cond_resched();

include/linux/bpf.h+38−6

 #include <linux/u64_stats_sync.h>
 struct bpf_verifier_env;
+struct bpf_verifier_log;
 struct perf_event;
 struct bpf_prog;
 struct bpf_map;
 struct seq_file;
 struct btf;
 struct btf_type;
+struct exception_table_entry;
 extern struct idr btf_idr;
 extern spinlock_t btf_idr_lock;
 	ARG_PTR_TO_INT,		/* pointer to int */
 	ARG_PTR_TO_LONG,	/* pointer to long */
 	ARG_PTR_TO_SOCKET,	/* pointer to bpf_sock (fullsock) */
+	ARG_PTR_TO_BTF_ID,	/* pointer to in-kernel struct */
 };
 /* type of values returned from helper functions */
 	bool gpl_only;
 	bool pkt_access;
 	enum bpf_return_type ret_type;
-	enum bpf_arg_type arg1_type;
+	union {
-	enum bpf_arg_type arg2_type;
+		struct {
-	enum bpf_arg_type arg3_type;
+			enum bpf_arg_type arg1_type;
-	enum bpf_arg_type arg4_type;
+			enum bpf_arg_type arg2_type;
-	enum bpf_arg_type arg5_type;
+			enum bpf_arg_type arg3_type;
+			enum bpf_arg_type arg4_type;
+			enum bpf_arg_type arg5_type;
+		};
+		enum bpf_arg_type arg_type[5];
+	};
+	u32 *btf_id; /* BTF ids of arguments */
 };
 /* bpf_context is intentionally undefined structure. Pointer to bpf_context is
 	PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
 	PTR_TO_TP_BUFFER,	 /* reg points to a writable raw tp's buffer */
 	PTR_TO_XDP_SOCK,	 /* reg points to struct xdp_sock */
+	PTR_TO_BTF_ID,		 /* reg points to kernel struct */
 };
 /* The information passed from prog-specific *_is_valid_access
  */
 struct bpf_insn_access_aux {
 	enum bpf_reg_type reg_type;
-	int ctx_field_size;
+	union {
+		int ctx_field_size;
+		u32 btf_id;
+	};
+	struct bpf_verifier_log *log; /* for verbose logs */
 };
 static inline void
 	u32 id;
 	u32 func_cnt; /* used by non-func prog as the number of func progs */
 	u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */
+	u32 attach_btf_id; /* in-kernel BTF type id to attach to */
 	bool verifier_zext; /* Zero extensions has been inserted by verifier. */
 	bool offload_requested;
+	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
+	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
+	const struct btf_type *attach_func_proto;
+	/* function name for valid attach_btf_id */
+	const char *attach_func_name;
 	struct bpf_prog **func;
 	void *jit_data; /* JIT specific data. arch dependent */
 	struct latch_tree_node ksym_tnode;
 	 * main prog always has linfo_idx == 0
 	 */
 	u32 linfo_idx;
+	u32 num_exentries;
+	struct exception_table_entry *extable;
 	struct bpf_prog_stats __percpu *stats;
 	union {
 		struct work_struct work;
 bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp);
 int bpf_prog_calc_tag(struct bpf_prog *fp);
+const char *kernel_type_name(u32 btf_type_id);
 const struct bpf_func_proto *bpf_get_trace_printk_proto(void);
 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
 				     const union bpf_attr *kattr,
 				     union bpf_attr __user *uattr);
+bool btf_ctx_access(int off, int size, enum bpf_access_type type,
+		    const struct bpf_prog *prog,
+		    struct bpf_insn_access_aux *info);
+int btf_struct_access(struct bpf_verifier_log *log,
+		      const struct btf_type *t, int off, int size,
+		      enum bpf_access_type atype,
+		      u32 *next_btf_id);
+u32 btf_resolve_helper_id(struct bpf_verifier_log *log, void *, int);
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {

include/linux/bpf_verifier.h+7−1

 		 */
 		struct bpf_map *map_ptr;
+		u32 btf_id; /* for PTR_TO_BTF_ID */
 		/* Max size from any of the above. */
 		unsigned long raw;
 	};
 #define BPF_LOG_STATS	4
 #define BPF_LOG_LEVEL	(BPF_LOG_LEVEL1 | BPF_LOG_LEVEL2)
 #define BPF_LOG_MASK	(BPF_LOG_LEVEL | BPF_LOG_STATS)
+#define BPF_LOG_KERNEL	(BPF_LOG_MASK + 1) /* kernel internal flag */
 static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
 {
-	return log->level && log->ubuf && !bpf_verifier_log_full(log);
+	return (log->level && log->ubuf && !bpf_verifier_log_full(log)) ||
+		log->level == BPF_LOG_KERNEL;
 }
 #define BPF_MAX_SUBPROGS 256
 				      const char *fmt, va_list args);
 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
 					   const char *fmt, ...);
+__printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
+			    const char *fmt, ...);
 static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env)
 {

include/linux/btf.h+32−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	#define _LINUX_BTF_H 1	5	#define _LINUX_BTF_H 1
6		6
7	#include <linux/types.h>	7	#include <linux/types.h>
		8	#include <uapi/linux/btf.h>
8		9
9	struct btf;	10	struct btf;
10	struct btf_member;	11	struct btf_member;
@@ -53,9 +54,40 @@ bool btf_member_is_reg_int(const struct btf btf, const struct btf_type s, · 42 unmodified lines
53	int btf_find_spin_lock(const struct btf btf, const struct btf_type t);	54	int btf_find_spin_lock(const struct btf btf, const struct btf_type t);
54	bool btf_type_is_void(const struct btf_type *t);	55	bool btf_type_is_void(const struct btf_type *t);
55		56
		57	static inline bool btf_type_is_ptr(const struct btf_type *t)
		58	{
		59	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
		60	}
		61
		62	static inline bool btf_type_is_int(const struct btf_type *t)
		63	{
		64	return BTF_INFO_KIND(t->info) == BTF_KIND_INT;
		65	}
		66
		67	static inline bool btf_type_is_enum(const struct btf_type *t)
		68	{
		69	return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM;
		70	}
		71
		72	static inline bool btf_type_is_typedef(const struct btf_type *t)
		73	{
		74	return BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF;
		75	}
		76
		77	static inline bool btf_type_is_func(const struct btf_type *t)
		78	{
		79	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC;
		80	}
		81
		82	static inline bool btf_type_is_func_proto(const struct btf_type *t)
		83	{
		84	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
		85	}
		86
56	#ifdef CONFIG_BPF_SYSCALL	87	#ifdef CONFIG_BPF_SYSCALL
57	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);	88	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);
58	const char btf_name_by_offset(const struct btf btf, u32 offset);	89	const char btf_name_by_offset(const struct btf btf, u32 offset);
		90	struct btf *btf_parse_vmlinux(void);
59	#else	91	#else
60	static inline const struct btf_type btf_type_by_id(const struct btf btf,	92	static inline const struct btf_type btf_type_by_id(const struct btf btf,
61	u32 type_id)	93	u32 type_id)

include/linux/extable.h+10−0

@@ -33,4 +33,14 @@ search_module_extables(unsigned long addr) · 32 unmodified lines
33	}	33	}
34	#endif /CONFIG_MODULES/	34	#endif /CONFIG_MODULES/
35		35
		36	#ifdef CONFIG_BPF_JIT
		37	const struct exception_table_entry *search_bpf_extables(unsigned long addr);
		38	#else
		39	static inline const struct exception_table_entry *
		40	search_bpf_extables(unsigned long addr)
		41	{
		42	return NULL;
		43	}
		44	#endif
		45
36	#endif /* _LINUX_EXTABLE_H */	46	#endif /* _LINUX_EXTABLE_H */

include/linux/filter.h+5−1

 /* unused opcode to mark special call to bpf_tail_call() helper */
 #define BPF_TAIL_CALL	0xf0
+/* unused opcode to mark special load instruction. Same as BPF_ABS */
+#define BPF_PROBE_MEM	0x20
 /* unused opcode to mark call to interpreter with arguments */
 #define BPF_CALL_ARGS	0xe0
 #define BPF_CALL_x(x, name, ...)					       \
 	static __always_inline						       \
 	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__));   \
+	typedef u64 (*btf_##name)(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__)); \
 	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__));	       \
 	u64 name(__BPF_REG(x, __BPF_DECL_REGS, __BPF_N, __VA_ARGS__))	       \
 	{								       \
-		return ____##name(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
+		return ((btf_##name)____##name)(__BPF_MAP(x,__BPF_CAST,__BPF_N,__VA_ARGS__));\
 	}								       \
 	static __always_inline						       \
 	u64 ____##name(__BPF_MAP(x, __BPF_DECL_ARGS, __BPF_V, __VA_ARGS__))

include/trace/bpf_probe.h+2−1

 {									\
 	check_trace_callback_type_##call(__bpf_trace_##template);	\
 }									\
+typedef void (*btf_trace_##call)(void *__data, proto);			\
 static struct bpf_raw_event_map	__used					\
 	__attribute__((section("__bpf_raw_tp_map")))			\
 __bpf_trace_tp_map_##call = {						\
 	.tp		= &__tracepoint_##call,				\
-	.bpf_func	= (void *)__bpf_trace_##template,		\
+	.bpf_func	= (void *)(btf_trace_##call)__bpf_trace_##template,	\
 	.num_args	= COUNT_ARGS(args),				\
 	.writable_size	= size,						\
 };

include/trace/events/xdp.h+1−1

@@ -22,7 +22,7 @@ · 21 unmodified lines
22	#define __XDP_ACT_SYM_FN(x) \	22	#define __XDP_ACT_SYM_FN(x) \
23	{ XDP_##x, #x },	23	{ XDP_##x, #x },
24	#define __XDP_ACT_SYM_TAB \	24	#define __XDP_ACT_SYM_TAB \
25	__XDP_ACT_MAP(__XDP_ACT_SYM_FN) { -1, 0 }	25	__XDP_ACT_MAP(__XDP_ACT_SYM_FN) { -1, NULL }
26	__XDP_ACT_MAP(__XDP_ACT_TP_FN)	26	__XDP_ACT_MAP(__XDP_ACT_TP_FN)
27		27
28	TRACE_EVENT(xdp_exception,	28	TRACE_EVENT(xdp_exception,

include/uapi/linux/bpf.h+27−1

@@ -420,6 +420,7 @@ union bpf_attr { · 419 unmodified lines
420	__u32 line_info_rec_size; /* userspace bpf_line_info size */	420	__u32 line_info_rec_size; /* userspace bpf_line_info size */
421	__aligned_u64 line_info; /* line info */	421	__aligned_u64 line_info; /* line info */
422	__u32 line_info_cnt; /* number of bpf_line_info records */	422	__u32 line_info_cnt; /* number of bpf_line_info records */
		423	__u32 attach_btf_id; /* in-kernel BTF type id to attach to */
423	};	424	};
424		425
425	struct { /* anonymous struct used by BPF_OBJ_* commands */	426	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -2750,6 +2751,30 @@ union bpf_attr { · 2324 unmodified lines
2750	* -EOPNOTSUPP kernel configuration does not enable SYN cookies	2751	* -EOPNOTSUPP kernel configuration does not enable SYN cookies
2751	*	2752	*
2752	* -EPROTONOSUPPORT IP packet version is not 4 or 6	2753	* -EPROTONOSUPPORT IP packet version is not 4 or 6
		2754	*
		2755	* int bpf_skb_output(void ctx, struct bpf_map map, u64 flags, void *data, u64 size)
		2756	* Description
		2757	* Write raw data blob into a special BPF perf event held by
		2758	* map of type BPF_MAP_TYPE_PERF_EVENT_ARRAY. This perf
		2759	* event must have the following attributes: PERF_SAMPLE_RAW
		2760	* as sample_type, PERF_TYPE_SOFTWARE as type, and
		2761	* PERF_COUNT_SW_BPF_OUTPUT as config.
		2762	*
		2763	* The flags are used to indicate the index in map for which
		2764	* the value must be put, masked with BPF_F_INDEX_MASK.
		2765	* Alternatively, flags can be set to BPF_F_CURRENT_CPU
		2766	* to indicate that the index of the current CPU core should be
		2767	* used.
		2768	*
		2769	* The value to write, of size, is passed through eBPF stack and
		2770	* pointed by data.
		2771	*
		2772	* ctx is a pointer to in-kernel struct sk_buff.
		2773	*
		2774	* This helper is similar to bpf_perf_event_output\ () but
		2775	* restricted to raw_tracepoint bpf programs.
		2776	* Return
		2777	* 0 on success, or a negative error in case of failure.
2753	*/	2778	*/
2754	#define __BPF_FUNC_MAPPER(FN) \	2779	#define __BPF_FUNC_MAPPER(FN) \
2755	FN(unspec), \	2780	FN(unspec), \
@@ -2862,7 +2887,8 @@ union bpf_attr { · 106 unmodified lines
2862	FN(sk_storage_get), \	2887	FN(sk_storage_get), \
2863	FN(sk_storage_delete), \	2888	FN(sk_storage_delete), \
2864	FN(send_signal), \	2889	FN(send_signal), \
2865	FN(tcp_gen_syncookie),	2890	FN(tcp_gen_syncookie), \
		2891	FN(skb_output),
2866		2892
2867	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	2893	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2868	* function eBPF program intends to call	2894	* function eBPF program intends to call

kernel/bpf/btf.c+301−21

@@ -336,16 +336,6 @@ static bool btf_type_is_fwd(const struct btf_type *t) · 335 unmodified lines
336	return BTF_INFO_KIND(t->info) == BTF_KIND_FWD;	336	return BTF_INFO_KIND(t->info) == BTF_KIND_FWD;
337	}	337	}
338		338
339	static bool btf_type_is_func(const struct btf_type *t)
340	{
341	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC;
342	}
343
344	static bool btf_type_is_func_proto(const struct btf_type *t)
345	{
346	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
347	}
348
349	static bool btf_type_nosize(const struct btf_type *t)	339	static bool btf_type_nosize(const struct btf_type *t)
350	{	340	{
351	return btf_type_is_void(t) \|\| btf_type_is_fwd(t) \|\|	341	return btf_type_is_void(t) \|\| btf_type_is_fwd(t) \|\|
@@ -377,16 +367,6 @@ static bool btf_type_is_array(const struct btf_type *t) · 25 unmodified lines
377	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;	367	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
378	}	368	}
379		369
380	static bool btf_type_is_ptr(const struct btf_type *t)
381	{
382	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
383	}
384
385	static bool btf_type_is_int(const struct btf_type *t)
386	{
387	return BTF_INFO_KIND(t->info) == BTF_KIND_INT;
388	}
389
390	static bool btf_type_is_var(const struct btf_type *t)	370	static bool btf_type_is_var(const struct btf_type *t)
391	{	371	{
392	return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;	372	return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
@@ -698,6 +678,13 @@ __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env, · 305 unmodified lines
698	if (!bpf_verifier_log_needed(log))	678	if (!bpf_verifier_log_needed(log))
699	return;	679	return;
700		680
		681	/* btf verifier prints all types it is processing via
		682	* btf_verifier_log_type(..., fmt = NULL).
		683	* Skip those prints for in-kernel BTF verification.
		684	*/
		685	if (log->level == BPF_LOG_KERNEL && !fmt)
		686	return;
		687
701	__btf_verifier_log(log, "[%u] %s %s%s",	688	__btf_verifier_log(log, "[%u] %s %s%s",
702	env->log_type_id,	689	env->log_type_id,
703	btf_kind_str[kind],	690	btf_kind_str[kind],
@@ -735,6 +722,8 @@ static void btf_verifier_log_member(struct btf_verifier_env *env, · 31 unmodified lines
735	if (!bpf_verifier_log_needed(log))	722	if (!bpf_verifier_log_needed(log))
736	return;	723	return;
737		724
		725	if (log->level == BPF_LOG_KERNEL && !fmt)
		726	return;
738	/* The CHECK_META phase already did a btf dump.	727	/* The CHECK_META phase already did a btf dump.
739	*	728	*
740	* If member is logged again, it must hit an error in	729	* If member is logged again, it must hit an error in
@@ -777,6 +766,8 @@ static void btf_verifier_log_vsi(struct btf_verifier_env *env, · 36 unmodified lines
777		766
778	if (!bpf_verifier_log_needed(log))	767	if (!bpf_verifier_log_needed(log))
779	return;	768	return;
		769	if (log->level == BPF_LOG_KERNEL && !fmt)
		770	return;
780	if (env->phase != CHECK_META)	771	if (env->phase != CHECK_META)
781	btf_verifier_log_type(env, datasec_type, NULL);	772	btf_verifier_log_type(env, datasec_type, NULL);
782		773
@@ -802,6 +793,8 @@ static void btf_verifier_log_hdr(struct btf_verifier_env *env, · 19 unmodified lines
802	if (!bpf_verifier_log_needed(log))	793	if (!bpf_verifier_log_needed(log))
803	return;	794	return;
804		795
		796	if (log->level == BPF_LOG_KERNEL)
		797	return;
805	hdr = &btf->hdr;	798	hdr = &btf->hdr;
806	__btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);	799	__btf_verifier_log(log, "magic: 0x%x\n", hdr->magic);
807	__btf_verifier_log(log, "version: %u\n", hdr->version);	800	__btf_verifier_log(log, "version: %u\n", hdr->version);
@@ -2405,7 +2398,8 @@ static s32 btf_enum_check_meta(struct btf_verifier_env *env, · 1597 unmodified lines
2405	return -EINVAL;	2398	return -EINVAL;
2406	}	2399	}
2407		2400
2408		2401	if (env->log.level == BPF_LOG_KERNEL)
		2402	continue;
2409	btf_verifier_log(env, "\t%s val=%d\n",	2403	btf_verifier_log(env, "\t%s val=%d\n",
2410	__btf_name_by_offset(btf, enums[i].name_off),	2404	__btf_name_by_offset(btf, enums[i].name_off),
2411	enums[i].val);	2405	enums[i].val);
@@ -3367,6 +3361,292 @@ errout: · 955 unmodified lines
3367	return ERR_PTR(err);	3361	return ERR_PTR(err);
3368	}	3362	}
3369		3363
		3364	extern char __weak _binary__btf_vmlinux_bin_start[];
		3365	extern char __weak _binary__btf_vmlinux_bin_end[];
		3366
		3367	struct btf *btf_parse_vmlinux(void)
		3368	{
		3369	struct btf_verifier_env *env = NULL;
		3370	struct bpf_verifier_log *log;
		3371	struct btf *btf = NULL;
		3372	int err;
		3373
		3374	env = kzalloc(sizeof(*env), GFP_KERNEL \| __GFP_NOWARN);
		3375	if (!env)
		3376	return ERR_PTR(-ENOMEM);
		3377
		3378	log = &env->log;
		3379	log->level = BPF_LOG_KERNEL;
		3380
		3381	btf = kzalloc(sizeof(*btf), GFP_KERNEL \| __GFP_NOWARN);
		3382	if (!btf) {
		3383	err = -ENOMEM;
		3384	goto errout;
		3385	}
		3386	env->btf = btf;
		3387
		3388	btf->data = _binary__btf_vmlinux_bin_start;
		3389	btf->data_size = _binary__btf_vmlinux_bin_end -
		3390	_binary__btf_vmlinux_bin_start;
		3391
		3392	err = btf_parse_hdr(env);
		3393	if (err)
		3394	goto errout;
		3395
		3396	btf->nohdr_data = btf->data + btf->hdr.hdr_len;
		3397
		3398	err = btf_parse_str_sec(env);
		3399	if (err)
		3400	goto errout;
		3401
		3402	err = btf_check_all_metas(env);
		3403	if (err)
		3404	goto errout;
		3405
		3406	btf_verifier_env_free(env);
		3407	refcount_set(&btf->refcnt, 1);
		3408	return btf;
		3409
		3410	errout:
		3411	btf_verifier_env_free(env);
		3412	if (btf) {
		3413	kvfree(btf->types);
		3414	kfree(btf);
		3415	}
		3416	return ERR_PTR(err);
		3417	}
		3418
		3419	extern struct btf *btf_vmlinux;
		3420
		3421	bool btf_ctx_access(int off, int size, enum bpf_access_type type,
		3422	const struct bpf_prog *prog,
		3423	struct bpf_insn_access_aux *info)
		3424	{
		3425	const struct btf_type *t = prog->aux->attach_func_proto;
		3426	const char *tname = prog->aux->attach_func_name;
		3427	struct bpf_verifier_log *log = info->log;
		3428	const struct btf_param *args;
		3429	u32 nr_args, arg;
		3430
		3431	if (off % 8) {
		3432	bpf_log(log, "func '%s' offset %d is not multiple of 8\n",
		3433	tname, off);
		3434	return false;
		3435	}
		3436	arg = off / 8;
		3437	args = (const struct btf_param *)(t + 1);
		3438	nr_args = btf_type_vlen(t);
		3439	if (prog->aux->attach_btf_trace) {
		3440	/* skip first 'void __data' argument in btf_trace_##name typedef /
		3441	args++;
		3442	nr_args--;
		3443	}
		3444	if (arg >= nr_args) {
		3445	bpf_log(log, "func '%s' doesn't have %d-th argument\n",
		3446	tname, arg);
		3447	return false;
		3448	}
		3449
		3450	t = btf_type_by_id(btf_vmlinux, args[arg].type);
		3451	/* skip modifiers */
		3452	while (btf_type_is_modifier(t))
		3453	t = btf_type_by_id(btf_vmlinux, t->type);
		3454	if (btf_type_is_int(t))
		3455	/* accessing a scalar */
		3456	return true;
		3457	if (!btf_type_is_ptr(t)) {
		3458	bpf_log(log,
		3459	"func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n",
		3460	tname, arg,
		3461	__btf_name_by_offset(btf_vmlinux, t->name_off),
		3462	btf_kind_str[BTF_INFO_KIND(t->info)]);
		3463	return false;
		3464	}
		3465	if (t->type == 0)
		3466	/* This is a pointer to void.
		3467	* It is the same as scalar from the verifier safety pov.
		3468	* No further pointer walking is allowed.
		3469	*/
		3470	return true;
		3471
		3472	/* this is a pointer to another type */
		3473	info->reg_type = PTR_TO_BTF_ID;
		3474	info->btf_id = t->type;
		3475
		3476	t = btf_type_by_id(btf_vmlinux, t->type);
		3477	/* skip modifiers */
		3478	while (btf_type_is_modifier(t))
		3479	t = btf_type_by_id(btf_vmlinux, t->type);
		3480	if (!btf_type_is_struct(t)) {
		3481	bpf_log(log,
		3482	"func '%s' arg%d type %s is not a struct\n",
		3483	tname, arg, btf_kind_str[BTF_INFO_KIND(t->info)]);
		3484	return false;
		3485	}
		3486	bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n",
		3487	tname, arg, info->btf_id, btf_kind_str[BTF_INFO_KIND(t->info)],
		3488	__btf_name_by_offset(btf_vmlinux, t->name_off));
		3489	return true;
		3490	}
		3491
		3492	int btf_struct_access(struct bpf_verifier_log *log,
		3493	const struct btf_type *t, int off, int size,
		3494	enum bpf_access_type atype,
		3495	u32 *next_btf_id)
		3496	{
		3497	const struct btf_member *member;
		3498	const struct btf_type *mtype;
		3499	const char tname, mname;
		3500	int i, moff = 0, msize;
		3501
		3502	again:
		3503	tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
		3504	if (!btf_type_is_struct(t)) {
		3505	bpf_log(log, "Type '%s' is not a struct", tname);
		3506	return -EINVAL;
		3507	}
		3508
		3509	for_each_member(i, t, member) {
		3510	/* offset of the field in bits */
		3511	moff = btf_member_bit_offset(t, member);
		3512
		3513	if (btf_member_bitfield_size(t, member))
		3514	/* bitfields are not supported yet */
		3515	continue;
		3516
		3517	if (off + size <= moff / 8)
		3518	/* won't find anything, field is already too far */
		3519	break;
		3520
		3521	/* type of the field */
		3522	mtype = btf_type_by_id(btf_vmlinux, member->type);
		3523	mname = __btf_name_by_offset(btf_vmlinux, member->name_off);
		3524
		3525	/* skip modifiers */
		3526	while (btf_type_is_modifier(mtype))
		3527	mtype = btf_type_by_id(btf_vmlinux, mtype->type);
		3528
		3529	if (btf_type_is_array(mtype))
		3530	/* array deref is not supported yet */
		3531	continue;
		3532
		3533	if (!btf_type_has_size(mtype) && !btf_type_is_ptr(mtype)) {
		3534	bpf_log(log, "field %s doesn't have size\n", mname);
		3535	return -EFAULT;
		3536	}
		3537	if (btf_type_is_ptr(mtype))
		3538	msize = 8;
		3539	else
		3540	msize = mtype->size;
		3541	if (off >= moff / 8 + msize)
		3542	/* no overlap with member, keep iterating */
		3543	continue;
		3544	/* the 'off' we're looking for is either equal to start
		3545	* of this field or inside of this struct
		3546	*/
		3547	if (btf_type_is_struct(mtype)) {
		3548	/* our field must be inside that union or struct */
		3549	t = mtype;
		3550
		3551	/* adjust offset we're looking for */
		3552	off -= moff / 8;
		3553	goto again;
		3554	}
		3555	if (msize != size) {
		3556	/* field access size doesn't match */
		3557	bpf_log(log,
		3558	"cannot access %d bytes in struct %s field %s that has size %d\n",
		3559	size, tname, mname, msize);
		3560	return -EACCES;
		3561	}
		3562
		3563	if (btf_type_is_ptr(mtype)) {
		3564	const struct btf_type *stype;
		3565
		3566	stype = btf_type_by_id(btf_vmlinux, mtype->type);
		3567	/* skip modifiers */
		3568	while (btf_type_is_modifier(stype))
		3569	stype = btf_type_by_id(btf_vmlinux, stype->type);
		3570	if (btf_type_is_struct(stype)) {
		3571	*next_btf_id = mtype->type;
		3572	return PTR_TO_BTF_ID;
		3573	}
		3574	}
		3575	/* all other fields are treated as scalars */
		3576	return SCALAR_VALUE;
		3577	}
		3578	bpf_log(log, "struct %s doesn't have field at offset %d\n", tname, off);
		3579	return -EINVAL;
		3580	}
		3581
		3582	u32 btf_resolve_helper_id(struct bpf_verifier_log log, void fn, int arg)
		3583	{
		3584	char fnname[KSYM_SYMBOL_LEN + 4] = "btf_";
		3585	const struct btf_param *args;
		3586	const struct btf_type *t;
		3587	const char tname, sym;
		3588	u32 btf_id, i;
		3589
		3590	if (IS_ERR(btf_vmlinux)) {
		3591	bpf_log(log, "btf_vmlinux is malformed\n");
		3592	return -EINVAL;
		3593	}
		3594
		3595	sym = kallsyms_lookup((long)fn, NULL, NULL, NULL, fnname + 4);
		3596	if (!sym) {
		3597	bpf_log(log, "kernel doesn't have kallsyms\n");
		3598	return -EFAULT;
		3599	}
		3600
		3601	for (i = 1; i <= btf_vmlinux->nr_types; i++) {
		3602	t = btf_type_by_id(btf_vmlinux, i);
		3603	if (BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF)
		3604	continue;
		3605	tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
		3606	if (!strcmp(tname, fnname))
		3607	break;
		3608	}
		3609	if (i > btf_vmlinux->nr_types) {
		3610	bpf_log(log, "helper %s type is not found\n", fnname);
		3611	return -ENOENT;
		3612	}
		3613
		3614	t = btf_type_by_id(btf_vmlinux, t->type);
		3615	if (!btf_type_is_ptr(t))
		3616	return -EFAULT;
		3617	t = btf_type_by_id(btf_vmlinux, t->type);
		3618	if (!btf_type_is_func_proto(t))
		3619	return -EFAULT;
		3620
		3621	args = (const struct btf_param *)(t + 1);
		3622	if (arg >= btf_type_vlen(t)) {
		3623	bpf_log(log, "bpf helper %s doesn't have %d-th argument\n",
		3624	fnname, arg);
		3625	return -EINVAL;
		3626	}
		3627
		3628	t = btf_type_by_id(btf_vmlinux, args[arg].type);
		3629	if (!btf_type_is_ptr(t) \|\| !t->type) {
		3630	/* anything but the pointer to struct is a helper config bug */
		3631	bpf_log(log, "ARG_PTR_TO_BTF is misconfigured\n");
		3632	return -EFAULT;
		3633	}
		3634	btf_id = t->type;
		3635	t = btf_type_by_id(btf_vmlinux, t->type);
		3636	/* skip modifiers */
		3637	while (btf_type_is_modifier(t)) {
		3638	btf_id = t->type;
		3639	t = btf_type_by_id(btf_vmlinux, t->type);
		3640	}
		3641	if (!btf_type_is_struct(t)) {
		3642	bpf_log(log, "ARG_PTR_TO_BTF is not a struct\n");
		3643	return -EFAULT;
		3644	}
		3645	bpf_log(log, "helper %s arg%d has btf_id %d struct %s\n", fnname + 4,
		3646	arg, btf_id, __btf_name_by_offset(btf_vmlinux, t->name_off));
		3647	return btf_id;
		3648	}
		3649
3370	void btf_type_seq_show(const struct btf btf, u32 type_id, void obj,	3650	void btf_type_seq_show(const struct btf btf, u32 type_id, void obj,
3371	struct seq_file *m)	3651	struct seq_file *m)
3372	{	3652	{

kernel/bpf/core.c+38−1

@@ -30,7 +30,7 @@ · 29 unmodified lines
30	#include <linux/kallsyms.h>	30	#include <linux/kallsyms.h>
31	#include <linux/rcupdate.h>	31	#include <linux/rcupdate.h>
32	#include <linux/perf_event.h>	32	#include <linux/perf_event.h>
33		33	#include <linux/extable.h>
34	#include <asm/unaligned.h>	34	#include <asm/unaligned.h>
35		35
36	/* Registers */	36	/* Registers */
@@ -712,6 +712,24 @@ bool is_bpf_text_address(unsigned long addr) · 675 unmodified lines
712	return ret;	712	return ret;
713	}	713	}
714		714
		715	const struct exception_table_entry *search_bpf_extables(unsigned long addr)
		716	{
		717	const struct exception_table_entry *e = NULL;
		718	struct bpf_prog *prog;
		719
		720	rcu_read_lock();
		721	prog = bpf_prog_kallsyms_find(addr);
		722	if (!prog)
		723	goto out;
		724	if (!prog->aux->num_exentries)
		725	goto out;
		726
		727	e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);
		728	out:
		729	rcu_read_unlock();
		730	return e;
		731	}
		732
715	int bpf_get_kallsym(unsigned int symnum, unsigned long value, char type,	733	int bpf_get_kallsym(unsigned int symnum, unsigned long value, char type,
716	char *sym)	734	char *sym)
717	{	735	{
@@ -1291,6 +1309,11 @@ bool bpf_opcode_in_insntable(u8 code) · 573 unmodified lines
1291	}	1309	}
1292		1310
1293	#ifndef CONFIG_BPF_JIT_ALWAYS_ON	1311	#ifndef CONFIG_BPF_JIT_ALWAYS_ON
		1312	u64 __weak bpf_probe_read(void * dst, u32 size, const void * unsafe_ptr)
		1313	{
		1314	memset(dst, 0, size);
		1315	return -EFAULT;
		1316	}
1294	/**	1317	/**
1295	* __bpf_prog_run - run eBPF program on a given context	1318	* __bpf_prog_run - run eBPF program on a given context
1296	* @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers	1319	* @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
@@ -1310,6 +1333,10 @@ static u64 __no_fgcse ___bpf_prog_run(u64 regs, const struct bpf_insn insn, u6 · 13 unmodified lines
1310	/* Non-UAPI available opcodes. */	1333	/* Non-UAPI available opcodes. */
1311	[BPF_JMP \| BPF_CALL_ARGS] = &&JMP_CALL_ARGS,	1334	[BPF_JMP \| BPF_CALL_ARGS] = &&JMP_CALL_ARGS,
1312	[BPF_JMP \| BPF_TAIL_CALL] = &&JMP_TAIL_CALL,	1335	[BPF_JMP \| BPF_TAIL_CALL] = &&JMP_TAIL_CALL,
		1336	[BPF_LDX \| BPF_PROBE_MEM \| BPF_B] = &&LDX_PROBE_MEM_B,
		1337	[BPF_LDX \| BPF_PROBE_MEM \| BPF_H] = &&LDX_PROBE_MEM_H,
		1338	[BPF_LDX \| BPF_PROBE_MEM \| BPF_W] = &&LDX_PROBE_MEM_W,
		1339	[BPF_LDX \| BPF_PROBE_MEM \| BPF_DW] = &&LDX_PROBE_MEM_DW,
1313	};	1340	};
1314	#undef BPF_INSN_3_LBL	1341	#undef BPF_INSN_3_LBL
1315	#undef BPF_INSN_2_LBL	1342	#undef BPF_INSN_2_LBL
@@ -1542,6 +1569,16 @@ out: · 226 unmodified lines
1542	LDST(W, u32)	1569	LDST(W, u32)
1543	LDST(DW, u64)	1570	LDST(DW, u64)
1544	#undef LDST	1571	#undef LDST
		1572	#define LDX_PROBE(SIZEOP, SIZE) \
		1573	LDX_PROBE_MEM_##SIZEOP: \
		1574	bpf_probe_read(&DST, SIZE, (const void *)(long) SRC); \
		1575	CONT;
		1576	LDX_PROBE(B, 1)
		1577	LDX_PROBE(H, 2)
		1578	LDX_PROBE(W, 4)
		1579	LDX_PROBE(DW, 8)
		1580	#undef LDX_PROBE
		1581
1545	STX_XADD_W: /* lock xadd (u32 )(dst_reg + off16) += src_reg */	1582	STX_XADD_W: /* lock xadd (u32 )(dst_reg + off16) += src_reg */
1546	atomic_add((u32) SRC, (atomic_t *)(unsigned long)	1583	atomic_add((u32) SRC, (atomic_t *)(unsigned long)
1547	(DST + insn->off));	1584	(DST + insn->off));

kernel/bpf/stackmap.c+4−3

 	bool irq_work_busy = false;
 	struct stack_map_irq_work *work = NULL;
-	if (in_nmi()) {
+	if (irqs_disabled()) {
 		work = this_cpu_ptr(&up_read_work);
 		if (work->irq_work.flags & IRQ_WORK_BUSY)
 			/* cannot queue more up_read, fallback */
 	}
 	/*
-	 * We cannot do up_read() in nmi context. To do build_id lookup
+	 * We cannot do up_read() when the irq is disabled, because of
-	 * in nmi context, we need to run up_read() in irq_work. We use
+	 * risk to deadlock with rq_lock. To do build_id lookup when the
+	 * irqs are disabled, we need to run up_read() in irq_work. We use
 	 * a percpu variable to do the irq_work. If the irq_work is
 	 * already used by another lookup, we fall back to report ips.
 	 *

kernel/bpf/syscall.c+64−27

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	#include <linux/timekeeping.h>	23	#include <linux/timekeeping.h>
24	#include <linux/ctype.h>	24	#include <linux/ctype.h>
25	#include <linux/nospec.h>	25	#include <linux/nospec.h>
		26	#include <uapi/linux/btf.h>
26		27
27	#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY \|\| \	28	#define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PROG_ARRAY \|\| \
28	(map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY \|\| \	29	(map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY \|\| \
@@ -1565,9 +1566,21 @@ static void bpf_prog_load_fixup_attach_type(union bpf_attr *attr) · 1536 unmodified lines
1565	}	1566	}
1566		1567
1567	static int	1568	static int
1568	bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type,	1569	bpf_prog_load_check_attach(enum bpf_prog_type prog_type,
1569	enum bpf_attach_type expected_attach_type)	1570	enum bpf_attach_type expected_attach_type,
		1571	u32 btf_id)
1570	{	1572	{
		1573	switch (prog_type) {
		1574	case BPF_PROG_TYPE_RAW_TRACEPOINT:
		1575	if (btf_id > BTF_MAX_TYPE)
		1576	return -EINVAL;
		1577	break;
		1578	default:
		1579	if (btf_id)
		1580	return -EINVAL;
		1581	break;
		1582	}
		1583
1571	switch (prog_type) {	1584	switch (prog_type) {
1572	case BPF_PROG_TYPE_CGROUP_SOCK:	1585	case BPF_PROG_TYPE_CGROUP_SOCK:
1573	switch (expected_attach_type) {	1586	switch (expected_attach_type) {
@@ -1614,7 +1627,7 @@ bpf_prog_load_check_attach_type(enum bpf_prog_type prog_type, · 40 unmodified lines
1614	}	1627	}
1615		1628
1616	/* last field in 'union bpf_attr' used by this command */	1629	/* last field in 'union bpf_attr' used by this command */
1617	#define BPF_PROG_LOAD_LAST_FIELD line_info_cnt	1630	#define BPF_PROG_LOAD_LAST_FIELD attach_btf_id
1618		1631
1619	static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr)	1632	static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr)
1620	{	1633	{
@@ -1656,7 +1669,8 @@ static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr) · 35 unmodified lines
1656	return -EPERM;	1669	return -EPERM;
1657		1670
1658	bpf_prog_load_fixup_attach_type(attr);	1671	bpf_prog_load_fixup_attach_type(attr);
1659	if (bpf_prog_load_check_attach_type(type, attr->expected_attach_type))	1672	if (bpf_prog_load_check_attach(type, attr->expected_attach_type,
		1673	attr->attach_btf_id))
1660	return -EINVAL;	1674	return -EINVAL;
1661		1675
1662	/* plain bpf_prog allocation */	1676	/* plain bpf_prog allocation */
@@ -1665,6 +1679,7 @@ static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr) · 2 unmodified lines
1665	return -ENOMEM;	1679	return -ENOMEM;
1666		1680
1667	prog->expected_attach_type = attr->expected_attach_type;	1681	prog->expected_attach_type = attr->expected_attach_type;
		1682	prog->aux->attach_btf_id = attr->attach_btf_id;
1668		1683
1669	prog->aux->offload_requested = !!attr->prog_ifindex;	1684	prog->aux->offload_requested = !!attr->prog_ifindex;
1670		1685
@@ -1806,17 +1821,50 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) · 135 unmodified lines
1806	struct bpf_raw_tracepoint *raw_tp;	1821	struct bpf_raw_tracepoint *raw_tp;
1807	struct bpf_raw_event_map *btp;	1822	struct bpf_raw_event_map *btp;
1808	struct bpf_prog *prog;	1823	struct bpf_prog *prog;
1809	char tp_name[128];	1824	const char *tp_name;
		1825	char buf[128];
1810	int tp_fd, err;	1826	int tp_fd, err;
1811		1827
1812	if (strncpy_from_user(tp_name, u64_to_user_ptr(attr->raw_tracepoint.name),	1828	if (CHECK_ATTR(BPF_RAW_TRACEPOINT_OPEN))
1813	sizeof(tp_name) - 1) < 0)	1829	return -EINVAL;
1814	return -EFAULT;	1830
1815	tp_name[sizeof(tp_name) - 1] = 0;	1831	prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
		1832	if (IS_ERR(prog))
		1833	return PTR_ERR(prog);
		1834
		1835	if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT &&
		1836	prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {
		1837	err = -EINVAL;
		1838	goto out_put_prog;
		1839	}
		1840
		1841	if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT &&
		1842	prog->aux->attach_btf_id) {
		1843	if (attr->raw_tracepoint.name) {
		1844	/* raw_tp name should not be specified in raw_tp
		1845	* programs that were verified via in-kernel BTF info
		1846	*/
		1847	err = -EINVAL;
		1848	goto out_put_prog;
		1849	}
		1850	/* raw_tp name is taken from type name instead */
		1851	tp_name = prog->aux->attach_func_name;
		1852	} else {
		1853	if (strncpy_from_user(buf,
		1854	u64_to_user_ptr(attr->raw_tracepoint.name),
		1855	sizeof(buf) - 1) < 0) {
		1856	err = -EFAULT;
		1857	goto out_put_prog;
		1858	}
		1859	buf[sizeof(buf) - 1] = 0;
		1860	tp_name = buf;
		1861	}
1816		1862
1817	btp = bpf_get_raw_tracepoint(tp_name);	1863	btp = bpf_get_raw_tracepoint(tp_name);
1818	if (!btp)	1864	if (!btp) {
1819	return -ENOENT;	1865	err = -ENOENT;
		1866	goto out_put_prog;
		1867	}
1820		1868
1821	raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER);	1869	raw_tp = kzalloc(sizeof(*raw_tp), GFP_USER);
1822	if (!raw_tp) {	1870	if (!raw_tp) {
@@ -1824,38 +1872,27 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) · 1 unmodified line
1824	goto out_put_btp;	1872	goto out_put_btp;
1825	}	1873	}
1826	raw_tp->btp = btp;	1874	raw_tp->btp = btp;
1827		1875	raw_tp->prog = prog;
1828	prog = bpf_prog_get(attr->raw_tracepoint.prog_fd);
1829	if (IS_ERR(prog)) {
1830	err = PTR_ERR(prog);
1831	goto out_free_tp;
1832	}
1833	if (prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT &&
1834	prog->type != BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE) {
1835	err = -EINVAL;
1836	goto out_put_prog;
1837	}
1838		1876
1839	err = bpf_probe_register(raw_tp->btp, prog);	1877	err = bpf_probe_register(raw_tp->btp, prog);
1840	if (err)	1878	if (err)
1841	goto out_put_prog;	1879	goto out_free_tp;
1842		1880
1843	raw_tp->prog = prog;
1844	tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp,	1881	tp_fd = anon_inode_getfd("bpf-raw-tracepoint", &bpf_raw_tp_fops, raw_tp,
1845	O_CLOEXEC);	1882	O_CLOEXEC);
1846	if (tp_fd < 0) {	1883	if (tp_fd < 0) {
1847	bpf_probe_unregister(raw_tp->btp, prog);	1884	bpf_probe_unregister(raw_tp->btp, prog);
1848	err = tp_fd;	1885	err = tp_fd;
1849	goto out_put_prog;	1886	goto out_free_tp;
1850	}	1887	}
1851	return tp_fd;	1888	return tp_fd;
1852		1889
1853	out_put_prog:
1854	bpf_prog_put(prog);
1855	out_free_tp:	1890	out_free_tp:
1856	kfree(raw_tp);	1891	kfree(raw_tp);
1857	out_put_btp:	1892	out_put_btp:
1858	bpf_put_raw_tracepoint(btp);	1893	bpf_put_raw_tracepoint(btp);
		1894	out_put_prog:
		1895	bpf_prog_put(prog);
1859	return err;	1896	return err;
1860	}	1897	}
1861		1898

kernel/bpf/verifier.c+193−18

@@ -205,8 +205,11 @@ struct bpf_call_arg_meta { · 204 unmodified lines
205	u64 msize_umax_value;	205	u64 msize_umax_value;
206	int ref_obj_id;	206	int ref_obj_id;
207	int func_id;	207	int func_id;
		208	u32 btf_id;
208	};	209	};
209		210
		211	struct btf *btf_vmlinux;
		212
210	static DEFINE_MUTEX(bpf_verifier_lock);	213	static DEFINE_MUTEX(bpf_verifier_lock);
211		214
212	static const struct bpf_line_info *	215	static const struct bpf_line_info *
@@ -243,6 +246,10 @@ void bpf_verifier_vlog(struct bpf_verifier_log log, const char fmt, · 30 unmodified lines
243	n = min(log->len_total - log->len_used - 1, n);	246	n = min(log->len_total - log->len_used - 1, n);
244	log->kbuf[n] = '\0';	247	log->kbuf[n] = '\0';
245		248
		249	if (log->level == BPF_LOG_KERNEL) {
		250	pr_err("BPF:%s\n", log->kbuf);
		251	return;
		252	}
246	if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))	253	if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
247	log->len_used += n;	254	log->len_used += n;
248	else	255	else
@@ -280,6 +287,19 @@ __printf(2, 3) static void verbose(void private_data, const char fmt, ...) · 31 unmodified lines
280	va_end(args);	287	va_end(args);
281	}	288	}
282		289
		290	__printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
		291	const char *fmt, ...)
		292	{
		293	va_list args;
		294
		295	if (!bpf_verifier_log_needed(log))
		296	return;
		297
		298	va_start(args, fmt);
		299	bpf_verifier_vlog(log, fmt, args);
		300	va_end(args);
		301	}
		302
283	static const char ltrim(const char s)	303	static const char ltrim(const char s)
284	{	304	{
285	while (isspace(*s))	305	while (isspace(*s))
@@ -400,6 +420,7 @@ static const char * const reg_type_str[] = { · 114 unmodified lines
400	[PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",	420	[PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
401	[PTR_TO_TP_BUFFER] = "tp_buffer",	421	[PTR_TO_TP_BUFFER] = "tp_buffer",
402	[PTR_TO_XDP_SOCK] = "xdp_sock",	422	[PTR_TO_XDP_SOCK] = "xdp_sock",
		423	[PTR_TO_BTF_ID] = "ptr_",
403	};	424	};
404		425
405	static char slot_type_char[] = {	426	static char slot_type_char[] = {
@@ -430,6 +451,12 @@ static struct bpf_func_state func(struct bpf_verifier_env env, · 24 unmodified lines
430	return cur->frame[reg->frameno];	451	return cur->frame[reg->frameno];
431	}	452	}
432		453
		454	const char *kernel_type_name(u32 id)
		455	{
		456	return btf_name_by_offset(btf_vmlinux,
		457	btf_type_by_id(btf_vmlinux, id)->name_off);
		458	}
		459
433	static void print_verifier_state(struct bpf_verifier_env *env,	460	static void print_verifier_state(struct bpf_verifier_env *env,
434	const struct bpf_func_state *state)	461	const struct bpf_func_state *state)
435	{	462	{
@@ -454,6 +481,8 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 18 unmodified lines
454	/* reg->off should be 0 for SCALAR_VALUE */	481	/* reg->off should be 0 for SCALAR_VALUE */
455	verbose(env, "%lld", reg->var_off.value + reg->off);	482	verbose(env, "%lld", reg->var_off.value + reg->off);
456	} else {	483	} else {
		484	if (t == PTR_TO_BTF_ID)
		485	verbose(env, "%s", kernel_type_name(reg->btf_id));
457	verbose(env, "(id=%d", reg->id);	486	verbose(env, "(id=%d", reg->id);
458	if (reg_type_may_be_refcounted_or_null(t))	487	if (reg_type_may_be_refcounted_or_null(t))
459	verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);	488	verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
@@ -2331,10 +2360,12 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, · 1871 unmodified lines
2331		2360
2332	/* check access to 'struct bpf_context' fields. Supports fixed offsets only */	2361	/* check access to 'struct bpf_context' fields. Supports fixed offsets only */
2333	static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,	2362	static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
2334	enum bpf_access_type t, enum bpf_reg_type *reg_type)	2363	enum bpf_access_type t, enum bpf_reg_type *reg_type,
		2364	u32 *btf_id)
2335	{	2365	{
2336	struct bpf_insn_access_aux info = {	2366	struct bpf_insn_access_aux info = {
2337	.reg_type = *reg_type,	2367	.reg_type = *reg_type,
		2368	.log = &env->log,
2338	};	2369	};
2339		2370
2340	if (env->ops->is_valid_access &&	2371	if (env->ops->is_valid_access &&
@@ -2348,7 +2379,10 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, · 7 unmodified lines
2348	*/	2379	*/
2349	*reg_type = info.reg_type;	2380	*reg_type = info.reg_type;
2350		2381
2351	env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;	2382	if (*reg_type == PTR_TO_BTF_ID)
		2383	*btf_id = info.btf_id;
		2384	else
		2385	env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
2352	/* remember the offset of last byte accessed in ctx */	2386	/* remember the offset of last byte accessed in ctx */
2353	if (env->prog->aux->max_ctx_offset < off + size)	2387	if (env->prog->aux->max_ctx_offset < off + size)
2354	env->prog->aux->max_ctx_offset = off + size;	2388	env->prog->aux->max_ctx_offset = off + size;
@@ -2774,6 +2808,53 @@ static int bpf_map_direct_read(struct bpf_map map, int off, int size, u64 val) · 419 unmodified lines
2774	return 0;	2808	return 0;
2775	}	2809	}
2776		2810
		2811	static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
		2812	struct bpf_reg_state *regs,
		2813	int regno, int off, int size,
		2814	enum bpf_access_type atype,
		2815	int value_regno)
		2816	{
		2817	struct bpf_reg_state *reg = regs + regno;
		2818	const struct btf_type *t = btf_type_by_id(btf_vmlinux, reg->btf_id);
		2819	const char *tname = btf_name_by_offset(btf_vmlinux, t->name_off);
		2820	u32 btf_id;
		2821	int ret;
		2822
		2823	if (atype != BPF_READ) {
		2824	verbose(env, "only read is supported\n");
		2825	return -EACCES;
		2826	}
		2827
		2828	if (off < 0) {
		2829	verbose(env,
		2830	"R%d is ptr_%s invalid negative access: off=%d\n",
		2831	regno, tname, off);
		2832	return -EACCES;
		2833	}
		2834	if (!tnum_is_const(reg->var_off) \|\| reg->var_off.value) {
		2835	char tn_buf[48];
		2836
		2837	tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
		2838	verbose(env,
		2839	"R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
		2840	regno, tname, off, tn_buf);
		2841	return -EACCES;
		2842	}
		2843
		2844	ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id);
		2845	if (ret < 0)
		2846	return ret;
		2847
		2848	if (ret == SCALAR_VALUE) {
		2849	mark_reg_unknown(env, regs, value_regno);
		2850	return 0;
		2851	}
		2852	mark_reg_known_zero(env, regs, value_regno);
		2853	regs[value_regno].type = PTR_TO_BTF_ID;
		2854	regs[value_regno].btf_id = btf_id;
		2855	return 0;
		2856	}
		2857
2777	/* check whether memory at (regno + off) is accessible for t = (read \| write)	2858	/* check whether memory at (regno + off) is accessible for t = (read \| write)
2778	* if t==write, value_regno is a register which value is stored into memory	2859	* if t==write, value_regno is a register which value is stored into memory
2779	* if t==read, value_regno is a register which will receive the value from memory	2860	* if t==read, value_regno is a register which will receive the value from memory
@@ -2834,6 +2915,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 54 unmodified lines
2834	}	2915	}
2835	} else if (reg->type == PTR_TO_CTX) {	2916	} else if (reg->type == PTR_TO_CTX) {
2836	enum bpf_reg_type reg_type = SCALAR_VALUE;	2917	enum bpf_reg_type reg_type = SCALAR_VALUE;
		2918	u32 btf_id = 0;
2837		2919
2838	if (t == BPF_WRITE && value_regno >= 0 &&	2920	if (t == BPF_WRITE && value_regno >= 0 &&
2839	is_pointer_value(env, value_regno)) {	2921	is_pointer_value(env, value_regno)) {
@@ -2845,7 +2927,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 5 unmodified lines
2845	if (err < 0)	2927	if (err < 0)
2846	return err;	2928	return err;
2847		2929
2848	err = check_ctx_access(env, insn_idx, off, size, t, &reg_type);	2930	err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf_id);
		2931	if (err)
		2932	verbose_linfo(env, insn_idx, "; ");
2849	if (!err && t == BPF_READ && value_regno >= 0) {	2933	if (!err && t == BPF_READ && value_regno >= 0) {
2850	/* ctx access returns either a scalar, or a	2934	/* ctx access returns either a scalar, or a
2851	* PTR_TO_PACKET[_META,_END]. In the latter	2935	* PTR_TO_PACKET[_META,_END]. In the latter
@@ -2864,6 +2948,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 12 unmodified lines
2864	* a sub-register.	2948	* a sub-register.
2865	*/	2949	*/
2866	regs[value_regno].subreg_def = DEF_NOT_SUBREG;	2950	regs[value_regno].subreg_def = DEF_NOT_SUBREG;
		2951	if (reg_type == PTR_TO_BTF_ID)
		2952	regs[value_regno].btf_id = btf_id;
2867	}	2953	}
2868	regs[value_regno].type = reg_type;	2954	regs[value_regno].type = reg_type;
2869	}	2955	}
@@ -2923,6 +3009,9 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn · 53 unmodified lines
2923	err = check_tp_buffer_access(env, reg, regno, off, size);	3009	err = check_tp_buffer_access(env, reg, regno, off, size);
2924	if (!err && t == BPF_READ && value_regno >= 0)	3010	if (!err && t == BPF_READ && value_regno >= 0)
2925	mark_reg_unknown(env, regs, value_regno);	3011	mark_reg_unknown(env, regs, value_regno);
		3012	} else if (reg->type == PTR_TO_BTF_ID) {
		3013	err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
		3014	value_regno);
2926	} else {	3015	} else {
2927	verbose(env, "R%d invalid mem access '%s'\n", regno,	3016	verbose(env, "R%d invalid mem access '%s'\n", regno,
2928	reg_type_str[reg->type]);	3017	reg_type_str[reg->type]);
@@ -3351,6 +3440,22 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno, · 422 unmodified lines
3351	expected_type = PTR_TO_SOCKET;	3440	expected_type = PTR_TO_SOCKET;
3352	if (type != expected_type)	3441	if (type != expected_type)
3353	goto err_type;	3442	goto err_type;
		3443	} else if (arg_type == ARG_PTR_TO_BTF_ID) {
		3444	expected_type = PTR_TO_BTF_ID;
		3445	if (type != expected_type)
		3446	goto err_type;
		3447	if (reg->btf_id != meta->btf_id) {
		3448	verbose(env, "Helper has type %s got %s in R%d\n",
		3449	kernel_type_name(meta->btf_id),
		3450	kernel_type_name(reg->btf_id), regno);
		3451
		3452	return -EACCES;
		3453	}
		3454	if (!tnum_is_const(reg->var_off) \|\| reg->var_off.value \|\| reg->off) {
		3455	verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",
		3456	regno);
		3457	return -EACCES;
		3458	}
3354	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {	3459	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
3355	if (meta->func_id == BPF_FUNC_spin_lock) {	3460	if (meta->func_id == BPF_FUNC_spin_lock) {
3356	if (process_spin_lock(env, regno, true))	3461	if (process_spin_lock(env, regno, true))
@@ -3498,6 +3603,7 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 141 unmodified lines
3498	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:	3603	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
3499	if (func_id != BPF_FUNC_perf_event_read &&	3604	if (func_id != BPF_FUNC_perf_event_read &&
3500	func_id != BPF_FUNC_perf_event_output &&	3605	func_id != BPF_FUNC_perf_event_output &&
		3606	func_id != BPF_FUNC_skb_output &&
3501	func_id != BPF_FUNC_perf_event_read_value)	3607	func_id != BPF_FUNC_perf_event_read_value)
3502	goto error;	3608	goto error;
3503	break;	3609	break;
@@ -3585,6 +3691,7 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 81 unmodified lines
3585	case BPF_FUNC_perf_event_read:	3691	case BPF_FUNC_perf_event_read:
3586	case BPF_FUNC_perf_event_output:	3692	case BPF_FUNC_perf_event_output:
3587	case BPF_FUNC_perf_event_read_value:	3693	case BPF_FUNC_perf_event_read_value:
		3694	case BPF_FUNC_skb_output:
3588	if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)	3695	if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
3589	goto error;	3696	goto error;
3590	break;	3697	break;
@@ -4039,21 +4146,16 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn · 448 unmodified lines
4039		4146
4040	meta.func_id = func_id;	4147	meta.func_id = func_id;
4041	/* check args */	4148	/* check args */
4042	err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta);	4149	for (i = 0; i < 5; i++) {
4043	if (err)	4150	if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID) {
4044	return err;	4151	if (!fn->btf_id[i])
4045	err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta);	4152	fn->btf_id[i] = btf_resolve_helper_id(&env->log, fn->func, i);
4046	if (err)	4153	meta.btf_id = fn->btf_id[i];
4047	return err;	4154	}
4048	err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta);	4155	err = check_func_arg(env, BPF_REG_1 + i, fn->arg_type[i], &meta);
4049	if (err)	4156	if (err)
4050	return err;	4157	return err;
4051	err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta);	4158	}
4052	if (err)
4053	return err;
4054	err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta);
4055	if (err)
4056	return err;
4057		4159
4058	err = record_func_map(env, &meta, func_id, insn_idx);	4160	err = record_func_map(env, &meta, func_id, insn_idx);
4059	if (err)	4161	if (err)
@@ -7493,6 +7595,7 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type) · 3433 unmodified lines
7493	case PTR_TO_TCP_SOCK:	7595	case PTR_TO_TCP_SOCK:
7494	case PTR_TO_TCP_SOCK_OR_NULL:	7596	case PTR_TO_TCP_SOCK_OR_NULL:
7495	case PTR_TO_XDP_SOCK:	7597	case PTR_TO_XDP_SOCK:
		7598	case PTR_TO_BTF_ID:
7496	return false;	7599	return false;
7497	default:	7600	default:
7498	return true;	7601	return true;
@@ -8634,6 +8737,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 1135 unmodified lines
8634	case PTR_TO_XDP_SOCK:	8737	case PTR_TO_XDP_SOCK:
8635	convert_ctx_access = bpf_xdp_sock_convert_ctx_access;	8738	convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
8636	break;	8739	break;
		8740	case PTR_TO_BTF_ID:
		8741	if (type == BPF_WRITE) {
		8742	verbose(env, "Writes through BTF pointers are not allowed\n");
		8743	return -EINVAL;
		8744	}
		8745	insn->code = BPF_LDX \| BPF_PROBE_MEM \| BPF_SIZE((insn)->code);
		8746	env->prog->aux->num_exentries++;
		8747	continue;
8637	default:	8748	default:
8638	continue;	8749	continue;
8639	}	8750	}
@@ -9261,6 +9372,52 @@ static void print_verification_stats(struct bpf_verifier_env *env) · 621 unmodified lines
9261	env->peak_states, env->longest_mark_read_walk);	9372	env->peak_states, env->longest_mark_read_walk);
9262	}	9373	}
9263		9374
		9375	static int check_attach_btf_id(struct bpf_verifier_env *env)
		9376	{
		9377	struct bpf_prog *prog = env->prog;
		9378	u32 btf_id = prog->aux->attach_btf_id;
		9379	const struct btf_type *t;
		9380	const char *tname;
		9381
		9382	if (prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT && btf_id) {
		9383	const char prefix[] = "btf_trace_";
		9384
		9385	t = btf_type_by_id(btf_vmlinux, btf_id);
		9386	if (!t) {
		9387	verbose(env, "attach_btf_id %u is invalid\n", btf_id);
		9388	return -EINVAL;
		9389	}
		9390	if (!btf_type_is_typedef(t)) {
		9391	verbose(env, "attach_btf_id %u is not a typedef\n",
		9392	btf_id);
		9393	return -EINVAL;
		9394	}
		9395	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
		9396	if (!tname \|\| strncmp(prefix, tname, sizeof(prefix) - 1)) {
		9397	verbose(env, "attach_btf_id %u points to wrong type name %s\n",
		9398	btf_id, tname);
		9399	return -EINVAL;
		9400	}
		9401	tname += sizeof(prefix) - 1;
		9402	t = btf_type_by_id(btf_vmlinux, t->type);
		9403	if (!btf_type_is_ptr(t))
		9404	/* should never happen in valid vmlinux build */
		9405	return -EINVAL;
		9406	t = btf_type_by_id(btf_vmlinux, t->type);
		9407	if (!btf_type_is_func_proto(t))
		9408	/* should never happen in valid vmlinux build */
		9409	return -EINVAL;
		9410
		9411	/* remember two read only pointers that are valid for
		9412	* the life time of the kernel
		9413	*/
		9414	prog->aux->attach_func_name = tname;
		9415	prog->aux->attach_func_proto = t;
		9416	prog->aux->attach_btf_trace = true;
		9417	}
		9418	return 0;
		9419	}
		9420
9264	int bpf_check(struct bpf_prog *prog, union bpf_attr attr,	9421	int bpf_check(struct bpf_prog *prog, union bpf_attr attr,
9265	union bpf_attr __user *uattr)	9422	union bpf_attr __user *uattr)
9266	{	9423	{
@@ -9294,6 +9451,13 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 27 unmodified lines
9294	env->ops = bpf_verifier_ops[env->prog->type];	9451	env->ops = bpf_verifier_ops[env->prog->type];
9295	is_priv = capable(CAP_SYS_ADMIN);	9452	is_priv = capable(CAP_SYS_ADMIN);
9296		9453
		9454	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
		9455	mutex_lock(&bpf_verifier_lock);
		9456	if (!btf_vmlinux)
		9457	btf_vmlinux = btf_parse_vmlinux();
		9458	mutex_unlock(&bpf_verifier_lock);
		9459	}
		9460
9297	/* grab the mutex to protect few globals used by verifier */	9461	/* grab the mutex to protect few globals used by verifier */
9298	if (!is_priv)	9462	if (!is_priv)
9299	mutex_lock(&bpf_verifier_lock);	9463	mutex_lock(&bpf_verifier_lock);
@@ -9313,6 +9477,17 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 13 unmodified lines
9313	goto err_unlock;	9477	goto err_unlock;
9314	}	9478	}
9315		9479
		9480	if (IS_ERR(btf_vmlinux)) {
		9481	/* Either gcc or pahole or kernel are broken. */
		9482	verbose(env, "in-kernel BTF is malformed\n");
		9483	ret = PTR_ERR(btf_vmlinux);
		9484	goto skip_full_check;
		9485	}
		9486
		9487	ret = check_attach_btf_id(env);
		9488	if (ret)
		9489	goto skip_full_check;
		9490
9316	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);	9491	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
9317	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))	9492	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
9318	env->strict_alignment = true;	9493	env->strict_alignment = true;

kernel/extable.c+2−0

@@ -56,6 +56,8 @@ const struct exception_table_entry *search_exception_tables(unsigned long addr) · 55 unmodified lines
56	e = search_kernel_exception_table(addr);	56	e = search_kernel_exception_table(addr);
57	if (!e)	57	if (!e)
58	e = search_module_extables(addr);	58	e = search_module_extables(addr);
		59	if (!e)
		60	e = search_bpf_extables(addr);
59	return e;	61	return e;
60	}	62	}
61		63

kernel/trace/bpf_trace.c+9−1

 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
 };
+extern const struct bpf_func_proto bpf_skb_output_proto;
 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
 	   struct bpf_map *, map, u64, flags)
 {
 	switch (func_id) {
 	case BPF_FUNC_perf_event_output:
 		return &bpf_perf_event_output_proto_raw_tp;
+#ifdef CONFIG_NET
+	case BPF_FUNC_skb_output:
+		return &bpf_skb_output_proto;
+#endif
 	case BPF_FUNC_get_stackid:
 		return &bpf_get_stackid_proto_raw_tp;
 	case BPF_FUNC_get_stack:
 		return false;
 	if (off % size != 0)
 		return false;
-	return true;
+	if (!prog->aux->attach_btf_id)
+		return true;
+	return btf_ctx_access(off, size, type, prog, info);
 }
 const struct bpf_verifier_ops raw_tracepoint_verifier_ops = {

net/bpf/test_run.c+9−0

 	if (!range_is_zero(__skb, offsetof(struct __sk_buff, cb) +
 			   FIELD_SIZEOF(struct __sk_buff, cb),
+			   offsetof(struct __sk_buff, tstamp)))
+		return -EINVAL;
+	/* tstamp is allowed */
+	if (!range_is_zero(__skb, offsetof(struct __sk_buff, tstamp) +
+			   FIELD_SIZEOF(struct __sk_buff, tstamp),
 			   sizeof(struct __sk_buff)))
 		return -EINVAL;
 	skb->priority = __skb->priority;
+	skb->tstamp = __skb->tstamp;
 	memcpy(&cb->data, __skb->cb, QDISC_CB_PRIV_LEN);
 	return 0;
 		return;
 	__skb->priority = skb->priority;
+	__skb->tstamp = skb->tstamp;
 	memcpy(__skb->cb, &cb->data, QDISC_CB_PRIV_LEN);
 }

net/core/filter.c+14−1

 	if (unlikely(flags & ~(BPF_F_CTXLEN_MASK | BPF_F_INDEX_MASK)))
 		return -EINVAL;
-	if (unlikely(skb_size > skb->len))
+	if (unlikely(!skb || skb_size > skb->len))
 		return -EFAULT;
 	return bpf_event_output(map, flags, meta, meta_size, skb, skb_size,
 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
 };
+static u32 bpf_skb_output_btf_ids[5];
+const struct bpf_func_proto bpf_skb_output_proto = {
+	.func		= bpf_skb_event_output,
+	.gpl_only	= true,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_BTF_ID,
+	.arg2_type	= ARG_CONST_MAP_PTR,
+	.arg3_type	= ARG_ANYTHING,
+	.arg4_type	= ARG_PTR_TO_MEM,
+	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
+	.btf_id		= bpf_skb_output_btf_ids,
+};
 static unsigned short bpf_tunnel_key_af(u64 flags)
 {
 	return flags & BPF_F_TUNINFO_IPV6 ? AF_INET6 : AF_INET;

scripts/bpf_helpers_doc.py+2−2

@@ -488,8 +488,8 @@ class PrinterHelpers(Printer): · 487 unmodified lines
488	return t	488	return t
489	if t in self.mapped_types:	489	if t in self.mapped_types:
490	return self.mapped_types[t]	490	return self.mapped_types[t]
491	print("")	491	print("Unrecognized type '%s', please add it to known types!" % t,
492	print("Unrecognized type '%s', please add it to known types!" % t)	492	file=sys.stderr)
493	sys.exit(1)	493	sys.exit(1)
494		494
495	seen_helpers = set()	495	seen_helpers = set()

tools/bpf/bpftool/btf.c+56−1

@@ -12,6 +12,9 @@ · 11 unmodified lines
12	#include <libbpf.h>	12	#include <libbpf.h>
13	#include <linux/btf.h>	13	#include <linux/btf.h>
14	#include <linux/hashtable.h>	14	#include <linux/hashtable.h>
		15	#include <sys/types.h>
		16	#include <sys/stat.h>
		17	#include <unistd.h>
15		18
16	#include "btf.h"	19	#include "btf.h"
17	#include "json_writer.h"	20	#include "json_writer.h"
@@ -388,6 +391,54 @@ done: · 370 unmodified lines
388	return err;	391	return err;
389	}	392	}
390		393
		394	static struct btf btf__parse_raw(const char file)
		395	{
		396	struct btf *btf;
		397	struct stat st;
		398	__u8 *buf;
		399	FILE *f;
		400
		401	if (stat(file, &st))
		402	return NULL;
		403
		404	f = fopen(file, "rb");
		405	if (!f)
		406	return NULL;
		407
		408	buf = malloc(st.st_size);
		409	if (!buf) {
		410	btf = ERR_PTR(-ENOMEM);
		411	goto exit_close;
		412	}
		413
		414	if ((size_t) st.st_size != fread(buf, 1, st.st_size, f)) {
		415	btf = ERR_PTR(-EINVAL);
		416	goto exit_free;
		417	}
		418
		419	btf = btf__new(buf, st.st_size);
		420
		421	exit_free:
		422	free(buf);
		423	exit_close:
		424	fclose(f);
		425	return btf;
		426	}
		427
		428	static bool is_btf_raw(const char *file)
		429	{
		430	__u16 magic = 0;
		431	int fd;
		432
		433	fd = open(file, O_RDONLY);
		434	if (fd < 0)
		435	return false;
		436
		437	read(fd, &magic, sizeof(magic));
		438	close(fd);
		439	return magic == BTF_MAGIC;
		440	}
		441
391	static int do_dump(int argc, char **argv)	442	static int do_dump(int argc, char **argv)
392	{	443	{
393	struct btf *btf = NULL;	444	struct btf *btf = NULL;
@@ -465,7 +516,11 @@ static int do_dump(int argc, char **argv) · 71 unmodified lines
465	}	516	}
466	NEXT_ARG();	517	NEXT_ARG();
467	} else if (is_prefix(src, "file")) {	518	} else if (is_prefix(src, "file")) {
468	btf = btf__parse_elf(*argv, NULL);	519	if (is_btf_raw(*argv))
		520	btf = btf__parse_raw(*argv);
		521	else
		522	btf = btf__parse_elf(*argv, NULL);
		523
469	if (IS_ERR(btf)) {	524	if (IS_ERR(btf)) {
470	err = PTR_ERR(btf);	525	err = PTR_ERR(btf);
471	btf = NULL;	526	btf = NULL;

tools/bpf/bpftool/prog.c+4−4

 static int load_with_options(int argc, char **argv, bool first_prog_only)
 {
-	struct bpf_object_load_attr load_attr = { 0 };
 	enum bpf_prog_type common_prog_type = BPF_PROG_TYPE_UNSPEC;
+	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
+		.relaxed_maps = relaxed_maps,
+	);
+	struct bpf_object_load_attr load_attr = { 0 };
 	enum bpf_attach_type expected_attach_type;
 	struct map_replace *map_replace = NULL;
 	struct bpf_program *prog = NULL, *pos;
 	const char *file;
 	int idx, err;
-	LIBBPF_OPTS(bpf_object_open_opts, open_opts,
-		.relaxed_maps = relaxed_maps,
-	);
 	if (!REQ_ARGS(2))
 		return -1;

tools/include/uapi/linux/bpf.h+27−1

@@ -420,6 +420,7 @@ union bpf_attr { · 419 unmodified lines
420	__u32 line_info_rec_size; /* userspace bpf_line_info size */	420	__u32 line_info_rec_size; /* userspace bpf_line_info size */
421	__aligned_u64 line_info; /* line info */	421	__aligned_u64 line_info; /* line info */
422	__u32 line_info_cnt; /* number of bpf_line_info records */	422	__u32 line_info_cnt; /* number of bpf_line_info records */
		423	__u32 attach_btf_id; /* in-kernel BTF type id to attach to */
423	};	424	};
424		425
425	struct { /* anonymous struct used by BPF_OBJ_* commands */	426	struct { /* anonymous struct used by BPF_OBJ_* commands */
@@ -2750,6 +2751,30 @@ union bpf_attr { · 2324 unmodified lines
2750	* -EOPNOTSUPP kernel configuration does not enable SYN cookies	2751	* -EOPNOTSUPP kernel configuration does not enable SYN cookies
2751	*	2752	*
2752	* -EPROTONOSUPPORT IP packet version is not 4 or 6	2753	* -EPROTONOSUPPORT IP packet version is not 4 or 6
		2754	*
		2755	* int bpf_skb_output(void ctx, struct bpf_map map, u64 flags, void *data, u64 size)
		2756	* Description
		2757	* Write raw data blob into a special BPF perf event held by
		2758	* map of type BPF_MAP_TYPE_PERF_EVENT_ARRAY. This perf
		2759	* event must have the following attributes: PERF_SAMPLE_RAW
		2760	* as sample_type, PERF_TYPE_SOFTWARE as type, and
		2761	* PERF_COUNT_SW_BPF_OUTPUT as config.
		2762	*
		2763	* The flags are used to indicate the index in map for which
		2764	* the value must be put, masked with BPF_F_INDEX_MASK.
		2765	* Alternatively, flags can be set to BPF_F_CURRENT_CPU
		2766	* to indicate that the index of the current CPU core should be
		2767	* used.
		2768	*
		2769	* The value to write, of size, is passed through eBPF stack and
		2770	* pointed by data.
		2771	*
		2772	* ctx is a pointer to in-kernel struct sk_buff.
		2773	*
		2774	* This helper is similar to bpf_perf_event_output\ () but
		2775	* restricted to raw_tracepoint bpf programs.
		2776	* Return
		2777	* 0 on success, or a negative error in case of failure.
2753	*/	2778	*/
2754	#define __BPF_FUNC_MAPPER(FN) \	2779	#define __BPF_FUNC_MAPPER(FN) \
2755	FN(unspec), \	2780	FN(unspec), \
@@ -2862,7 +2887,8 @@ union bpf_attr { · 106 unmodified lines
2862	FN(sk_storage_get), \	2887	FN(sk_storage_get), \
2863	FN(sk_storage_delete), \	2888	FN(sk_storage_delete), \
2864	FN(send_signal), \	2889	FN(send_signal), \
2865	FN(tcp_gen_syncookie),	2890	FN(tcp_gen_syncookie), \
		2891	FN(skb_output),
2866		2892
2867	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	2893	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2868	* function eBPF program intends to call	2894	* function eBPF program intends to call

tools/include/uapi/linux/if_link.h+2−0

@@ -167,6 +167,8 @@ enum { · 166 unmodified lines
167	IFLA_NEW_IFINDEX,	167	IFLA_NEW_IFINDEX,
168	IFLA_MIN_MTU,	168	IFLA_MIN_MTU,
169	IFLA_MAX_MTU,	169	IFLA_MAX_MTU,
		170	IFLA_PROP_LIST,
		171	IFLA_ALT_IFNAME, /* Alternative ifname */
170	__IFLA_MAX	172	__IFLA_MAX
171	};	173	};
172		174

tools/lib/bpf/Makefile+3−0

@@ -300,3 +300,6 @@ tags: · 299 unmodified lines
300	# Declare the contents of the .PHONY variable as phony. We keep that	300	# Declare the contents of the .PHONY variable as phony. We keep that
301	# information in a variable so we can use it in if_changed and friends.	301	# information in a variable so we can use it in if_changed and friends.
302	.PHONY: $(PHONY)	302	.PHONY: $(PHONY)
		303
		304	# Delete partially updated (corrupted) files on error
		305	.DELETE_ON_ERROR:

tools/lib/bpf/bpf.c+3−0

 	memset(&attr, 0, sizeof(attr));
 	attr.prog_type = load_attr->prog_type;
 	attr.expected_attach_type = load_attr->expected_attach_type;
+	if (attr.prog_type == BPF_PROG_TYPE_RAW_TRACEPOINT)
+		/* expected_attach_type is ignored for tracing progs */
+		attr.attach_btf_id = attr.expected_attach_type;
 	attr.insn_cnt = (__u32)load_attr->insns_cnt;
 	attr.insns = ptr_to_u64(load_attr->insns);
 	attr.license = ptr_to_u64(load_attr->license);

tools/lib/bpf/bpf_core_read.h+23−1

@@ -2,6 +2,28 @@ · 1 unmodified line
2	#ifndef __BPF_CORE_READ_H__	2	#ifndef __BPF_CORE_READ_H__
3	#define __BPF_CORE_READ_H__	3	#define __BPF_CORE_READ_H__
4		4
		5	/*
		6	* enum bpf_field_info_kind is passed as a second argument into
		7	* __builtin_preserve_field_info() built-in to get a specific aspect of
		8	* a field, captured as a first argument. __builtin_preserve_field_info(field,
		9	* info_kind) returns __u32 integer and produces BTF field relocation, which
		10	* is understood and processed by libbpf during BPF object loading. See
		11	* selftests/bpf for examples.
		12	*/
		13	enum bpf_field_info_kind {
		14	BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */
		15	BPF_FIELD_EXISTS = 2, /* field existence in target kernel */
		16	};
		17
		18	/*
		19	* Convenience macro to check that field actually exists in target kernel's.
		20	* Returns:
		21	* 1, if matching field is present in target kernel;
		22	* 0, if no matching field found.
		23	*/
		24	#define bpf_core_field_exists(field) \
		25	__builtin_preserve_field_info(field, BPF_FIELD_EXISTS)
		26
5	/*	27	/*
6	* bpf_core_read() abstracts away bpf_probe_read() call and captures offset	28	* bpf_core_read() abstracts away bpf_probe_read() call and captures offset
7	* relocation for source address using __builtin_preserve_access_index()	29	* relocation for source address using __builtin_preserve_access_index()
@@ -12,7 +34,7 @@ · 4 unmodified lines
12	* a relocation, which records BTF type ID describing root struct/union and an	34	* a relocation, which records BTF type ID describing root struct/union and an
13	* accessor string which describes exact embedded field that was used to take	35	* accessor string which describes exact embedded field that was used to take
14	* an address. See detailed description of this relocation format and	36	* an address. See detailed description of this relocation format and
15	* semantics in comments to struct bpf_offset_reloc in libbpf_internal.h.	37	* semantics in comments to struct bpf_field_reloc in libbpf_internal.h.
16	*	38	*
17	* This relocation allows libbpf to adjust BPF instruction to use correct	39	* This relocation allows libbpf to adjust BPF instruction to use correct
18	* actual field offset, based on target kernel BTF type that matches original	40	* actual field offset, based on target kernel BTF type that matches original

tools/lib/bpf/btf.c+36−36

@@ -390,14 +390,14 @@ struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext) · 389 unmodified lines
390	GElf_Ehdr ehdr;	390	GElf_Ehdr ehdr;
391		391
392	if (elf_version(EV_CURRENT) == EV_NONE) {	392	if (elf_version(EV_CURRENT) == EV_NONE) {
393	pr_warning("failed to init libelf for %s\n", path);	393	pr_warn("failed to init libelf for %s\n", path);
394	return ERR_PTR(-LIBBPF_ERRNO__LIBELF);	394	return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
395	}	395	}
396		396
397	fd = open(path, O_RDONLY);	397	fd = open(path, O_RDONLY);
398	if (fd < 0) {	398	if (fd < 0) {
399	err = -errno;	399	err = -errno;
400	pr_warning("failed to open %s: %s\n", path, strerror(errno));	400	pr_warn("failed to open %s: %s\n", path, strerror(errno));
401	return ERR_PTR(err);	401	return ERR_PTR(err);
402	}	402	}
403		403
@@ -405,19 +405,19 @@ struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext) · 1 unmodified line
405		405
406	elf = elf_begin(fd, ELF_C_READ, NULL);	406	elf = elf_begin(fd, ELF_C_READ, NULL);
407	if (!elf) {	407	if (!elf) {
408	pr_warning("failed to open %s as ELF file\n", path);	408	pr_warn("failed to open %s as ELF file\n", path);
409	goto done;	409	goto done;
410	}	410	}
411	if (!gelf_getehdr(elf, &ehdr)) {	411	if (!gelf_getehdr(elf, &ehdr)) {
412	pr_warning("failed to get EHDR from %s\n", path);	412	pr_warn("failed to get EHDR from %s\n", path);
413	goto done;	413	goto done;
414	}	414	}
415	if (!btf_check_endianness(&ehdr)) {	415	if (!btf_check_endianness(&ehdr)) {
416	pr_warning("non-native ELF endianness is not supported\n");	416	pr_warn("non-native ELF endianness is not supported\n");
417	goto done;	417	goto done;
418	}	418	}
419	if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {	419	if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {
420	pr_warning("failed to get e_shstrndx from %s\n", path);	420	pr_warn("failed to get e_shstrndx from %s\n", path);
421	goto done;	421	goto done;
422	}	422	}
423		423
@@ -427,29 +427,29 @@ struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext) · 3 unmodified lines
427		427
428	idx++;	428	idx++;
429	if (gelf_getshdr(scn, &sh) != &sh) {	429	if (gelf_getshdr(scn, &sh) != &sh) {
430	pr_warning("failed to get section(%d) header from %s\n",	430	pr_warn("failed to get section(%d) header from %s\n",
431	idx, path);	431	idx, path);
432	goto done;	432	goto done;
433	}	433	}
434	name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name);	434	name = elf_strptr(elf, ehdr.e_shstrndx, sh.sh_name);
435	if (!name) {	435	if (!name) {
436	pr_warning("failed to get section(%d) name from %s\n",	436	pr_warn("failed to get section(%d) name from %s\n",
437	idx, path);	437	idx, path);
438	goto done;	438	goto done;
439	}	439	}
440	if (strcmp(name, BTF_ELF_SEC) == 0) {	440	if (strcmp(name, BTF_ELF_SEC) == 0) {
441	btf_data = elf_getdata(scn, 0);	441	btf_data = elf_getdata(scn, 0);
442	if (!btf_data) {	442	if (!btf_data) {
443	pr_warning("failed to get section(%d, %s) data from %s\n",	443	pr_warn("failed to get section(%d, %s) data from %s\n",
444	idx, name, path);	444	idx, name, path);
445	goto done;	445	goto done;
446	}	446	}
447	continue;	447	continue;
448	} else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) {	448	} else if (btf_ext && strcmp(name, BTF_EXT_ELF_SEC) == 0) {
449	btf_ext_data = elf_getdata(scn, 0);	449	btf_ext_data = elf_getdata(scn, 0);
450	if (!btf_ext_data) {	450	if (!btf_ext_data) {
451	pr_warning("failed to get section(%d, %s) data from %s\n",	451	pr_warn("failed to get section(%d, %s) data from %s\n",
452	idx, name, path);	452	idx, name, path);
453	goto done;	453	goto done;
454	}	454	}
455	continue;	455	continue;
@@ -600,9 +600,9 @@ int btf__load(struct btf *btf) · 144 unmodified lines
600	log_buf, log_buf_size, false);	600	log_buf, log_buf_size, false);
601	if (btf->fd < 0) {	601	if (btf->fd < 0) {
602	err = -errno;	602	err = -errno;
603	pr_warning("Error loading BTF: %s(%d)\n", strerror(errno), errno);	603	pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno);
604	if (*log_buf)	604	if (*log_buf)
605	pr_warning("%s\n", log_buf);	605	pr_warn("%s\n", log_buf);
606	goto done;	606	goto done;
607	}	607	}
608		608
@@ -707,8 +707,8 @@ int btf__get_map_kv_tids(const struct btf btf, const char map_name, · 98 unmodified lines
707		707
708	if (snprintf(container_name, max_name, "____btf_map_%s", map_name) ==	708	if (snprintf(container_name, max_name, "____btf_map_%s", map_name) ==
709	max_name) {	709	max_name) {
710	pr_warning("map:%s length of '____btf_map_%s' is too long\n",	710	pr_warn("map:%s length of '____btf_map_%s' is too long\n",
711	map_name, map_name);	711	map_name, map_name);
712	return -EINVAL;	712	return -EINVAL;
713	}	713	}
714		714
@@ -721,14 +721,14 @@ int btf__get_map_kv_tids(const struct btf btf, const char map_name, · 6 unmodified lines
721		721
722	container_type = btf__type_by_id(btf, container_id);	722	container_type = btf__type_by_id(btf, container_id);
723	if (!container_type) {	723	if (!container_type) {
724	pr_warning("map:%s cannot find BTF type for container_id:%u\n",	724	pr_warn("map:%s cannot find BTF type for container_id:%u\n",
725	map_name, container_id);	725	map_name, container_id);
726	return -EINVAL;	726	return -EINVAL;
727	}	727	}
728		728
729	if (!btf_is_struct(container_type) \|\| btf_vlen(container_type) < 2) {	729	if (!btf_is_struct(container_type) \|\| btf_vlen(container_type) < 2) {
730	pr_warning("map:%s container_name:%s is an invalid container struct\n",	730	pr_warn("map:%s container_name:%s is an invalid container struct\n",
731	map_name, container_name);	731	map_name, container_name);
732	return -EINVAL;	732	return -EINVAL;
733	}	733	}
734		734
@@ -737,25 +737,25 @@ int btf__get_map_kv_tids(const struct btf btf, const char map_name, · 2 unmodified lines
737		737
738	key_size = btf__resolve_size(btf, key->type);	738	key_size = btf__resolve_size(btf, key->type);
739	if (key_size < 0) {	739	if (key_size < 0) {
740	pr_warning("map:%s invalid BTF key_type_size\n", map_name);	740	pr_warn("map:%s invalid BTF key_type_size\n", map_name);
741	return key_size;	741	return key_size;
742	}	742	}
743		743
744	if (expected_key_size != key_size) {	744	if (expected_key_size != key_size) {
745	pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",	745	pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
746	map_name, (__u32)key_size, expected_key_size);	746	map_name, (__u32)key_size, expected_key_size);
747	return -EINVAL;	747	return -EINVAL;
748	}	748	}
749		749
750	value_size = btf__resolve_size(btf, value->type);	750	value_size = btf__resolve_size(btf, value->type);
751	if (value_size < 0) {	751	if (value_size < 0) {
752	pr_warning("map:%s invalid BTF value_type_size\n", map_name);	752	pr_warn("map:%s invalid BTF value_type_size\n", map_name);
753	return value_size;	753	return value_size;
754	}	754	}
755		755
756	if (expected_value_size != value_size) {	756	if (expected_value_size != value_size) {
757	pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",	757	pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
758	map_name, (__u32)value_size, expected_value_size);	758	map_name, (__u32)value_size, expected_value_size);
759	return -EINVAL;	759	return -EINVAL;
760	}	760	}
761		761
@@ -888,14 +888,14 @@ static int btf_ext_setup_line_info(struct btf_ext *btf_ext) · 126 unmodified lines
888	return btf_ext_setup_info(btf_ext, &param);	888	return btf_ext_setup_info(btf_ext, &param);
889	}	889	}
890		890
891	static int btf_ext_setup_offset_reloc(struct btf_ext *btf_ext)	891	static int btf_ext_setup_field_reloc(struct btf_ext *btf_ext)
892	{	892	{
893	struct btf_ext_sec_setup_param param = {	893	struct btf_ext_sec_setup_param param = {
894	.off = btf_ext->hdr->offset_reloc_off,	894	.off = btf_ext->hdr->field_reloc_off,
895	.len = btf_ext->hdr->offset_reloc_len,	895	.len = btf_ext->hdr->field_reloc_len,
896	.min_rec_size = sizeof(struct bpf_offset_reloc),	896	.min_rec_size = sizeof(struct bpf_field_reloc),
897	.ext_info = &btf_ext->offset_reloc_info,	897	.ext_info = &btf_ext->field_reloc_info,
898	.desc = "offset_reloc",	898	.desc = "field_reloc",
899	};	899	};
900		900
901	return btf_ext_setup_info(btf_ext, &param);	901	return btf_ext_setup_info(btf_ext, &param);
@@ -975,9 +975,9 @@ struct btf_ext btf_ext__new(__u8 data, __u32 size) · 73 unmodified lines
975	goto done;	975	goto done;
976		976
977	if (btf_ext->hdr->hdr_len <	977	if (btf_ext->hdr->hdr_len <
978	offsetofend(struct btf_ext_header, offset_reloc_len))	978	offsetofend(struct btf_ext_header, field_reloc_len))
979	goto done;	979	goto done;
980	err = btf_ext_setup_offset_reloc(btf_ext);	980	err = btf_ext_setup_field_reloc(btf_ext);
981	if (err)	981	if (err)
982	goto done;	982	goto done;
983		983

tools/lib/bpf/btf.h+2−2

@@ -60,8 +60,8 @@ struct btf_ext_header { · 59 unmodified lines
60	__u32 line_info_len;	60	__u32 line_info_len;
61		61
62	/* optional part of .BTF.ext header */	62	/* optional part of .BTF.ext header */
63	__u32 offset_reloc_off;	63	__u32 field_reloc_off;
64	__u32 offset_reloc_len;	64	__u32 field_reloc_len;
65	};	65	};
66		66
67	LIBBPF_API void btf__free(struct btf *btf);	67	LIBBPF_API void btf__free(struct btf *btf);

tools/lib/bpf/btf_dump.c+9−9

@@ -428,7 +428,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) · 427 unmodified lines
428	/* type loop, but resolvable through fwd declaration */	428	/* type loop, but resolvable through fwd declaration */
429	if (btf_is_composite(t) && through_ptr && t->name_off != 0)	429	if (btf_is_composite(t) && through_ptr && t->name_off != 0)
430	return 0;	430	return 0;
431	pr_warning("unsatisfiable type cycle, id:[%u]\n", id);	431	pr_warn("unsatisfiable type cycle, id:[%u]\n", id);
432	return -ELOOP;	432	return -ELOOP;
433	}	433	}
434		434
@@ -636,8 +636,8 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) · 201 unmodified lines
636	if (id == cont_id)	636	if (id == cont_id)
637	return;	637	return;
638	if (t->name_off == 0) {	638	if (t->name_off == 0) {
639	pr_warning("anonymous struct/union loop, id:[%u]\n",	639	pr_warn("anonymous struct/union loop, id:[%u]\n",
640	id);	640	id);
641	return;	641	return;
642	}	642	}
643	btf_dump_emit_struct_fwd(d, id, t);	643	btf_dump_emit_struct_fwd(d, id, t);
@@ -782,7 +782,7 @@ static int btf_align_of(const struct btf *btf, __u32 id) · 138 unmodified lines
782	return align;	782	return align;
783	}	783	}
784	default:	784	default:
785	pr_warning("unsupported BTF_KIND:%u\n", btf_kind(t));	785	pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
786	return 1;	786	return 1;
787	}	787	}
788	}	788	}
@@ -1067,7 +1067,7 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, · 278 unmodified lines
1067	* chain, restore stack, emit warning, and try to	1067	* chain, restore stack, emit warning, and try to
1068	* proceed nevertheless	1068	* proceed nevertheless
1069	*/	1069	*/
1070	pr_warning("not enough memory for decl stack:%d", err);	1070	pr_warn("not enough memory for decl stack:%d", err);
1071	d->decl_stack_cnt = stack_start;	1071	d->decl_stack_cnt = stack_start;
1072	return;	1072	return;
1073	}	1073	}
@@ -1096,8 +1096,8 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, · 22 unmodified lines
1096	case BTF_KIND_TYPEDEF:	1096	case BTF_KIND_TYPEDEF:
1097	goto done;	1097	goto done;
1098	default:	1098	default:
1099	pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n",	1099	pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n",
1100	btf_kind(t), id);	1100	btf_kind(t), id);
1101	goto done;	1101	goto done;
1102	}	1102	}
1103	}	1103	}
@@ -1323,8 +1323,8 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, · 219 unmodified lines
1323	return;	1323	return;
1324	}	1324	}
1325	default:	1325	default:
1326	pr_warning("unexpected type in decl chain, kind:%u, id:[%u]\n",	1326	pr_warn("unexpected type in decl chain, kind:%u, id:[%u]\n",
1327	kind, id);	1327	kind, id);
1328	return;	1328	return;
1329	}	1329	}
1330		1330

tools/lib/bpf/libbpf.c+541−440

@@ -249,6 +249,7 @@ struct bpf_object { · 248 unmodified lines
249		249
250	bool loaded;	250	bool loaded;
251	bool has_pseudo_calls;	251	bool has_pseudo_calls;
		252	bool relaxed_core_relocs;
252		253
253	/*	254	/*
254	* Information when doing elf related work. Only valid if fd	255	* Information when doing elf related work. Only valid if fd
@@ -311,8 +312,8 @@ void bpf_program__unload(struct bpf_program *prog) · 56 unmodified lines
311	for (i = 0; i < prog->instances.nr; i++)	312	for (i = 0; i < prog->instances.nr; i++)
312	zclose(prog->instances.fds[i]);	313	zclose(prog->instances.fds[i]);
313	} else if (prog->instances.nr != -1) {	314	} else if (prog->instances.nr != -1) {
314	pr_warning("Internal error: instances.nr is %d\n",	315	pr_warn("Internal error: instances.nr is %d\n",
315	prog->instances.nr);	316	prog->instances.nr);
316	}	317	}
317		318
318	prog->instances.nr = -1;	319	prog->instances.nr = -1;
@@ -363,8 +364,8 @@ bpf_program__init(void data, size_t size, char section_name, int idx, · 44 unmodified lines
363	const size_t bpf_insn_sz = sizeof(struct bpf_insn);	364	const size_t bpf_insn_sz = sizeof(struct bpf_insn);
364		365
365	if (size == 0 \|\| size % bpf_insn_sz) {	366	if (size == 0 \|\| size % bpf_insn_sz) {
366	pr_warning("corrupted section '%s', size: %zu\n",	367	pr_warn("corrupted section '%s', size: %zu\n",
367	section_name, size);	368	section_name, size);
368	return -EINVAL;	369	return -EINVAL;
369	}	370	}
370		371
@@ -372,22 +373,22 @@ bpf_program__init(void data, size_t size, char section_name, int idx, · 1 unmodified line
372		373
373	prog->section_name = strdup(section_name);	374	prog->section_name = strdup(section_name);
374	if (!prog->section_name) {	375	if (!prog->section_name) {
375	pr_warning("failed to alloc name for prog under section(%d) %s\n",	376	pr_warn("failed to alloc name for prog under section(%d) %s\n",
376	idx, section_name);	377	idx, section_name);
377	goto errout;	378	goto errout;
378	}	379	}
379		380
380	prog->pin_name = __bpf_program__pin_name(prog);	381	prog->pin_name = __bpf_program__pin_name(prog);
381	if (!prog->pin_name) {	382	if (!prog->pin_name) {
382	pr_warning("failed to alloc pin name for prog under section(%d) %s\n",	383	pr_warn("failed to alloc pin name for prog under section(%d) %s\n",
383	idx, section_name);	384	idx, section_name);
384	goto errout;	385	goto errout;
385	}	386	}
386		387
387	prog->insns = malloc(size);	388	prog->insns = malloc(size);
388	if (!prog->insns) {	389	if (!prog->insns) {
389	pr_warning("failed to alloc insns for prog under section %s\n",	390	pr_warn("failed to alloc insns for prog under section %s\n",
390	section_name);	391	section_name);
391	goto errout;	392	goto errout;
392	}	393	}
393	prog->insns_cnt = size / bpf_insn_sz;	394	prog->insns_cnt = size / bpf_insn_sz;
@@ -425,8 +426,8 @@ bpf_object__add_program(struct bpf_object obj, void data, size_t size, · 31 unmodified lines
425	* is still valid, so don't need special treat for	426	* is still valid, so don't need special treat for
426	* bpf_close_object().	427	* bpf_close_object().
427	*/	428	*/
428	pr_warning("failed to alloc a new program under section '%s'\n",	429	pr_warn("failed to alloc a new program under section '%s'\n",
429	section_name);	430	section_name);
430	bpf_program__exit(&prog);	431	bpf_program__exit(&prog);
431	return -ENOMEM;	432	return -ENOMEM;
432	}	433	}
@@ -466,8 +467,8 @@ bpf_object__init_prog_names(struct bpf_object *obj) · 33 unmodified lines
466	obj->efile.strtabidx,	467	obj->efile.strtabidx,
467	sym.st_name);	468	sym.st_name);
468	if (!name) {	469	if (!name) {
469	pr_warning("failed to get sym name string for prog %s\n",	470	pr_warn("failed to get sym name string for prog %s\n",
470	prog->section_name);	471	prog->section_name);
471	return -LIBBPF_ERRNO__LIBELF;	472	return -LIBBPF_ERRNO__LIBELF;
472	}	473	}
473	}	474	}
@@ -476,15 +477,15 @@ bpf_object__init_prog_names(struct bpf_object *obj) · 2 unmodified lines
476	name = ".text";	477	name = ".text";
477		478
478	if (!name) {	479	if (!name) {
479	pr_warning("failed to find sym for prog %s\n",	480	pr_warn("failed to find sym for prog %s\n",
480	prog->section_name);	481	prog->section_name);
481	return -EINVAL;	482	return -EINVAL;
482	}	483	}
483		484
484	prog->name = strdup(name);	485	prog->name = strdup(name);
485	if (!prog->name) {	486	if (!prog->name) {
486	pr_warning("failed to allocate memory for prog sym %s\n",	487	pr_warn("failed to allocate memory for prog sym %s\n",
487	name);	488	name);
488	return -ENOMEM;	489	return -ENOMEM;
489	}	490	}
490	}	491	}
@@ -513,7 +514,7 @@ static struct bpf_object bpf_object__new(const char path, · 22 unmodified lines
513		514
514	obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);	515	obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
515	if (!obj) {	516	if (!obj) {
516	pr_warning("alloc memory failed for %s\n", path);	517	pr_warn("alloc memory failed for %s\n", path);
517	return ERR_PTR(-ENOMEM);	518	return ERR_PTR(-ENOMEM);
518	}	519	}
519		520
@@ -580,7 +581,7 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 60 unmodified lines
580	GElf_Ehdr *ep;	581	GElf_Ehdr *ep;
581		582
582	if (obj_elf_valid(obj)) {	583	if (obj_elf_valid(obj)) {
583	pr_warning("elf init: internal error\n");	584	pr_warn("elf init: internal error\n");
584	return -LIBBPF_ERRNO__LIBELF;	585	return -LIBBPF_ERRNO__LIBELF;
585	}	586	}
586		587
@@ -598,7 +599,7 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 11 unmodified lines
598		599
599	err = -errno;	600	err = -errno;
600	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));	601	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
601	pr_warning("failed to open %s: %s\n", obj->path, cp);	602	pr_warn("failed to open %s: %s\n", obj->path, cp);
602	return err;	603	return err;
603	}	604	}
604		605
@@ -607,13 +608,13 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 2 unmodified lines
607	}	608	}
608		609
609	if (!obj->efile.elf) {	610	if (!obj->efile.elf) {
610	pr_warning("failed to open %s as ELF file\n", obj->path);	611	pr_warn("failed to open %s as ELF file\n", obj->path);
611	err = -LIBBPF_ERRNO__LIBELF;	612	err = -LIBBPF_ERRNO__LIBELF;
612	goto errout;	613	goto errout;
613	}	614	}
614		615
615	if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {	616	if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
616	pr_warning("failed to get EHDR from %s\n", obj->path);	617	pr_warn("failed to get EHDR from %s\n", obj->path);
617	err = -LIBBPF_ERRNO__FORMAT;	618	err = -LIBBPF_ERRNO__FORMAT;
618	goto errout;	619	goto errout;
619	}	620	}
@@ -622,7 +623,7 @@ static int bpf_object__elf_init(struct bpf_object *obj) · 2 unmodified lines
622	/* Old LLVM set e_machine to EM_NONE */	623	/* Old LLVM set e_machine to EM_NONE */
623	if (ep->e_type != ET_REL \|\|	624	if (ep->e_type != ET_REL \|\|
624	(ep->e_machine && ep->e_machine != EM_BPF)) {	625	(ep->e_machine && ep->e_machine != EM_BPF)) {
625	pr_warning("%s is not an eBPF object file\n", obj->path);	626	pr_warn("%s is not an eBPF object file\n", obj->path);
626	err = -LIBBPF_ERRNO__FORMAT;	627	err = -LIBBPF_ERRNO__FORMAT;
627	goto errout;	628	goto errout;
628	}	629	}
@@ -644,7 +645,7 @@ static int bpf_object__check_endianness(struct bpf_object *obj) · 15 unmodified lines
644	#else	645	#else
645	# error "Unrecognized __BYTE_ORDER__"	646	# error "Unrecognized __BYTE_ORDER__"
646	#endif	647	#endif
647	pr_warning("endianness mismatch.\n");	648	pr_warn("endianness mismatch.\n");
648	return -LIBBPF_ERRNO__ENDIAN;	649	return -LIBBPF_ERRNO__ENDIAN;
649	}	650	}
650		651
@@ -656,6 +657,21 @@ bpf_object__init_license(struct bpf_object obj, void data, size_t size) · 5 unmodified lines
656	return 0;	657	return 0;
657	}	658	}
658		659
		660	static int
		661	bpf_object__init_kversion(struct bpf_object obj, void data, size_t size)
		662	{
		663	__u32 kver;
		664
		665	if (size != sizeof(kver)) {
		666	pr_warn("invalid kver section in %s\n", obj->path);
		667	return -LIBBPF_ERRNO__FORMAT;
		668	}
		669	memcpy(&kver, data, sizeof(kver));
		670	obj->kern_version = kver;
		671	pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
		672	return 0;
		673	}
		674
659	static int compare_bpf_map(const void _a, const void _b)	675	static int compare_bpf_map(const void _a, const void _b)
660	{	676	{
661	const struct bpf_map *a = _a;	677	const struct bpf_map *a = _a;
@@ -689,15 +705,15 @@ static int bpf_object_search_section_size(const struct bpf_object *obj, · 27 unmodified lines
689		705
690	idx++;	706	idx++;
691	if (gelf_getshdr(scn, &sh) != &sh) {	707	if (gelf_getshdr(scn, &sh) != &sh) {
692	pr_warning("failed to get section(%d) header from %s\n",	708	pr_warn("failed to get section(%d) header from %s\n",
693	idx, obj->path);	709	idx, obj->path);
694	return -EIO;	710	return -EIO;
695	}	711	}
696		712
697	sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);	713	sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
698	if (!sec_name) {	714	if (!sec_name) {
699	pr_warning("failed to get section(%d) name from %s\n",	715	pr_warn("failed to get section(%d) name from %s\n",
700	idx, obj->path);	716	idx, obj->path);
701	return -EIO;	717	return -EIO;
702	}	718	}
703		719
@@ -706,8 +722,8 @@ static int bpf_object_search_section_size(const struct bpf_object *obj, · 2 unmodified lines
706		722
707	data = elf_getdata(scn, 0);	723	data = elf_getdata(scn, 0);
708	if (!data) {	724	if (!data) {
709	pr_warning("failed to get section(%d) data from %s(%s)\n",	725	pr_warn("failed to get section(%d) data from %s(%s)\n",
710	idx, name, obj->path);	726	idx, name, obj->path);
711	return -EIO;	727	return -EIO;
712	}	728	}
713		729
@@ -767,8 +783,8 @@ int bpf_object__variable_offset(const struct bpf_object obj, const char name, · 53 unmodified lines
767	sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx,	783	sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
768	sym.st_name);	784	sym.st_name);
769	if (!sname) {	785	if (!sname) {
770	pr_warning("failed to get sym name string for var %s\n",	786	pr_warn("failed to get sym name string for var %s\n",
771	name);	787	name);
772	return -EIO;	788	return -EIO;
773	}	789	}
774	if (strcmp(name, sname) == 0) {	790	if (strcmp(name, sname) == 0) {
@@ -792,7 +808,7 @@ static struct bpf_map bpf_object__add_map(struct bpf_object obj) · 17 unmodified lines
792	new_cap = max((size_t)4, obj->maps_cap * 3 / 2);	808	new_cap = max((size_t)4, obj->maps_cap * 3 / 2);
793	new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));	809	new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));
794	if (!new_maps) {	810	if (!new_maps) {
795	pr_warning("alloc maps for object failed\n");	811	pr_warn("alloc maps for object failed\n");
796	return ERR_PTR(-ENOMEM);	812	return ERR_PTR(-ENOMEM);
797	}	813	}
798		814
@@ -833,7 +849,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, · 34 unmodified lines
833	libbpf_type_to_btf_name[type]);	849	libbpf_type_to_btf_name[type]);
834	map->name = strdup(map_name);	850	map->name = strdup(map_name);
835	if (!map->name) {	851	if (!map->name) {
836	pr_warning("failed to alloc map name\n");	852	pr_warn("failed to alloc map name\n");
837	return -ENOMEM;	853	return -ENOMEM;
838	}	854	}
839	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n",	855	pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n",
@@ -849,7 +865,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, · 9 unmodified lines
849	*data_buff = malloc(data->d_size);	865	*data_buff = malloc(data->d_size);
850	if (!*data_buff) {	866	if (!*data_buff) {
851	zfree(&map->name);	867	zfree(&map->name);
852	pr_warning("failed to alloc map content buffer\n");	868	pr_warn("failed to alloc map content buffer\n");
853	return -ENOMEM;	869	return -ENOMEM;
854	}	870	}
855	memcpy(*data_buff, data->d_buf, data->d_size);	871	memcpy(*data_buff, data->d_buf, data->d_size);
@@ -911,8 +927,8 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) · 55 unmodified lines
911	if (scn)	927	if (scn)
912	data = elf_getdata(scn, NULL);	928	data = elf_getdata(scn, NULL);
913	if (!scn \|\| !data) {	929	if (!scn \|\| !data) {
914	pr_warning("failed to get Elf_Data from map section %d\n",	930	pr_warn("failed to get Elf_Data from map section %d\n",
915	obj->efile.maps_shndx);	931	obj->efile.maps_shndx);
916	return -EINVAL;	932	return -EINVAL;
917	}	933	}
918		934
@@ -939,9 +955,9 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) · 20 unmodified lines
939		955
940	map_def_sz = data->d_size / nr_maps;	956	map_def_sz = data->d_size / nr_maps;
941	if (!data->d_size \|\| (data->d_size % nr_maps) != 0) {	957	if (!data->d_size \|\| (data->d_size % nr_maps) != 0) {
942	pr_warning("unable to determine map definition size "	958	pr_warn("unable to determine map definition size "
943	"section %s, %d maps in %zd bytes\n",	959	"section %s, %d maps in %zd bytes\n",
944	obj->path, nr_maps, data->d_size);	960	obj->path, nr_maps, data->d_size);
945	return -EINVAL;	961	return -EINVAL;
946	}	962	}
947		963
@@ -964,8 +980,8 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) · 16 unmodified lines
964	map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,	980	map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
965	sym.st_name);	981	sym.st_name);
966	if (!map_name) {	982	if (!map_name) {
967	pr_warning("failed to get map #%d name sym string for obj %s\n",	983	pr_warn("failed to get map #%d name sym string for obj %s\n",
968	i, obj->path);	984	i, obj->path);
969	return -LIBBPF_ERRNO__FORMAT;	985	return -LIBBPF_ERRNO__FORMAT;
970	}	986	}
971		987
@@ -975,14 +991,14 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) · 3 unmodified lines
975	pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",	991	pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
976	map_name, map->sec_idx, map->sec_offset);	992	map_name, map->sec_idx, map->sec_offset);
977	if (sym.st_value + map_def_sz > data->d_size) {	993	if (sym.st_value + map_def_sz > data->d_size) {
978	pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",	994	pr_warn("corrupted maps section in %s: last map \"%s\" too small\n",
979	obj->path, map_name);	995	obj->path, map_name);
980	return -EINVAL;	996	return -EINVAL;
981	}	997	}
982		998
983	map->name = strdup(map_name);	999	map->name = strdup(map_name);
984	if (!map->name) {	1000	if (!map->name) {
985	pr_warning("failed to alloc map name\n");	1001	pr_warn("failed to alloc map name\n");
986	return -ENOMEM;	1002	return -ENOMEM;
987	}	1003	}
988	pr_debug("map %d is \"%s\"\n", i, map->name);	1004	pr_debug("map %d is \"%s\"\n", i, map->name);
@@ -1006,10 +1022,10 @@ static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) · 17 unmodified lines
1006	for (b = ((char *)def) + sizeof(struct bpf_map_def);	1022	for (b = ((char *)def) + sizeof(struct bpf_map_def);
1007	b < ((char *)def) + map_def_sz; b++) {	1023	b < ((char *)def) + map_def_sz; b++) {
1008	if (*b != 0) {	1024	if (*b != 0) {
1009	pr_warning("maps section in %s: \"%s\" "	1025	pr_warn("maps section in %s: \"%s\" "
1010	"has unrecognized, non-zero "	1026	"has unrecognized, non-zero "
1011	"options\n",	1027	"options\n",
1012	obj->path, map_name);	1028	obj->path, map_name);
1013	if (strict)	1029	if (strict)
1014	return -EINVAL;	1030	return -EINVAL;
1015	}	1031	}
@@ -1053,20 +1069,20 @@ static bool get_map_field_int(const char map_name, const struct btf btf, · 37 unmodified lines
1053	const struct btf_type *arr_t;	1069	const struct btf_type *arr_t;
1054		1070
1055	if (!btf_is_ptr(t)) {	1071	if (!btf_is_ptr(t)) {
1056	pr_warning("map '%s': attr '%s': expected PTR, got %u.\n",	1072	pr_warn("map '%s': attr '%s': expected PTR, got %u.\n",
1057	map_name, name, btf_kind(t));	1073	map_name, name, btf_kind(t));
1058	return false;	1074	return false;
1059	}	1075	}
1060		1076
1061	arr_t = btf__type_by_id(btf, t->type);	1077	arr_t = btf__type_by_id(btf, t->type);
1062	if (!arr_t) {	1078	if (!arr_t) {
1063	pr_warning("map '%s': attr '%s': type [%u] not found.\n",	1079	pr_warn("map '%s': attr '%s': type [%u] not found.\n",
1064	map_name, name, t->type);	1080	map_name, name, t->type);
1065	return false;	1081	return false;
1066	}	1082	}
1067	if (!btf_is_array(arr_t)) {	1083	if (!btf_is_array(arr_t)) {
1068	pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n",	1084	pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n",
1069	map_name, name, btf_kind(arr_t));	1085	map_name, name, btf_kind(arr_t));
1070	return false;	1086	return false;
1071	}	1087	}
1072	arr_info = btf_array(arr_t);	1088	arr_info = btf_array(arr_t);
@@ -1094,33 +1110,33 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 21 unmodified lines
1094	vlen = btf_vlen(var);	1110	vlen = btf_vlen(var);
1095		1111
1096	if (map_name == NULL \|\| map_name[0] == '\0') {	1112	if (map_name == NULL \|\| map_name[0] == '\0') {
1097	pr_warning("map #%d: empty name.\n", var_idx);	1113	pr_warn("map #%d: empty name.\n", var_idx);
1098	return -EINVAL;	1114	return -EINVAL;
1099	}	1115	}
1100	if ((__u64)vi->offset + vi->size > data->d_size) {	1116	if ((__u64)vi->offset + vi->size > data->d_size) {
1101	pr_warning("map '%s' BTF data is corrupted.\n", map_name);	1117	pr_warn("map '%s' BTF data is corrupted.\n", map_name);
1102	return -EINVAL;	1118	return -EINVAL;
1103	}	1119	}
1104	if (!btf_is_var(var)) {	1120	if (!btf_is_var(var)) {
1105	pr_warning("map '%s': unexpected var kind %u.\n",	1121	pr_warn("map '%s': unexpected var kind %u.\n",
1106	map_name, btf_kind(var));	1122	map_name, btf_kind(var));
1107	return -EINVAL;	1123	return -EINVAL;
1108	}	1124	}
1109	if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&	1125	if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
1110	var_extra->linkage != BTF_VAR_STATIC) {	1126	var_extra->linkage != BTF_VAR_STATIC) {
1111	pr_warning("map '%s': unsupported var linkage %u.\n",	1127	pr_warn("map '%s': unsupported var linkage %u.\n",
1112	map_name, var_extra->linkage);	1128	map_name, var_extra->linkage);
1113	return -EOPNOTSUPP;	1129	return -EOPNOTSUPP;
1114	}	1130	}
1115		1131
1116	def = skip_mods_and_typedefs(obj->btf, var->type, NULL);	1132	def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
1117	if (!btf_is_struct(def)) {	1133	if (!btf_is_struct(def)) {
1118	pr_warning("map '%s': unexpected def kind %u.\n",	1134	pr_warn("map '%s': unexpected def kind %u.\n",
1119	map_name, btf_kind(var));	1135	map_name, btf_kind(var));
1120	return -EINVAL;	1136	return -EINVAL;
1121	}	1137	}
1122	if (def->size > vi->size) {	1138	if (def->size > vi->size) {
1123	pr_warning("map '%s': invalid def size.\n", map_name);	1139	pr_warn("map '%s': invalid def size.\n", map_name);
1124	return -EINVAL;	1140	return -EINVAL;
1125	}	1141	}
1126		1142
@@ -1129,7 +1145,7 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 2 unmodified lines
1129	return PTR_ERR(map);	1145	return PTR_ERR(map);
1130	map->name = strdup(map_name);	1146	map->name = strdup(map_name);
1131	if (!map->name) {	1147	if (!map->name) {
1132	pr_warning("map '%s': failed to alloc map name.\n", map_name);	1148	pr_warn("map '%s': failed to alloc map name.\n", map_name);
1133	return -ENOMEM;	1149	return -ENOMEM;
1134	}	1150	}
1135	map->libbpf_type = LIBBPF_MAP_UNSPEC;	1151	map->libbpf_type = LIBBPF_MAP_UNSPEC;
@@ -1145,8 +1161,7 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 9 unmodified lines
1145	const char *name = btf__name_by_offset(obj->btf, m->name_off);	1161	const char *name = btf__name_by_offset(obj->btf, m->name_off);
1146		1162
1147	if (!name) {	1163	if (!name) {
1148	pr_warning("map '%s': invalid field #%d.\n",	1164	pr_warn("map '%s': invalid field #%d.\n", map_name, i);
1149	map_name, i);
1150	return -EINVAL;	1165	return -EINVAL;
1151	}	1166	}
1152	if (strcmp(name, "type") == 0) {	1167	if (strcmp(name, "type") == 0) {
@@ -1176,8 +1191,8 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 23 unmodified lines
1176	pr_debug("map '%s': found key_size = %u.\n",	1191	pr_debug("map '%s': found key_size = %u.\n",
1177	map_name, sz);	1192	map_name, sz);
1178	if (map->def.key_size && map->def.key_size != sz) {	1193	if (map->def.key_size && map->def.key_size != sz) {
1179	pr_warning("map '%s': conflicting key size %u != %u.\n",	1194	pr_warn("map '%s': conflicting key size %u != %u.\n",
1180	map_name, map->def.key_size, sz);	1195	map_name, map->def.key_size, sz);
1181	return -EINVAL;	1196	return -EINVAL;
1182	}	1197	}
1183	map->def.key_size = sz;	1198	map->def.key_size = sz;
@@ -1186,26 +1201,26 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 2 unmodified lines
1186		1201
1187	t = btf__type_by_id(obj->btf, m->type);	1202	t = btf__type_by_id(obj->btf, m->type);
1188	if (!t) {	1203	if (!t) {
1189	pr_warning("map '%s': key type [%d] not found.\n",	1204	pr_warn("map '%s': key type [%d] not found.\n",
1190	map_name, m->type);	1205	map_name, m->type);
1191	return -EINVAL;	1206	return -EINVAL;
1192	}	1207	}
1193	if (!btf_is_ptr(t)) {	1208	if (!btf_is_ptr(t)) {
1194	pr_warning("map '%s': key spec is not PTR: %u.\n",	1209	pr_warn("map '%s': key spec is not PTR: %u.\n",
1195	map_name, btf_kind(t));	1210	map_name, btf_kind(t));
1196	return -EINVAL;	1211	return -EINVAL;
1197	}	1212	}
1198	sz = btf__resolve_size(obj->btf, t->type);	1213	sz = btf__resolve_size(obj->btf, t->type);
1199	if (sz < 0) {	1214	if (sz < 0) {
1200	pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n",	1215	pr_warn("map '%s': can't determine key size for type [%u]: %lld.\n",
1201	map_name, t->type, sz);	1216	map_name, t->type, sz);
1202	return sz;	1217	return sz;
1203	}	1218	}
1204	pr_debug("map '%s': found key [%u], sz = %lld.\n",	1219	pr_debug("map '%s': found key [%u], sz = %lld.\n",
1205	map_name, t->type, sz);	1220	map_name, t->type, sz);
1206	if (map->def.key_size && map->def.key_size != sz) {	1221	if (map->def.key_size && map->def.key_size != sz) {
1207	pr_warning("map '%s': conflicting key size %u != %lld.\n",	1222	pr_warn("map '%s': conflicting key size %u != %lld.\n",
1208	map_name, map->def.key_size, sz);	1223	map_name, map->def.key_size, sz);
1209	return -EINVAL;	1224	return -EINVAL;
1210	}	1225	}
1211	map->def.key_size = sz;	1226	map->def.key_size = sz;
@@ -1219,8 +1234,8 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 7 unmodified lines
1219	pr_debug("map '%s': found value_size = %u.\n",	1234	pr_debug("map '%s': found value_size = %u.\n",
1220	map_name, sz);	1235	map_name, sz);
1221	if (map->def.value_size && map->def.value_size != sz) {	1236	if (map->def.value_size && map->def.value_size != sz) {
1222	pr_warning("map '%s': conflicting value size %u != %u.\n",	1237	pr_warn("map '%s': conflicting value size %u != %u.\n",
1223	map_name, map->def.value_size, sz);	1238	map_name, map->def.value_size, sz);
1224	return -EINVAL;	1239	return -EINVAL;
1225	}	1240	}
1226	map->def.value_size = sz;	1241	map->def.value_size = sz;
@@ -1229,34 +1244,34 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 2 unmodified lines
1229		1244
1230	t = btf__type_by_id(obj->btf, m->type);	1245	t = btf__type_by_id(obj->btf, m->type);
1231	if (!t) {	1246	if (!t) {
1232	pr_warning("map '%s': value type [%d] not found.\n",	1247	pr_warn("map '%s': value type [%d] not found.\n",
1233	map_name, m->type);	1248	map_name, m->type);
1234	return -EINVAL;	1249	return -EINVAL;
1235	}	1250	}
1236	if (!btf_is_ptr(t)) {	1251	if (!btf_is_ptr(t)) {
1237	pr_warning("map '%s': value spec is not PTR: %u.\n",	1252	pr_warn("map '%s': value spec is not PTR: %u.\n",
1238	map_name, btf_kind(t));	1253	map_name, btf_kind(t));
1239	return -EINVAL;	1254	return -EINVAL;
1240	}	1255	}
1241	sz = btf__resolve_size(obj->btf, t->type);	1256	sz = btf__resolve_size(obj->btf, t->type);
1242	if (sz < 0) {	1257	if (sz < 0) {
1243	pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n",	1258	pr_warn("map '%s': can't determine value size for type [%u]: %lld.\n",
1244	map_name, t->type, sz);	1259	map_name, t->type, sz);
1245	return sz;	1260	return sz;
1246	}	1261	}
1247	pr_debug("map '%s': found value [%u], sz = %lld.\n",	1262	pr_debug("map '%s': found value [%u], sz = %lld.\n",
1248	map_name, t->type, sz);	1263	map_name, t->type, sz);
1249	if (map->def.value_size && map->def.value_size != sz) {	1264	if (map->def.value_size && map->def.value_size != sz) {
1250	pr_warning("map '%s': conflicting value size %u != %lld.\n",	1265	pr_warn("map '%s': conflicting value size %u != %lld.\n",
1251	map_name, map->def.value_size, sz);	1266	map_name, map->def.value_size, sz);
1252	return -EINVAL;	1267	return -EINVAL;
1253	}	1268	}
1254	map->def.value_size = sz;	1269	map->def.value_size = sz;
1255	map->btf_value_type_id = t->type;	1270	map->btf_value_type_id = t->type;
1256	} else {	1271	} else {
1257	if (strict) {	1272	if (strict) {
1258	pr_warning("map '%s': unknown field '%s'.\n",	1273	pr_warn("map '%s': unknown field '%s'.\n",
1259	map_name, name);	1274	map_name, name);
1260	return -ENOTSUP;	1275	return -ENOTSUP;
1261	}	1276	}
1262	pr_debug("map '%s': ignoring unknown field '%s'.\n",	1277	pr_debug("map '%s': ignoring unknown field '%s'.\n",
@@ -1265,7 +1280,7 @@ static int bpf_object__init_user_btf_map(struct bpf_object *obj, · 2 unmodified lines
1265	}	1280	}
1266		1281
1267	if (map->def.type == BPF_MAP_TYPE_UNSPEC) {	1282	if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
1268	pr_warning("map '%s': map type isn't specified.\n", map_name);	1283	pr_warn("map '%s': map type isn't specified.\n", map_name);
1269	return -EINVAL;	1284	return -EINVAL;
1270	}	1285	}
1271		1286
@@ -1288,8 +1303,8 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) · 16 unmodified lines
1288	if (scn)	1303	if (scn)
1289	data = elf_getdata(scn, NULL);	1304	data = elf_getdata(scn, NULL);
1290	if (!scn \|\| !data) {	1305	if (!scn \|\| !data) {
1291	pr_warning("failed to get Elf_Data from map section %d (%s)\n",	1306	pr_warn("failed to get Elf_Data from map section %d (%s)\n",
1292	obj->efile.maps_shndx, MAPS_ELF_SEC);	1307	obj->efile.maps_shndx, MAPS_ELF_SEC);
1293	return -EINVAL;	1308	return -EINVAL;
1294	}	1309	}
1295		1310
@@ -1306,7 +1321,7 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) · 10 unmodified lines
1306	}	1321	}
1307		1322
1308	if (!sec) {	1323	if (!sec) {
1309	pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC);	1324	pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
1310	return -ENOENT;	1325	return -ENOENT;
1311	}	1326	}
1312		1327
@@ -1322,9 +1337,9 @@ static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) · 9 unmodified lines
1322	return 0;	1337	return 0;
1323	}	1338	}
1324		1339
1325	static int bpf_object__init_maps(struct bpf_object *obj, int flags)	1340	static int bpf_object__init_maps(struct bpf_object *obj, bool relaxed_maps)
1326	{	1341	{
1327	bool strict = !(flags & MAPS_RELAX_COMPAT);	1342	bool strict = !relaxed_maps;
1328	int err;	1343	int err;
1329		1344
1330	err = bpf_object__init_user_maps(obj, strict);	1345	err = bpf_object__init_user_maps(obj, strict);
@@ -1450,14 +1465,13 @@ static int bpf_object__init_btf(struct bpf_object *obj, · 119 unmodified lines
1450	if (btf_data) {	1465	if (btf_data) {
1451	obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);	1466	obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
1452	if (IS_ERR(obj->btf)) {	1467	if (IS_ERR(obj->btf)) {
1453	pr_warning("Error loading ELF section %s: %d.\n",	1468	pr_warn("Error loading ELF section %s: %d.\n",
1454	BTF_ELF_SEC, err);	1469	BTF_ELF_SEC, err);
1455	goto out;	1470	goto out;
1456	}	1471	}
1457	err = btf__finalize_data(obj, obj->btf);	1472	err = btf__finalize_data(obj, obj->btf);
1458	if (err) {	1473	if (err) {
1459	pr_warning("Error finalizing %s: %d.\n",	1474	pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err);
1460	BTF_ELF_SEC, err);
1461	goto out;	1475	goto out;
1462	}	1476	}
1463	}	1477	}
@@ -1470,8 +1484,8 @@ static int bpf_object__init_btf(struct bpf_object *obj, · 6 unmodified lines
1470	obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,	1484	obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
1471	btf_ext_data->d_size);	1485	btf_ext_data->d_size);
1472	if (IS_ERR(obj->btf_ext)) {	1486	if (IS_ERR(obj->btf_ext)) {
1473	pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",	1487	pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n",
1474	BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));	1488	BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
1475	obj->btf_ext = NULL;	1489	obj->btf_ext = NULL;
1476	goto out;	1490	goto out;
1477	}	1491	}
@@ -1487,7 +1501,7 @@ out: · 9 unmodified lines
1487	obj->btf = NULL;	1501	obj->btf = NULL;
1488	}	1502	}
1489	if (btf_required && !obj->btf) {	1503	if (btf_required && !obj->btf) {
1490	pr_warning("BTF is required, but is missing or corrupted.\n");	1504	pr_warn("BTF is required, but is missing or corrupted.\n");
1491	return err == 0 ? -ENOENT : err;	1505	return err == 0 ? -ENOENT : err;
1492	}	1506	}
1493	return 0;	1507	return 0;
@@ -1505,8 +1519,8 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) · 11 unmodified lines
1505		1519
1506	err = btf__load(obj->btf);	1520	err = btf__load(obj->btf);
1507	if (err) {	1521	if (err) {
1508	pr_warning("Error loading %s into kernel: %d.\n",	1522	pr_warn("Error loading %s into kernel: %d.\n",
1509	BTF_ELF_SEC, err);	1523	BTF_ELF_SEC, err);
1510	btf__free(obj->btf);	1524	btf__free(obj->btf);
1511	obj->btf = NULL;	1525	obj->btf = NULL;
1512	/* btf_ext can't exist without btf, so free it as well */	1526	/* btf_ext can't exist without btf, so free it as well */
@@ -1521,7 +1535,7 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) · 8 unmodified lines
1521	return 0;	1535	return 0;
1522	}	1536	}
1523		1537
1524	static int bpf_object__elf_collect(struct bpf_object *obj, int flags)	1538	static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps)
1525	{	1539	{
1526	Elf *elf = obj->efile.elf;	1540	Elf *elf = obj->efile.elf;
1527	GElf_Ehdr *ep = &obj->efile.ehdr;	1541	GElf_Ehdr *ep = &obj->efile.ehdr;
@@ -1532,7 +1546,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 4 unmodified lines
1532		1546
1533	/* Elf is corrupted/truncated, avoid calling elf_strptr. */	1547	/* Elf is corrupted/truncated, avoid calling elf_strptr. */
1534	if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {	1548	if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
1535	pr_warning("failed to get e_shstrndx from %s\n", obj->path);	1549	pr_warn("failed to get e_shstrndx from %s\n", obj->path);
1536	return -LIBBPF_ERRNO__FORMAT;	1550	return -LIBBPF_ERRNO__FORMAT;
1537	}	1551	}
1538		1552
@@ -1543,22 +1557,22 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 4 unmodified lines
1543		1557
1544	idx++;	1558	idx++;
1545	if (gelf_getshdr(scn, &sh) != &sh) {	1559	if (gelf_getshdr(scn, &sh) != &sh) {
1546	pr_warning("failed to get section(%d) header from %s\n",	1560	pr_warn("failed to get section(%d) header from %s\n",
1547	idx, obj->path);	1561	idx, obj->path);
1548	return -LIBBPF_ERRNO__FORMAT;	1562	return -LIBBPF_ERRNO__FORMAT;
1549	}	1563	}
1550		1564
1551	name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);	1565	name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
1552	if (!name) {	1566	if (!name) {
1553	pr_warning("failed to get section(%d) name from %s\n",	1567	pr_warn("failed to get section(%d) name from %s\n",
1554	idx, obj->path);	1568	idx, obj->path);
1555	return -LIBBPF_ERRNO__FORMAT;	1569	return -LIBBPF_ERRNO__FORMAT;
1556	}	1570	}
1557		1571
1558	data = elf_getdata(scn, 0);	1572	data = elf_getdata(scn, 0);
1559	if (!data) {	1573	if (!data) {
1560	pr_warning("failed to get section(%d) data from %s(%s)\n",	1574	pr_warn("failed to get section(%d) data from %s(%s)\n",
1561	idx, name, obj->path);	1575	idx, name, obj->path);
1562	return -LIBBPF_ERRNO__FORMAT;	1576	return -LIBBPF_ERRNO__FORMAT;
1563	}	1577	}
1564	pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",	1578	pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
@@ -1573,7 +1587,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 8 unmodified lines
1573	if (err)	1587	if (err)
1574	return err;	1588	return err;
1575	} else if (strcmp(name, "version") == 0) {	1589	} else if (strcmp(name, "version") == 0) {
1576	/* skip, we don't need it anymore */	1590	err = bpf_object__init_kversion(obj,
		1591	data->d_buf,
		1592	data->d_size);
		1593	if (err)
		1594	return err;
1577	} else if (strcmp(name, "maps") == 0) {	1595	} else if (strcmp(name, "maps") == 0) {
1578	obj->efile.maps_shndx = idx;	1596	obj->efile.maps_shndx = idx;
1579	} else if (strcmp(name, MAPS_ELF_SEC) == 0) {	1597	} else if (strcmp(name, MAPS_ELF_SEC) == 0) {
@@ -1584,8 +1602,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 4 unmodified lines
1584	btf_ext_data = data;	1602	btf_ext_data = data;
1585	} else if (sh.sh_type == SHT_SYMTAB) {	1603	} else if (sh.sh_type == SHT_SYMTAB) {
1586	if (obj->efile.symbols) {	1604	if (obj->efile.symbols) {
1587	pr_warning("bpf: multiple SYMTAB in %s\n",	1605	pr_warn("bpf: multiple SYMTAB in %s\n",
1588	obj->path);	1606	obj->path);
1589	return -LIBBPF_ERRNO__FORMAT;	1607	return -LIBBPF_ERRNO__FORMAT;
1590	}	1608	}
1591	obj->efile.symbols = data;	1609	obj->efile.symbols = data;
@@ -1601,8 +1619,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 9 unmodified lines
1601	char *cp = libbpf_strerror_r(-err, errmsg,	1619	char *cp = libbpf_strerror_r(-err, errmsg,
1602	sizeof(errmsg));	1620	sizeof(errmsg));
1603		1621
1604	pr_warning("failed to alloc program %s (%s): %s",	1622	pr_warn("failed to alloc program %s (%s): %s",
1605	name, obj->path, cp);	1623	name, obj->path, cp);
1606	return err;	1624	return err;
1607	}	1625	}
1608	} else if (strcmp(name, ".data") == 0) {	1626	} else if (strcmp(name, ".data") == 0) {
@@ -1629,7 +1647,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 20 unmodified lines
1629	reloc = reallocarray(reloc, nr_reloc + 1,	1647	reloc = reallocarray(reloc, nr_reloc + 1,
1630	sizeof(*obj->efile.reloc));	1648	sizeof(*obj->efile.reloc));
1631	if (!reloc) {	1649	if (!reloc) {
1632	pr_warning("realloc failed\n");	1650	pr_warn("realloc failed\n");
1633	return -ENOMEM;	1651	return -ENOMEM;
1634	}	1652	}
1635		1653
@@ -1647,12 +1665,12 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 11 unmodified lines
1647	}	1665	}
1648		1666
1649	if (!obj->efile.strtabidx \|\| obj->efile.strtabidx >= idx) {	1667	if (!obj->efile.strtabidx \|\| obj->efile.strtabidx >= idx) {
1650	pr_warning("Corrupted ELF file: index of strtab invalid\n");	1668	pr_warn("Corrupted ELF file: index of strtab invalid\n");
1651	return -LIBBPF_ERRNO__FORMAT;	1669	return -LIBBPF_ERRNO__FORMAT;
1652	}	1670	}
1653	err = bpf_object__init_btf(obj, btf_data, btf_ext_data);	1671	err = bpf_object__init_btf(obj, btf_data, btf_ext_data);
1654	if (!err)	1672	if (!err)
1655	err = bpf_object__init_maps(obj, flags);	1673	err = bpf_object__init_maps(obj, relaxed_maps);
1656	if (!err)	1674	if (!err)
1657	err = bpf_object__sanitize_and_load_btf(obj);	1675	err = bpf_object__sanitize_and_load_btf(obj);
1658	if (!err)	1676	if (!err)
@@ -1737,7 +1755,7 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 78 unmodified lines
1737		1755
1738	prog->reloc_desc = malloc(sizeof(prog->reloc_desc) nrels);	1756	prog->reloc_desc = malloc(sizeof(prog->reloc_desc) nrels);
1739	if (!prog->reloc_desc) {	1757	if (!prog->reloc_desc) {
1740	pr_warning("failed to alloc memory in relocation\n");	1758	pr_warn("failed to alloc memory in relocation\n");
1741	return -ENOMEM;	1759	return -ENOMEM;
1742	}	1760	}
1743	prog->nr_reloc = nrels;	1761	prog->nr_reloc = nrels;
@@ -1753,13 +1771,13 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 9 unmodified lines
1753	GElf_Rel rel;	1771	GElf_Rel rel;
1754		1772
1755	if (!gelf_getrel(data, i, &rel)) {	1773	if (!gelf_getrel(data, i, &rel)) {
1756	pr_warning("relocation: failed to get %d reloc\n", i);	1774	pr_warn("relocation: failed to get %d reloc\n", i);
1757	return -LIBBPF_ERRNO__FORMAT;	1775	return -LIBBPF_ERRNO__FORMAT;
1758	}	1776	}
1759		1777
1760	if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {	1778	if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
1761	pr_warning("relocation: symbol %"PRIx64" not found\n",	1779	pr_warn("relocation: symbol %"PRIx64" not found\n",
1762	GELF_R_SYM(rel.r_info));	1780	GELF_R_SYM(rel.r_info));
1763	return -LIBBPF_ERRNO__FORMAT;	1781	return -LIBBPF_ERRNO__FORMAT;
1764	}	1782	}
1765		1783
@@ -1776,20 +1794,20 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 10 unmodified lines
1776	insn_idx, shdr_idx);	1794	insn_idx, shdr_idx);
1777		1795
1778	if (shdr_idx >= SHN_LORESERVE) {	1796	if (shdr_idx >= SHN_LORESERVE) {
1779	pr_warning("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n",	1797	pr_warn("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n",
1780	name, shdr_idx, insn_idx,	1798	name, shdr_idx, insn_idx,
1781	insns[insn_idx].code);	1799	insns[insn_idx].code);
1782	return -LIBBPF_ERRNO__RELOC;	1800	return -LIBBPF_ERRNO__RELOC;
1783	}	1801	}
1784	if (!bpf_object__relo_in_known_section(obj, shdr_idx)) {	1802	if (!bpf_object__relo_in_known_section(obj, shdr_idx)) {
1785	pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n",	1803	pr_warn("Program '%s' contains unrecognized relo data pointing to section %u\n",
1786	prog->section_name, shdr_idx);	1804	prog->section_name, shdr_idx);
1787	return -LIBBPF_ERRNO__RELOC;	1805	return -LIBBPF_ERRNO__RELOC;
1788	}	1806	}
1789		1807
1790	if (insns[insn_idx].code == (BPF_JMP \| BPF_CALL)) {	1808	if (insns[insn_idx].code == (BPF_JMP \| BPF_CALL)) {
1791	if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {	1809	if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
1792	pr_warning("incorrect bpf_call opcode\n");	1810	pr_warn("incorrect bpf_call opcode\n");
1793	return -LIBBPF_ERRNO__RELOC;	1811	return -LIBBPF_ERRNO__RELOC;
1794	}	1812	}
1795	prog->reloc_desc[i].type = RELO_CALL;	1813	prog->reloc_desc[i].type = RELO_CALL;
@@ -1800,8 +1818,8 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 4 unmodified lines
1800	}	1818	}
1801		1819
1802	if (insns[insn_idx].code != (BPF_LD \| BPF_IMM \| BPF_DW)) {	1820	if (insns[insn_idx].code != (BPF_LD \| BPF_IMM \| BPF_DW)) {
1803	pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",	1821	pr_warn("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
1804	insn_idx, insns[insn_idx].code);	1822	insn_idx, insns[insn_idx].code);
1805	return -LIBBPF_ERRNO__RELOC;	1823	return -LIBBPF_ERRNO__RELOC;
1806	}	1824	}
1807		1825
@@ -1810,13 +1828,13 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 2 unmodified lines
1810	type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);	1828	type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
1811	if (type != LIBBPF_MAP_UNSPEC) {	1829	if (type != LIBBPF_MAP_UNSPEC) {
1812	if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) {	1830	if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) {
1813	pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n",	1831	pr_warn("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n",
1814	name, insn_idx, insns[insn_idx].code);	1832	name, insn_idx, insns[insn_idx].code);
1815	return -LIBBPF_ERRNO__RELOC;	1833	return -LIBBPF_ERRNO__RELOC;
1816	}	1834	}
1817	if (!obj->caps.global_data) {	1835	if (!obj->caps.global_data) {
1818	pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n",	1836	pr_warn("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n",
1819	name, insn_idx);	1837	name, insn_idx);
1820	return -LIBBPF_ERRNO__RELOC;	1838	return -LIBBPF_ERRNO__RELOC;
1821	}	1839	}
1822	}	1840	}
@@ -1837,8 +1855,8 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 14 unmodified lines
1837	}	1855	}
1838		1856
1839	if (map_idx >= nr_maps) {	1857	if (map_idx >= nr_maps) {
1840	pr_warning("bpf relocation: map_idx %d larger than %d\n",	1858	pr_warn("bpf relocation: map_idx %d larger than %d\n",
1841	(int)map_idx, (int)nr_maps - 1);	1859	(int)map_idx, (int)nr_maps - 1);
1842	return -LIBBPF_ERRNO__RELOC;	1860	return -LIBBPF_ERRNO__RELOC;
1843	}	1861	}
1844		1862
@@ -1965,8 +1983,8 @@ bpf_object__probe_name(struct bpf_object *obj) · 120 unmodified lines
1965	ret = bpf_load_program_xattr(&attr, NULL, 0);	1983	ret = bpf_load_program_xattr(&attr, NULL, 0);
1966	if (ret < 0) {	1984	if (ret < 0) {
1967	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));	1985	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1968	pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",	1986	pr_warn("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",
1969	__func__, cp, errno);	1987	__func__, cp, errno);
1970	return -errno;	1988	return -errno;
1971	}	1989	}
1972	close(ret);	1990	close(ret);
@@ -2006,8 +2024,8 @@ bpf_object__probe_global_data(struct bpf_object *obj) · 33 unmodified lines
2006	map = bpf_create_map_xattr(&map_attr);	2024	map = bpf_create_map_xattr(&map_attr);
2007	if (map < 0) {	2025	if (map < 0) {
2008	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));	2026	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
2009	pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n",	2027	pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
2010	__func__, cp, errno);	2028	__func__, cp, errno);
2011	return -errno;	2029	return -errno;
2012	}	2030	}
2013		2031
@@ -2122,8 +2140,8 @@ bpf_object__populate_internal_map(struct bpf_object obj, struct bpf_map map) · 108 unmodified lines
2122	err = bpf_map_freeze(map->fd);	2140	err = bpf_map_freeze(map->fd);
2123	if (err) {	2141	if (err) {
2124	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));	2142	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
2125	pr_warning("Error freezing map(%s) as read-only: %s\n",	2143	pr_warn("Error freezing map(%s) as read-only: %s\n",
2126	map->name, cp);	2144	map->name, cp);
2127	err = 0;	2145	err = 0;
2128	}	2146	}
2129	}	2147	}
@@ -2162,8 +2180,8 @@ bpf_object__create_maps(struct bpf_object *obj) · 32 unmodified lines
2162	if (!nr_cpus)	2180	if (!nr_cpus)
2163	nr_cpus = libbpf_num_possible_cpus();	2181	nr_cpus = libbpf_num_possible_cpus();
2164	if (nr_cpus < 0) {	2182	if (nr_cpus < 0) {
2165	pr_warning("failed to determine number of system CPUs: %d\n",	2183	pr_warn("failed to determine number of system CPUs: %d\n",
2166	nr_cpus);	2184	nr_cpus);
2167	err = nr_cpus;	2185	err = nr_cpus;
2168	goto err_out;	2186	goto err_out;
2169	}	2187	}
@@ -2191,8 +2209,8 @@ bpf_object__create_maps(struct bpf_object *obj) · 21 unmodified lines
2191	create_attr.btf_value_type_id)) {	2209	create_attr.btf_value_type_id)) {
2192	err = -errno;	2210	err = -errno;
2193	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));	2211	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
2194	pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",	2212	pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
2195	map->name, cp, err);	2213	map->name, cp, err);
2196	create_attr.btf_fd = 0;	2214	create_attr.btf_fd = 0;
2197	create_attr.btf_key_type_id = 0;	2215	create_attr.btf_key_type_id = 0;
2198	create_attr.btf_value_type_id = 0;	2216	create_attr.btf_value_type_id = 0;
@@ -2207,8 +2225,8 @@ bpf_object__create_maps(struct bpf_object *obj) · 8 unmodified lines
2207	err = -errno;	2225	err = -errno;
2208	err_out:	2226	err_out:
2209	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));	2227	cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
2210	pr_warning("failed to create map (name: '%s'): %s(%d)\n",	2228	pr_warn("failed to create map (name: '%s'): %s(%d)\n",
2211	map->name, cp, err);	2229	map->name, cp, err);
2212	for (j = 0; j < i; j++)	2230	for (j = 0; j < i; j++)
2213	zclose(obj->maps[j].fd);	2231	zclose(obj->maps[j].fd);
2214	return err;	2232	return err;
@@ -2233,8 +2251,8 @@ check_btf_ext_reloc_err(struct bpf_program *prog, int err, · 18 unmodified lines
2233	void btf_prog_info, const char info_name)	2251	void btf_prog_info, const char info_name)
2234	{	2252	{
2235	if (err != -ENOENT) {	2253	if (err != -ENOENT) {
2236	pr_warning("Error in loading %s for sec %s.\n",	2254	pr_warn("Error in loading %s for sec %s.\n",
2237	info_name, prog->section_name);	2255	info_name, prog->section_name);
2238	return err;	2256	return err;
2239	}	2257	}
2240		2258
@@ -2245,14 +2263,14 @@ check_btf_ext_reloc_err(struct bpf_program *prog, int err, · 4 unmodified lines
2245	* Some info has already been found but has problem	2263	* Some info has already been found but has problem
2246	* in the last btf_ext reloc. Must have to error out.	2264	* in the last btf_ext reloc. Must have to error out.
2247	*/	2265	*/
2248	pr_warning("Error in relocating %s for sec %s.\n",	2266	pr_warn("Error in relocating %s for sec %s.\n",
2249	info_name, prog->section_name);	2267	info_name, prog->section_name);
2250	return err;	2268	return err;
2251	}	2269	}
2252		2270
2253	/* Have problem loading the very first info. Ignore the rest. */	2271	/* Have problem loading the very first info. Ignore the rest. */
2254	pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n",	2272	pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n",
2255	info_name, prog->section_name, info_name);	2273	info_name, prog->section_name, info_name);
2256	return 0;	2274	return 0;
2257	}	2275	}
2258		2276
@@ -2326,7 +2344,7 @@ static bool str_is_empty(const char *s) · 67 unmodified lines
2326	}	2344	}
2327		2345
2328	/*	2346	/*
2329	* Turn bpf_offset_reloc into a low- and high-level spec representation,	2347	* Turn bpf_field_reloc into a low- and high-level spec representation,
2330	* validating correctness along the way, as well as calculating resulting	2348	* validating correctness along the way, as well as calculating resulting
2331	* field offset (in bytes), specified by accessor string. Low-level spec	2349	* field offset (in bytes), specified by accessor string. Low-level spec
2332	* captures every single level of nestedness, including traversing anonymous	2350	* captures every single level of nestedness, including traversing anonymous
@@ -2453,8 +2471,8 @@ static int bpf_core_spec_parse(const struct btf *btf, · 120 unmodified lines
2453	return sz;	2471	return sz;
2454	spec->offset += access_idx * sz;	2472	spec->offset += access_idx * sz;
2455	} else {	2473	} else {
2456	pr_warning("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n",	2474	pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n",
2457	type_id, spec_str, i, id, btf_kind(t));	2475	type_id, spec_str, i, id, btf_kind(t));
2458	return -EINVAL;	2476	return -EINVAL;
2459	}	2477	}
2460	}	2478	}
@@ -2598,8 +2616,8 @@ recur: · 137 unmodified lines
2598	targ_id = btf_array(targ_type)->type;	2616	targ_id = btf_array(targ_type)->type;
2599	goto recur;	2617	goto recur;
2600	default:	2618	default:
2601	pr_warning("unexpected kind %d relocated, local [%d], target [%d]\n",	2619	pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n",
2602	btf_kind(local_type), local_id, targ_id);	2620	btf_kind(local_type), local_id, targ_id);
2603	return 0;	2621	return 0;
2604	}	2622	}
2605	}	2623	}
@@ -2771,26 +2789,54 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, · 165 unmodified lines
2771		2789
2772	/*	2790	/*
2773	* Patch relocatable BPF instruction.	2791	* Patch relocatable BPF instruction.
2774	* Expected insn->imm value is provided for validation, as well as the new	2792	*
2775	* relocated value.	2793	* Patched value is determined by relocation kind and target specification.
		2794	* For field existence relocation target spec will be NULL if field is not
		2795	* found.
		2796	* Expected insn->imm value is determined using relocation kind and local
		2797	* spec, and is checked before patching instruction. If actual insn->imm value
		2798	* is wrong, bail out with error.
2776	*	2799	*
2777	* Currently three kinds of BPF instructions are supported:	2800	* Currently three kinds of BPF instructions are supported:
2778	* 1. rX = <imm> (assignment with immediate operand);	2801	* 1. rX = <imm> (assignment with immediate operand);
2779	* 2. rX += <imm> (arithmetic operations with immediate operand);	2802	* 2. rX += <imm> (arithmetic operations with immediate operand);
2780	* 3. *(rX) = <imm> (indirect memory assignment with immediate operand).
2781	*
2782	* If actual insn->imm value is wrong, bail out.
2783	*/	2803	*/
2784	static int bpf_core_reloc_insn(struct bpf_program *prog, int insn_off,	2804	static int bpf_core_reloc_insn(struct bpf_program *prog,
2785	__u32 orig_off, __u32 new_off)	2805	const struct bpf_field_reloc *relo,
		2806	const struct bpf_core_spec *local_spec,
		2807	const struct bpf_core_spec *targ_spec)
2786	{	2808	{
		2809	__u32 orig_val, new_val;
2787	struct bpf_insn *insn;	2810	struct bpf_insn *insn;
2788	int insn_idx;	2811	int insn_idx;
2789	__u8 class;	2812	__u8 class;
2790		2813
2791	if (insn_off % sizeof(struct bpf_insn))	2814	if (relo->insn_off % sizeof(struct bpf_insn))
		2815	return -EINVAL;
		2816	insn_idx = relo->insn_off / sizeof(struct bpf_insn);
		2817
		2818	switch (relo->kind) {
		2819	case BPF_FIELD_BYTE_OFFSET:
		2820	orig_val = local_spec->offset;
		2821	if (targ_spec) {
		2822	new_val = targ_spec->offset;
		2823	} else {
		2824	pr_warn("prog '%s': patching insn #%d w/ failed reloc, imm %d -> %d\n",
		2825	bpf_program__title(prog, false), insn_idx,
		2826	orig_val, -1);
		2827	new_val = (__u32)-1;
		2828	}
		2829	break;
		2830	case BPF_FIELD_EXISTS:
		2831	orig_val = 1; /* can't generate EXISTS relo w/o local field */
		2832	new_val = targ_spec ? 1 : 0;
		2833	break;
		2834	default:
		2835	pr_warn("prog '%s': unknown relo %d at insn #%d'\n",
		2836	bpf_program__title(prog, false),
		2837	relo->kind, insn_idx);
2792	return -EINVAL;	2838	return -EINVAL;
2793	insn_idx = insn_off / sizeof(struct bpf_insn);	2839	}
2794		2840
2795	insn = &prog->insns[insn_idx];	2841	insn = &prog->insns[insn_idx];
2796	class = BPF_CLASS(insn->code);	2842	class = BPF_CLASS(insn->code);
@@ -2798,19 +2844,20 @@ static int bpf_core_reloc_insn(struct bpf_program *prog, int insn_off, · 1 unmodified line
2798	if (class == BPF_ALU \|\| class == BPF_ALU64) {	2844	if (class == BPF_ALU \|\| class == BPF_ALU64) {
2799	if (BPF_SRC(insn->code) != BPF_K)	2845	if (BPF_SRC(insn->code) != BPF_K)
2800	return -EINVAL;	2846	return -EINVAL;
2801	if (insn->imm != orig_off)	2847	if (insn->imm != orig_val)
2802	return -EINVAL;	2848	return -EINVAL;
2803	insn->imm = new_off;	2849	insn->imm = new_val;
2804	pr_debug("prog '%s': patched insn #%d (ALU/ALU64) imm %d -> %d\n",	2850	pr_debug("prog '%s': patched insn #%d (ALU/ALU64) imm %d -> %d\n",
2805	bpf_program__title(prog, false),	2851	bpf_program__title(prog, false),
2806	insn_idx, orig_off, new_off);	2852	insn_idx, orig_val, new_val);
2807	} else {	2853	} else {
2808	pr_warning("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",	2854	pr_warn("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n",
2809	bpf_program__title(prog, false),	2855	bpf_program__title(prog, false),
2810	insn_idx, insn->code, insn->src_reg, insn->dst_reg,	2856	insn_idx, insn->code, insn->src_reg, insn->dst_reg,
2811	insn->off, insn->imm);	2857	insn->off, insn->imm);
2812	return -EINVAL;	2858	return -EINVAL;
2813	}	2859	}
		2860
2814	return 0;	2861	return 0;
2815	}	2862	}
2816		2863
@@ -2896,7 +2943,7 @@ static struct btf *bpf_core_find_kernel_btf(void) · 79 unmodified lines
2896	return btf;	2943	return btf;
2897	}	2944	}
2898		2945
2899	pr_warning("failed to find valid kernel BTF\n");	2946	pr_warn("failed to find valid kernel BTF\n");
2900	return ERR_PTR(-ESRCH);	2947	return ERR_PTR(-ESRCH);
2901	}	2948	}
2902		2949
@@ -2977,7 +3024,7 @@ static void *u32_as_hash_key(__u32 x) · 74 unmodified lines
2977	* types should be compatible (see bpf_core_fields_are_compat for details).	3024	* types should be compatible (see bpf_core_fields_are_compat for details).
2978	* 3. It is supported and expected that there might be multiple flavors	3025	* 3. It is supported and expected that there might be multiple flavors
2979	* matching the spec. As long as all the specs resolve to the same set of	3026	* matching the spec. As long as all the specs resolve to the same set of
2980	* offsets across all candidates, there is not error. If there is any	3027	* offsets across all candidates, there is no error. If there is any
2981	* ambiguity, CO-RE relocation will fail. This is necessary to accomodate	3028	* ambiguity, CO-RE relocation will fail. This is necessary to accomodate
2982	* imprefection of BTF deduplication, which can cause slight duplication of	3029	* imprefection of BTF deduplication, which can cause slight duplication of
2983	* the same BTF type, if some directly or indirectly referenced (by	3030	* the same BTF type, if some directly or indirectly referenced (by
@@ -2992,12 +3039,12 @@ static void *u32_as_hash_key(__u32 x) · 8 unmodified lines
2992	* CPU-wise compared to prebuilding a map from all local type names to	3039	* CPU-wise compared to prebuilding a map from all local type names to
2993	* a list of candidate type names. It's also sped up by caching resolved	3040	* a list of candidate type names. It's also sped up by caching resolved
2994	* list of matching candidates per each local "root" type ID, that has at	3041	* list of matching candidates per each local "root" type ID, that has at
2995	* least one bpf_offset_reloc associated with it. This list is shared	3042	* least one bpf_field_reloc associated with it. This list is shared
2996	* between multiple relocations for the same type ID and is updated as some	3043	* between multiple relocations for the same type ID and is updated as some
2997	* of the candidates are pruned due to structural incompatibility.	3044	* of the candidates are pruned due to structural incompatibility.
2998	*/	3045	*/
2999	static int bpf_core_reloc_offset(struct bpf_program *prog,	3046	static int bpf_core_reloc_field(struct bpf_program *prog,
3000	const struct bpf_offset_reloc *relo,	3047	const struct bpf_field_reloc *relo,
3001	int relo_idx,	3048	int relo_idx,
3002	const struct btf *local_btf,	3049	const struct btf *local_btf,
3003	const struct btf *targ_btf,	3050	const struct btf *targ_btf,
@@ -3028,9 +3075,9 @@ static int bpf_core_reloc_offset(struct bpf_program *prog, · 24 unmodified lines
3028		3075
3029	err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec);	3076	err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec);
3030	if (err) {	3077	if (err) {
3031	pr_warning("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n",	3078	pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n",
3032	prog_name, relo_idx, local_id, local_name, spec_str,	3079	prog_name, relo_idx, local_id, local_name, spec_str,
3033	err);	3080	err);
3034	return -EINVAL;	3081	return -EINVAL;
3035	}	3082	}
3036		3083
@@ -3041,9 +3088,9 @@ static int bpf_core_reloc_offset(struct bpf_program *prog, · 4 unmodified lines
3041	if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {	3088	if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) {
3042	cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);	3089	cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf);
3043	if (IS_ERR(cand_ids)) {	3090	if (IS_ERR(cand_ids)) {
3044	pr_warning("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld",	3091	pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld",
3045	prog_name, relo_idx, local_id, local_name,	3092	prog_name, relo_idx, local_id, local_name,
3046	PTR_ERR(cand_ids));	3093	PTR_ERR(cand_ids));
3047	return PTR_ERR(cand_ids);	3094	return PTR_ERR(cand_ids);
3048	}	3095	}
3049	err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);	3096	err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL);
@@ -3065,8 +3112,8 @@ static int bpf_core_reloc_offset(struct bpf_program *prog, · 15 unmodified lines
3065	bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);	3112	bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec);
3066	libbpf_print(LIBBPF_DEBUG, ": %d\n", err);	3113	libbpf_print(LIBBPF_DEBUG, ": %d\n", err);
3067	if (err < 0) {	3114	if (err < 0) {
3068	pr_warning("prog '%s': relo #%d: matching error: %d\n",	3115	pr_warn("prog '%s': relo #%d: matching error: %d\n",
3069	prog_name, relo_idx, err);	3116	prog_name, relo_idx, err);
3070	return err;	3117	return err;
3071	}	3118	}
3072	if (err == 0)	3119	if (err == 0)
@@ -3078,27 +3125,38 @@ static int bpf_core_reloc_offset(struct bpf_program *prog, · 5 unmodified lines
3078	/* if there are many candidates, they should all	3125	/* if there are many candidates, they should all
3079	* resolve to the same offset	3126	* resolve to the same offset
3080	*/	3127	*/
3081	pr_warning("prog '%s': relo #%d: offset ambiguity: %u != %u\n",	3128	pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n",
3082	prog_name, relo_idx, cand_spec.offset,	3129	prog_name, relo_idx, cand_spec.offset,
3083	targ_spec.offset);	3130	targ_spec.offset);
3084	return -EINVAL;	3131	return -EINVAL;
3085	}	3132	}
3086		3133
3087	cand_ids->data[j++] = cand_spec.spec[0].type_id;	3134	cand_ids->data[j++] = cand_spec.spec[0].type_id;
3088	}	3135	}
3089		3136
3090	cand_ids->len = j;	3137	/*
3091	if (cand_ids->len == 0) {	3138	* For BPF_FIELD_EXISTS relo or when relaxed CO-RE reloc mode is
3092	pr_warning("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",	3139	* requested, it's expected that we might not find any candidates.
3093	prog_name, relo_idx, local_id, local_name, spec_str);	3140	* In this case, if field wasn't found in any candidate, the list of
		3141	* candidates shouldn't change at all, we'll just handle relocating
		3142	* appropriately, depending on relo's kind.
		3143	*/
		3144	if (j > 0)
		3145	cand_ids->len = j;
		3146
		3147	if (j == 0 && !prog->obj->relaxed_core_relocs &&
		3148	relo->kind != BPF_FIELD_EXISTS) {
		3149	pr_warn("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n",
		3150	prog_name, relo_idx, local_id, local_name, spec_str);
3094	return -ESRCH;	3151	return -ESRCH;
3095	}	3152	}
3096		3153
3097	err = bpf_core_reloc_insn(prog, relo->insn_off,	3154	/* bpf_core_reloc_insn should know how to handle missing targ_spec */
3098	local_spec.offset, targ_spec.offset);	3155	err = bpf_core_reloc_insn(prog, relo, &local_spec,
		3156	j ? &targ_spec : NULL);
3099	if (err) {	3157	if (err) {
3100	pr_warning("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",	3158	pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n",
3101	prog_name, relo_idx, relo->insn_off, err);	3159	prog_name, relo_idx, relo->insn_off, err);
3102	return -EINVAL;	3160	return -EINVAL;
3103	}	3161	}
3104		3162
@@ -3106,10 +3164,10 @@ static int bpf_core_reloc_offset(struct bpf_program *prog, · 1 unmodified line
3106	}	3164	}
3107		3165
3108	static int	3166	static int
3109	bpf_core_reloc_offsets(struct bpf_object obj, const char targ_btf_path)	3167	bpf_core_reloc_fields(struct bpf_object obj, const char targ_btf_path)
3110	{	3168	{
3111	const struct btf_ext_info_sec *sec;	3169	const struct btf_ext_info_sec *sec;
3112	const struct bpf_offset_reloc *rec;	3170	const struct bpf_field_reloc *rec;
3113	const struct btf_ext_info *seg;	3171	const struct btf_ext_info *seg;
3114	struct hashmap_entry *entry;	3172	struct hashmap_entry *entry;
3115	struct hashmap *cand_cache = NULL;	3173	struct hashmap *cand_cache = NULL;
@@ -3123,8 +3181,7 @@ bpf_core_reloc_offsets(struct bpf_object obj, const char targ_btf_path) · 7 unmodified lines
3123	else	3181	else
3124	targ_btf = bpf_core_find_kernel_btf();	3182	targ_btf = bpf_core_find_kernel_btf();
3125	if (IS_ERR(targ_btf)) {	3183	if (IS_ERR(targ_btf)) {
3126	pr_warning("failed to get target BTF: %ld\n",	3184	pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
3127	PTR_ERR(targ_btf));
3128	return PTR_ERR(targ_btf);	3185	return PTR_ERR(targ_btf);
3129	}	3186	}
3130		3187
@@ -3134,7 +3191,7 @@ bpf_core_reloc_offsets(struct bpf_object obj, const char targ_btf_path) · 3 unmodified lines
3134	goto out;	3191	goto out;
3135	}	3192	}
3136		3193
3137	seg = &obj->btf_ext->offset_reloc_info;	3194	seg = &obj->btf_ext->field_reloc_info;
3138	for_each_btf_ext_sec(seg, sec) {	3195	for_each_btf_ext_sec(seg, sec) {
3139	sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);	3196	sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3140	if (str_is_empty(sec_name)) {	3197	if (str_is_empty(sec_name)) {
@@ -3143,8 +3200,8 @@ bpf_core_reloc_offsets(struct bpf_object obj, const char targ_btf_path) · 2 unmodified lines
3143	}	3200	}
3144	prog = bpf_object__find_program_by_title(obj, sec_name);	3201	prog = bpf_object__find_program_by_title(obj, sec_name);
3145	if (!prog) {	3202	if (!prog) {
3146	pr_warning("failed to find program '%s' for CO-RE offset relocation\n",	3203	pr_warn("failed to find program '%s' for CO-RE offset relocation\n",
3147	sec_name);	3204	sec_name);
3148	err = -EINVAL;	3205	err = -EINVAL;
3149	goto out;	3206	goto out;
3150	}	3207	}
@@ -3153,11 +3210,11 @@ bpf_core_reloc_offsets(struct bpf_object obj, const char targ_btf_path) · 2 unmodified lines
3153	sec_name, sec->num_info);	3210	sec_name, sec->num_info);
3154		3211
3155	for_each_btf_ext_rec(seg, sec, i, rec) {	3212	for_each_btf_ext_rec(seg, sec, i, rec) {
3156	err = bpf_core_reloc_offset(prog, rec, i, obj->btf,	3213	err = bpf_core_reloc_field(prog, rec, i, obj->btf,
3157	targ_btf, cand_cache);	3214	targ_btf, cand_cache);
3158	if (err) {	3215	if (err) {
3159	pr_warning("prog '%s': relo #%d: failed to relocate: %d\n",	3216	pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
3160	sec_name, i, err);	3217	sec_name, i, err);
3161	goto out;	3218	goto out;
3162	}	3219	}
3163	}	3220	}
@@ -3179,8 +3236,8 @@ bpf_object__relocate_core(struct bpf_object obj, const char targ_btf_path) · 15 unmodified lines
3179	{	3236	{
3180	int err = 0;	3237	int err = 0;
3181		3238
3182	if (obj->btf_ext->offset_reloc_info.len)	3239	if (obj->btf_ext->field_reloc_info.len)
3183	err = bpf_core_reloc_offsets(obj, targ_btf_path);	3240	err = bpf_core_reloc_fields(obj, targ_btf_path);
3184		3241
3185	return err;	3242	return err;
3186	}	3243	}
@@ -3198,21 +3255,21 @@ bpf_program__reloc_text(struct bpf_program prog, struct bpf_object obj, · 11 unmodified lines
3198	return -LIBBPF_ERRNO__RELOC;	3255	return -LIBBPF_ERRNO__RELOC;
3199		3256
3200	if (prog->idx == obj->efile.text_shndx) {	3257	if (prog->idx == obj->efile.text_shndx) {
3201	pr_warning("relo in .text insn %d into off %d\n",	3258	pr_warn("relo in .text insn %d into off %d\n",
3202	relo->insn_idx, relo->text_off);	3259	relo->insn_idx, relo->text_off);
3203	return -LIBBPF_ERRNO__RELOC;	3260	return -LIBBPF_ERRNO__RELOC;
3204	}	3261	}
3205		3262
3206	if (prog->main_prog_cnt == 0) {	3263	if (prog->main_prog_cnt == 0) {
3207	text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);	3264	text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
3208	if (!text) {	3265	if (!text) {
3209	pr_warning("no .text section found yet relo into text exist\n");	3266	pr_warn("no .text section found yet relo into text exist\n");
3210	return -LIBBPF_ERRNO__RELOC;	3267	return -LIBBPF_ERRNO__RELOC;
3211	}	3268	}
3212	new_cnt = prog->insns_cnt + text->insns_cnt;	3269	new_cnt = prog->insns_cnt + text->insns_cnt;
3213	new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));	3270	new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
3214	if (!new_insn) {	3271	if (!new_insn) {
3215	pr_warning("oom in prog realloc\n");	3272	pr_warn("oom in prog realloc\n");
3216	return -ENOMEM;	3273	return -ENOMEM;
3217	}	3274	}
3218		3275
@@ -3267,8 +3324,8 @@ bpf_program__relocate(struct bpf_program prog, struct bpf_object obj) · 48 unmodified lines
3267	map_idx = prog->reloc_desc[i].map_idx;	3324	map_idx = prog->reloc_desc[i].map_idx;
3268		3325
3269	if (insn_idx + 1 >= (int)prog->insns_cnt) {	3326	if (insn_idx + 1 >= (int)prog->insns_cnt) {
3270	pr_warning("relocation out of range: '%s'\n",	3327	pr_warn("relocation out of range: '%s'\n",
3271	prog->section_name);	3328	prog->section_name);
3272	return -LIBBPF_ERRNO__RELOC;	3329	return -LIBBPF_ERRNO__RELOC;
3273	}	3330	}
3274		3331
@@ -3302,8 +3359,8 @@ bpf_object__relocate(struct bpf_object obj, const char targ_btf_path) · 27 unmodified lines
3302	if (obj->btf_ext) {	3359	if (obj->btf_ext) {
3303	err = bpf_object__relocate_core(obj, targ_btf_path);	3360	err = bpf_object__relocate_core(obj, targ_btf_path);
3304	if (err) {	3361	if (err) {
3305	pr_warning("failed to perform CO-RE relocations: %d\n",	3362	pr_warn("failed to perform CO-RE relocations: %d\n",
3306	err);	3363	err);
3307	return err;	3364	return err;
3308	}	3365	}
3309	}	3366	}
@@ -3312,8 +3369,7 @@ bpf_object__relocate(struct bpf_object obj, const char targ_btf_path) · 2 unmodified lines
3312		3369
3313	err = bpf_program__relocate(prog, obj);	3370	err = bpf_program__relocate(prog, obj);
3314	if (err) {	3371	if (err) {
3315	pr_warning("failed to relocate '%s'\n",	3372	pr_warn("failed to relocate '%s'\n", prog->section_name);
3316	prog->section_name);
3317	return err;	3373	return err;
3318	}	3374	}
3319	}	3375	}
@@ -3325,7 +3381,7 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) · 5 unmodified lines
3325	int i, err;	3381	int i, err;
3326		3382
3327	if (!obj_elf_valid(obj)) {	3383	if (!obj_elf_valid(obj)) {
3328	pr_warning("Internal error: elf object is closed\n");	3384	pr_warn("Internal error: elf object is closed\n");
3329	return -LIBBPF_ERRNO__INTERNAL;	3385	return -LIBBPF_ERRNO__INTERNAL;
3330	}	3386	}
3331		3387
@@ -3336,13 +3392,13 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) · 4 unmodified lines
3336	struct bpf_program *prog;	3392	struct bpf_program *prog;
3337		3393
3338	if (shdr->sh_type != SHT_REL) {	3394	if (shdr->sh_type != SHT_REL) {
3339	pr_warning("internal error at %d\n", __LINE__);	3395	pr_warn("internal error at %d\n", __LINE__);
3340	return -LIBBPF_ERRNO__INTERNAL;	3396	return -LIBBPF_ERRNO__INTERNAL;
3341	}	3397	}
3342		3398
3343	prog = bpf_object__find_prog_by_idx(obj, idx);	3399	prog = bpf_object__find_prog_by_idx(obj, idx);
3344	if (!prog) {	3400	if (!prog) {
3345	pr_warning("relocation failed: no section(%d)\n", idx);	3401	pr_warn("relocation failed: no section(%d)\n", idx);
3346	return -LIBBPF_ERRNO__RELOC;	3402	return -LIBBPF_ERRNO__RELOC;
3347	}	3403	}
3348		3404
@@ -3394,7 +3450,7 @@ load_program(struct bpf_program prog, struct bpf_insn insns, int insns_cnt, · 45 unmodified lines
3394	retry_load:	3450	retry_load:
3395	log_buf = malloc(log_buf_size);	3451	log_buf = malloc(log_buf_size);
3396	if (!log_buf)	3452	if (!log_buf)
3397	pr_warning("Alloc log buffer for bpf loader error, continue without log\n");	3453	pr_warn("Alloc log buffer for bpf loader error, continue without log\n");
3398		3454
3399	ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);	3455	ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
3400		3456
@@ -3413,16 +3469,16 @@ retry_load: · 12 unmodified lines
3413	}	3469	}
3414	ret = -LIBBPF_ERRNO__LOAD;	3470	ret = -LIBBPF_ERRNO__LOAD;
3415	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));	3471	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3416	pr_warning("load bpf program failed: %s\n", cp);	3472	pr_warn("load bpf program failed: %s\n", cp);
3417		3473
3418	if (log_buf && log_buf[0] != '\0') {	3474	if (log_buf && log_buf[0] != '\0') {
3419	ret = -LIBBPF_ERRNO__VERIFY;	3475	ret = -LIBBPF_ERRNO__VERIFY;
3420	pr_warning("-- BEGIN DUMP LOG ---\n");	3476	pr_warn("-- BEGIN DUMP LOG ---\n");
3421	pr_warning("\n%s\n", log_buf);	3477	pr_warn("\n%s\n", log_buf);
3422	pr_warning("-- END LOG --\n");	3478	pr_warn("-- END LOG --\n");
3423	} else if (load_attr.insns_cnt >= BPF_MAXINSNS) {	3479	} else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
3424	pr_warning("Program too large (%zu insns), at most %d insns\n",	3480	pr_warn("Program too large (%zu insns), at most %d insns\n",
3425	load_attr.insns_cnt, BPF_MAXINSNS);	3481	load_attr.insns_cnt, BPF_MAXINSNS);
3426	ret = -LIBBPF_ERRNO__PROG2BIG;	3482	ret = -LIBBPF_ERRNO__PROG2BIG;
3427	} else {	3483	} else {
3428	/* Wrong program type? */	3484	/* Wrong program type? */
@@ -3456,14 +3512,14 @@ bpf_program__load(struct bpf_program *prog, · 27 unmodified lines
3456		3512
3457	if (prog->instances.nr < 0 \|\| !prog->instances.fds) {	3513	if (prog->instances.nr < 0 \|\| !prog->instances.fds) {
3458	if (prog->preprocessor) {	3514	if (prog->preprocessor) {
3459	pr_warning("Internal error: can't load program '%s'\n",	3515	pr_warn("Internal error: can't load program '%s'\n",
3460	prog->section_name);	3516	prog->section_name);
3461	return -LIBBPF_ERRNO__INTERNAL;	3517	return -LIBBPF_ERRNO__INTERNAL;
3462	}	3518	}
3463		3519
3464	prog->instances.fds = malloc(sizeof(int));	3520	prog->instances.fds = malloc(sizeof(int));
3465	if (!prog->instances.fds) {	3521	if (!prog->instances.fds) {
3466	pr_warning("Not enough memory for BPF fds\n");	3522	pr_warn("Not enough memory for BPF fds\n");
3467	return -ENOMEM;	3523	return -ENOMEM;
3468	}	3524	}
3469	prog->instances.nr = 1;	3525	prog->instances.nr = 1;
@@ -3472,8 +3528,8 @@ bpf_program__load(struct bpf_program *prog, · 2 unmodified lines
3472		3528
3473	if (!prog->preprocessor) {	3529	if (!prog->preprocessor) {
3474	if (prog->instances.nr != 1) {	3530	if (prog->instances.nr != 1) {
3475	pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",	3531	pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n",
3476	prog->section_name, prog->instances.nr);	3532	prog->section_name, prog->instances.nr);
3477	}	3533	}
3478	err = load_program(prog, prog->insns, prog->insns_cnt,	3534	err = load_program(prog, prog->insns, prog->insns_cnt,
3479	license, kern_version, &fd);	3535	license, kern_version, &fd);
@@ -3490,8 +3546,8 @@ bpf_program__load(struct bpf_program *prog, · 10 unmodified lines
3490	err = preprocessor(prog, i, prog->insns,	3546	err = preprocessor(prog, i, prog->insns,
3491	prog->insns_cnt, &result);	3547	prog->insns_cnt, &result);
3492	if (err) {	3548	if (err) {
3493	pr_warning("Preprocessing the %dth instance of program '%s' failed\n",	3549	pr_warn("Preprocessing the %dth instance of program '%s' failed\n",
3494	i, prog->section_name);	3550	i, prog->section_name);
3495	goto out;	3551	goto out;
3496	}	3552	}
3497		3553
@@ -3509,8 +3565,8 @@ bpf_program__load(struct bpf_program *prog, · 11 unmodified lines
3509	license, kern_version, &fd);	3565	license, kern_version, &fd);
3510		3566
3511	if (err) {	3567	if (err) {
3512	pr_warning("Loading the %dth instance of program '%s' failed\n",	3568	pr_warn("Loading the %dth instance of program '%s' failed\n",
3513	i, prog->section_name);	3569	i, prog->section_name);
3514	goto out;	3570	goto out;
3515	}	3571	}
3516		3572
@@ -3520,8 +3576,7 @@ bpf_program__load(struct bpf_program *prog, · 3 unmodified lines
3520	}	3576	}
3521	out:	3577	out:
3522	if (err)	3578	if (err)
3523	pr_warning("failed to load program '%s'\n",	3579	pr_warn("failed to load program '%s'\n", prog->section_name);
3524	prog->section_name);
3525	zfree(&prog->insns);	3580	zfree(&prog->insns);
3526	prog->insns_cnt = 0;	3581	prog->insns_cnt = 0;
3527	return err;	3582	return err;
@@ -3554,27 +3609,66 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) · 26 unmodified lines
3554		3609
3555	static struct bpf_object *	3610	static struct bpf_object *
3556	__bpf_object__open(const char path, const void obj_buf, size_t obj_buf_sz,	3611	__bpf_object__open(const char path, const void obj_buf, size_t obj_buf_sz,
3557	const char *obj_name, int flags)	3612	struct bpf_object_open_opts *opts)
3558	{	3613	{
		3614	struct bpf_program *prog;
3559	struct bpf_object *obj;	3615	struct bpf_object *obj;
		3616	const char *obj_name;
		3617	char tmp_name[64];
		3618	bool relaxed_maps;
3560	int err;	3619	int err;
3561		3620
3562	if (elf_version(EV_CURRENT) == EV_NONE) {	3621	if (elf_version(EV_CURRENT) == EV_NONE) {
3563	pr_warning("failed to init libelf for %s\n", path);	3622	pr_warn("failed to init libelf for %s\n",
		3623	path ? : "(mem buf)");
3564	return ERR_PTR(-LIBBPF_ERRNO__LIBELF);	3624	return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
3565	}	3625	}
3566		3626
		3627	if (!OPTS_VALID(opts, bpf_object_open_opts))
		3628	return ERR_PTR(-EINVAL);
		3629
		3630	obj_name = OPTS_GET(opts, object_name, path);
		3631	if (obj_buf) {
		3632	if (!obj_name) {
		3633	snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
		3634	(unsigned long)obj_buf,
		3635	(unsigned long)obj_buf_sz);
		3636	obj_name = tmp_name;
		3637	}
		3638	path = obj_name;
		3639	pr_debug("loading object '%s' from buffer\n", obj_name);
		3640	}
		3641
3567	obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);	3642	obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
3568	if (IS_ERR(obj))	3643	if (IS_ERR(obj))
3569	return obj;	3644	return obj;
3570		3645
		3646	obj->relaxed_core_relocs = OPTS_GET(opts, relaxed_core_relocs, false);
		3647	relaxed_maps = OPTS_GET(opts, relaxed_maps, false);
		3648
3571	CHECK_ERR(bpf_object__elf_init(obj), err, out);	3649	CHECK_ERR(bpf_object__elf_init(obj), err, out);
3572	CHECK_ERR(bpf_object__check_endianness(obj), err, out);	3650	CHECK_ERR(bpf_object__check_endianness(obj), err, out);
3573	CHECK_ERR(bpf_object__probe_caps(obj), err, out);	3651	CHECK_ERR(bpf_object__probe_caps(obj), err, out);
3574	CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out);	3652	CHECK_ERR(bpf_object__elf_collect(obj, relaxed_maps), err, out);
3575	CHECK_ERR(bpf_object__collect_reloc(obj), err, out);	3653	CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
3576
3577	bpf_object__elf_finish(obj);	3654	bpf_object__elf_finish(obj);
		3655
		3656	bpf_object__for_each_program(prog, obj) {
		3657	enum bpf_prog_type prog_type;
		3658	enum bpf_attach_type attach_type;
		3659
		3660	err = libbpf_prog_type_by_name(prog->section_name, &prog_type,
		3661	&attach_type);
		3662	if (err == -ESRCH)
		3663	/* couldn't guess, but user might manually specify */
		3664	continue;
		3665	if (err)
		3666	goto out;
		3667
		3668	bpf_program__set_type(prog, prog_type);
		3669	bpf_program__set_expected_attach_type(prog, attach_type);
		3670	}
		3671
3578	return obj;	3672	return obj;
3579	out:	3673	out:
3580	bpf_object__close(obj);	3674	bpf_object__close(obj);
@@ -3584,13 +3678,16 @@ out: · 3 unmodified lines
3584	static struct bpf_object *	3678	static struct bpf_object *
3585	__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)	3679	__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags)
3586	{	3680	{
		3681	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
		3682	.relaxed_maps = flags & MAPS_RELAX_COMPAT,
		3683	);
		3684
3587	/* param validation */	3685	/* param validation */
3588	if (!attr->file)	3686	if (!attr->file)
3589	return NULL;	3687	return NULL;
3590		3688
3591	pr_debug("loading %s\n", attr->file);	3689	pr_debug("loading %s\n", attr->file);
3592		3690	return __bpf_object__open(attr->file, NULL, 0, &opts);
3593	return __bpf_object__open(attr->file, NULL, 0, NULL, flags);
3594	}	3691	}
3595		3692
3596	struct bpf_object bpf_object__open_xattr(struct bpf_object_open_attr attr)	3693	struct bpf_object bpf_object__open_xattr(struct bpf_object_open_attr attr)
@@ -3611,54 +3708,29 @@ struct bpf_object bpf_object__open(const char path) · 14 unmodified lines
3611	struct bpf_object *	3708	struct bpf_object *
3612	bpf_object__open_file(const char path, struct bpf_object_open_opts opts)	3709	bpf_object__open_file(const char path, struct bpf_object_open_opts opts)
3613	{	3710	{
3614	const char *obj_name;
3615	bool relaxed_maps;
3616
3617	if (!OPTS_VALID(opts, bpf_object_open_opts))
3618	return ERR_PTR(-EINVAL);
3619	if (!path)	3711	if (!path)
3620	return ERR_PTR(-EINVAL);	3712	return ERR_PTR(-EINVAL);
3621		3713
3622	pr_debug("loading %s\n", path);	3714	pr_debug("loading %s\n", path);
3623		3715
3624	obj_name = OPTS_GET(opts, object_name, path);	3716	return __bpf_object__open(path, NULL, 0, opts);
3625	relaxed_maps = OPTS_GET(opts, relaxed_maps, false);
3626	return __bpf_object__open(path, NULL, 0, obj_name,
3627	relaxed_maps ? MAPS_RELAX_COMPAT : 0);
3628	}	3717	}
3629		3718
3630	struct bpf_object *	3719	struct bpf_object *
3631	bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,	3720	bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz,
3632	struct bpf_object_open_opts *opts)	3721	struct bpf_object_open_opts *opts)
3633	{	3722	{
3634	char tmp_name[64];
3635	const char *obj_name;
3636	bool relaxed_maps;
3637
3638	if (!OPTS_VALID(opts, bpf_object_open_opts))
3639	return ERR_PTR(-EINVAL);
3640	if (!obj_buf \|\| obj_buf_sz == 0)	3723	if (!obj_buf \|\| obj_buf_sz == 0)
3641	return ERR_PTR(-EINVAL);	3724	return ERR_PTR(-EINVAL);
3642		3725
3643	obj_name = OPTS_GET(opts, object_name, NULL);	3726	return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts);
3644	if (!obj_name) {
3645	snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
3646	(unsigned long)obj_buf,
3647	(unsigned long)obj_buf_sz);
3648	obj_name = tmp_name;
3649	}
3650	pr_debug("loading object '%s' from buffer\n", obj_name);
3651
3652	relaxed_maps = OPTS_GET(opts, relaxed_maps, false);
3653	return __bpf_object__open(obj_name, obj_buf, obj_buf_sz, obj_name,
3654	relaxed_maps ? MAPS_RELAX_COMPAT : 0);
3655	}	3727	}
3656		3728
3657	struct bpf_object *	3729	struct bpf_object *
3658	bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,	3730	bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz,
3659	const char *name)	3731	const char *name)
3660	{	3732	{
3661	LIBBPF_OPTS(bpf_object_open_opts, opts,	3733	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
3662	.object_name = name,	3734	.object_name = name,
3663	/* wrong default, but backwards-compatible */	3735	/* wrong default, but backwards-compatible */
3664	.relaxed_maps = true,	3736	.relaxed_maps = true,
@@ -3699,7 +3771,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) · 34 unmodified lines
3699	return -EINVAL;	3771	return -EINVAL;
3700		3772
3701	if (obj->loaded) {	3773	if (obj->loaded) {
3702	pr_warning("object should not be loaded twice\n");	3774	pr_warn("object should not be loaded twice\n");
3703	return -EINVAL;	3775	return -EINVAL;
3704	}	3776	}
3705		3777
@@ -3712,7 +3784,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) · 6 unmodified lines
3712	return 0;	3784	return 0;
3713	out:	3785	out:
3714	bpf_object__unload(obj);	3786	bpf_object__unload(obj);
3715	pr_warning("failed to load object '%s'\n", obj->path);	3787	pr_warn("failed to load object '%s'\n", obj->path);
3716	return err;	3788	return err;
3717	}	3789	}
3718		3790
@@ -3742,13 +3814,13 @@ static int check_path(const char *path) · 23 unmodified lines
3742	dir = dirname(dname);	3814	dir = dirname(dname);
3743	if (statfs(dir, &st_fs)) {	3815	if (statfs(dir, &st_fs)) {
3744	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));	3816	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3745	pr_warning("failed to statfs %s: %s\n", dir, cp);	3817	pr_warn("failed to statfs %s: %s\n", dir, cp);
3746	err = -errno;	3818	err = -errno;
3747	}	3819	}
3748	free(dname);	3820	free(dname);
3749		3821
3750	if (!err && st_fs.f_type != BPF_FS_MAGIC) {	3822	if (!err && st_fs.f_type != BPF_FS_MAGIC) {
3751	pr_warning("specified path %s is not on BPF FS\n", path);	3823	pr_warn("specified path %s is not on BPF FS\n", path);
3752	err = -EINVAL;	3824	err = -EINVAL;
3753	}	3825	}
3754		3826
@@ -3766,19 +3838,19 @@ int bpf_program__pin_instance(struct bpf_program prog, const char path, · 11 unmodified lines
3766	return err;	3838	return err;
3767		3839
3768	if (prog == NULL) {	3840	if (prog == NULL) {
3769	pr_warning("invalid program pointer\n");	3841	pr_warn("invalid program pointer\n");
3770	return -EINVAL;	3842	return -EINVAL;
3771	}	3843	}
3772		3844
3773	if (instance < 0 \|\| instance >= prog->instances.nr) {	3845	if (instance < 0 \|\| instance >= prog->instances.nr) {
3774	pr_warning("invalid prog instance %d of prog %s (max %d)\n",	3846	pr_warn("invalid prog instance %d of prog %s (max %d)\n",
3775	instance, prog->section_name, prog->instances.nr);	3847	instance, prog->section_name, prog->instances.nr);
3776	return -EINVAL;	3848	return -EINVAL;
3777	}	3849	}
3778		3850
3779	if (bpf_obj_pin(prog->instances.fds[instance], path)) {	3851	if (bpf_obj_pin(prog->instances.fds[instance], path)) {
3780	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));	3852	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3781	pr_warning("failed to pin program: %s\n", cp);	3853	pr_warn("failed to pin program: %s\n", cp);
3782	return -errno;	3854	return -errno;
3783	}	3855	}
3784	pr_debug("pinned program '%s'\n", path);	3856	pr_debug("pinned program '%s'\n", path);
@@ -3796,13 +3868,13 @@ int bpf_program__unpin_instance(struct bpf_program prog, const char path, · 11 unmodified lines
3796	return err;	3868	return err;
3797		3869
3798	if (prog == NULL) {	3870	if (prog == NULL) {
3799	pr_warning("invalid program pointer\n");	3871	pr_warn("invalid program pointer\n");
3800	return -EINVAL;	3872	return -EINVAL;
3801	}	3873	}
3802		3874
3803	if (instance < 0 \|\| instance >= prog->instances.nr) {	3875	if (instance < 0 \|\| instance >= prog->instances.nr) {
3804	pr_warning("invalid prog instance %d of prog %s (max %d)\n",	3876	pr_warn("invalid prog instance %d of prog %s (max %d)\n",
3805	instance, prog->section_name, prog->instances.nr);	3877	instance, prog->section_name, prog->instances.nr);
3806	return -EINVAL;	3878	return -EINVAL;
3807	}	3879	}
3808		3880
@@ -3824,7 +3896,7 @@ static int make_dir(const char *path) · 15 unmodified lines
3824		3896
3825	if (err) {	3897	if (err) {
3826	cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));	3898	cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
3827	pr_warning("failed to mkdir %s: %s\n", path, cp);	3899	pr_warn("failed to mkdir %s: %s\n", path, cp);
3828	}	3900	}
3829	return err;	3901	return err;
3830	}	3902	}
@@ -3838,12 +3910,12 @@ int bpf_program__pin(struct bpf_program prog, const char path) · 7 unmodified lines
3838	return err;	3910	return err;
3839		3911
3840	if (prog == NULL) {	3912	if (prog == NULL) {
3841	pr_warning("invalid program pointer\n");	3913	pr_warn("invalid program pointer\n");
3842	return -EINVAL;	3914	return -EINVAL;
3843	}	3915	}
3844		3916
3845	if (prog->instances.nr <= 0) {	3917	if (prog->instances.nr <= 0) {
3846	pr_warning("no instances of prog %s to pin\n",	3918	pr_warn("no instances of prog %s to pin\n",
3847	prog->section_name);	3919	prog->section_name);
3848	return -EINVAL;	3920	return -EINVAL;
3849	}	3921	}
@@ -3905,12 +3977,12 @@ int bpf_program__unpin(struct bpf_program prog, const char path) · 55 unmodified lines
3905	return err;	3977	return err;
3906		3978
3907	if (prog == NULL) {	3979	if (prog == NULL) {
3908	pr_warning("invalid program pointer\n");	3980	pr_warn("invalid program pointer\n");
3909	return -EINVAL;	3981	return -EINVAL;
3910	}	3982	}
3911		3983
3912	if (prog->instances.nr <= 0) {	3984	if (prog->instances.nr <= 0) {
3913	pr_warning("no instances of prog %s to pin\n",	3985	pr_warn("no instances of prog %s to pin\n",
3914	prog->section_name);	3986	prog->section_name);
3915	return -EINVAL;	3987	return -EINVAL;
3916	}	3988	}
@@ -3952,13 +4024,13 @@ int bpf_map__pin(struct bpf_map map, const char path) · 35 unmodified lines
3952	return err;	4024	return err;
3953		4025
3954	if (map == NULL) {	4026	if (map == NULL) {
3955	pr_warning("invalid map pointer\n");	4027	pr_warn("invalid map pointer\n");
3956	return -EINVAL;	4028	return -EINVAL;
3957	}	4029	}
3958		4030
3959	if (bpf_obj_pin(map->fd, path)) {	4031	if (bpf_obj_pin(map->fd, path)) {
3960	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));	4032	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
3961	pr_warning("failed to pin map: %s\n", cp);	4033	pr_warn("failed to pin map: %s\n", cp);
3962	return -errno;	4034	return -errno;
3963	}	4035	}
3964		4036
@@ -3976,7 +4048,7 @@ int bpf_map__unpin(struct bpf_map map, const char path) · 11 unmodified lines
3976	return err;	4048	return err;
3977		4049
3978	if (map == NULL) {	4050	if (map == NULL) {
3979	pr_warning("invalid map pointer\n");	4051	pr_warn("invalid map pointer\n");
3980	return -EINVAL;	4052	return -EINVAL;
3981	}	4053	}
3982		4054
@@ -3997,7 +4069,7 @@ int bpf_object__pin_maps(struct bpf_object obj, const char path) · 14 unmodified lines
3997	return -ENOENT;	4069	return -ENOENT;
3998		4070
3999	if (!obj->loaded) {	4071	if (!obj->loaded) {
4000	pr_warning("object not yet loaded; load it first\n");	4072	pr_warn("object not yet loaded; load it first\n");
4001	return -ENOENT;	4073	return -ENOENT;
4002	}	4074	}
4003		4075
@@ -4080,7 +4152,7 @@ int bpf_object__pin_programs(struct bpf_object obj, const char path) · 76 unmodified lines
4080	return -ENOENT;	4152	return -ENOENT;
4081		4153
4082	if (!obj->loaded) {	4154	if (!obj->loaded) {
4083	pr_warning("object not yet loaded; load it first\n");	4155	pr_warn("object not yet loaded; load it first\n");
4084	return -ENOENT;	4156	return -ENOENT;
4085	}	4157	}
4086		4158
@@ -4281,7 +4353,7 @@ __bpf_program__iter(const struct bpf_program p, const struct bpf_object obj, · 194 unmodified lines
4281	&obj->programs[nr_programs - 1];	4353	&obj->programs[nr_programs - 1];
4282		4354
4283	if (p->obj != obj) {	4355	if (p->obj != obj) {
4284	pr_warning("error: program handler doesn't match object\n");	4356	pr_warn("error: program handler doesn't match object\n");
4285	return NULL;	4357	return NULL;
4286	}	4358	}
4287		4359
@@ -4344,7 +4416,7 @@ const char bpf_program__title(const struct bpf_program prog, bool needs_copy) · 56 unmodified lines
4344	if (needs_copy) {	4416	if (needs_copy) {
4345	title = strdup(title);	4417	title = strdup(title);
4346	if (!title) {	4418	if (!title) {
4347	pr_warning("failed to strdup program title\n");	4419	pr_warn("failed to strdup program title\n");
4348	return ERR_PTR(-ENOMEM);	4420	return ERR_PTR(-ENOMEM);
4349	}	4421	}
4350	}	4422	}
@@ -4366,13 +4438,13 @@ int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, · 15 unmodified lines
4366	return -EINVAL;	4438	return -EINVAL;
4367		4439
4368	if (prog->instances.nr > 0 \|\| prog->instances.fds) {	4440	if (prog->instances.nr > 0 \|\| prog->instances.fds) {
4369	pr_warning("Can't set pre-processor after loading\n");	4441	pr_warn("Can't set pre-processor after loading\n");
4370	return -EINVAL;	4442	return -EINVAL;
4371	}	4443	}
4372		4444
4373	instances_fds = malloc(sizeof(int) * nr_instances);	4445	instances_fds = malloc(sizeof(int) * nr_instances);
4374	if (!instances_fds) {	4446	if (!instances_fds) {
4375	pr_warning("alloc memory failed for fds\n");	4447	pr_warn("alloc memory failed for fds\n");
4376	return -ENOMEM;	4448	return -ENOMEM;
4377	}	4449	}
4378		4450
@@ -4393,21 +4465,26 @@ int bpf_program__nth_fd(const struct bpf_program *prog, int n) · 14 unmodified lines
4393	return -EINVAL;	4465	return -EINVAL;
4394		4466
4395	if (n >= prog->instances.nr \|\| n < 0) {	4467	if (n >= prog->instances.nr \|\| n < 0) {
4396	pr_warning("Can't get the %dth fd from program %s: only %d instances\n",	4468	pr_warn("Can't get the %dth fd from program %s: only %d instances\n",
4397	n, prog->section_name, prog->instances.nr);	4469	n, prog->section_name, prog->instances.nr);
4398	return -EINVAL;	4470	return -EINVAL;
4399	}	4471	}
4400		4472
4401	fd = prog->instances.fds[n];	4473	fd = prog->instances.fds[n];
4402	if (fd < 0) {	4474	if (fd < 0) {
4403	pr_warning("%dth instance of program '%s' is invalid\n",	4475	pr_warn("%dth instance of program '%s' is invalid\n",
4404	n, prog->section_name);	4476	n, prog->section_name);
4405	return -ENOENT;	4477	return -ENOENT;
4406	}	4478	}
4407		4479
4408	return fd;	4480	return fd;
4409	}	4481	}
4410		4482
		4483	enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog)
		4484	{
		4485	return prog->type;
		4486	}
		4487
4411	void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)	4488	void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
4412	{	4489	{
4413	prog->type = type;	4490	prog->type = type;
@@ -4442,25 +4519,34 @@ BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); · 28 unmodified lines
4442	BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);	4519	BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
4443	BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);	4520	BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
4444		4521
		4522	enum bpf_attach_type
		4523	bpf_program__get_expected_attach_type(struct bpf_program *prog)
		4524	{
		4525	return prog->expected_attach_type;
		4526	}
		4527
4445	void bpf_program__set_expected_attach_type(struct bpf_program *prog,	4528	void bpf_program__set_expected_attach_type(struct bpf_program *prog,
4446	enum bpf_attach_type type)	4529	enum bpf_attach_type type)
4447	{	4530	{
4448	prog->expected_attach_type = type;	4531	prog->expected_attach_type = type;
4449	}	4532	}
4450		4533
4451	#define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \	4534	#define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, btf, atype) \
4452	{ string, sizeof(string) - 1, ptype, eatype, is_attachable, atype }	4535	{ string, sizeof(string) - 1, ptype, eatype, is_attachable, btf, atype }
4453		4536
4454	/* Programs that can NOT be attached. */	4537	/* Programs that can NOT be attached. */
4455	#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0)	4538	#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0)
4456		4539
4457	/* Programs that can be attached. */	4540	/* Programs that can be attached. */
4458	#define BPF_APROG_SEC(string, ptype, atype) \	4541	#define BPF_APROG_SEC(string, ptype, atype) \
4459	BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype)	4542	BPF_PROG_SEC_IMPL(string, ptype, 0, 1, 0, atype)
4460		4543
4461	/* Programs that must specify expected attach type at load time. */	4544	/* Programs that must specify expected attach type at load time. */
4462	#define BPF_EAPROG_SEC(string, ptype, eatype) \	4545	#define BPF_EAPROG_SEC(string, ptype, eatype) \
4463	BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype)	4546	BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, 0, eatype)
		4547
		4548	/* Programs that use BTF to identify attach point */
		4549	#define BPF_PROG_BTF(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 1, 0)
4464		4550
4465	/* Programs that can be attached but attach type can't be identified by section	4551	/* Programs that can be attached but attach type can't be identified by section
4466	* name. Kept for backward compatibility.	4552	* name. Kept for backward compatibility.
@@ -4472,16 +4558,22 @@ static const struct { · 5 unmodified lines
4472	size_t len;	4558	size_t len;
4473	enum bpf_prog_type prog_type;	4559	enum bpf_prog_type prog_type;
4474	enum bpf_attach_type expected_attach_type;	4560	enum bpf_attach_type expected_attach_type;
4475	int is_attachable;	4561	bool is_attachable;
		4562	bool is_attach_btf;
4476	enum bpf_attach_type attach_type;	4563	enum bpf_attach_type attach_type;
4477	} section_names[] = {	4564	} section_names[] = {
4478	BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),	4565	BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
4479	BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE),	4566	BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE),
		4567	BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE),
4480	BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE),	4568	BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE),
		4569	BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE),
4481	BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),	4570	BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
4482	BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),	4571	BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
4483	BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT),	4572	BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT),
		4573	BPF_PROG_SEC("tp/", BPF_PROG_TYPE_TRACEPOINT),
4484	BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT),	4574	BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT),
		4575	BPF_PROG_SEC("raw_tp/", BPF_PROG_TYPE_RAW_TRACEPOINT),
		4576	BPF_PROG_BTF("tp_btf/", BPF_PROG_TYPE_RAW_TRACEPOINT),
4485	BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),	4577	BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),
4486	BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),	4578	BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
4487	BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),	4579	BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
@@ -4586,16 +4678,37 @@ int libbpf_prog_type_by_name(const char name, enum bpf_prog_type prog_type, · 98 unmodified lines
4586	continue;	4678	continue;
4587	*prog_type = section_names[i].prog_type;	4679	*prog_type = section_names[i].prog_type;
4588	*expected_attach_type = section_names[i].expected_attach_type;	4680	*expected_attach_type = section_names[i].expected_attach_type;
		4681	if (section_names[i].is_attach_btf) {
		4682	struct btf *btf = bpf_core_find_kernel_btf();
		4683	char raw_tp_btf_name[128] = "btf_trace_";
		4684	char *dst = raw_tp_btf_name + sizeof("btf_trace_") - 1;
		4685	int ret;
		4686
		4687	if (IS_ERR(btf)) {
		4688	pr_warn("vmlinux BTF is not found\n");
		4689	return -EINVAL;
		4690	}
		4691	/* prepend "btf_trace_" prefix per kernel convention */
		4692	strncat(dst, name + section_names[i].len,
		4693	sizeof(raw_tp_btf_name) - sizeof("btf_trace_"));
		4694	ret = btf__find_by_name(btf, raw_tp_btf_name);
		4695	btf__free(btf);
		4696	if (ret <= 0) {
		4697	pr_warn("%s is not found in vmlinux BTF\n", dst);
		4698	return -EINVAL;
		4699	}
		4700	*expected_attach_type = ret;
		4701	}
4589	return 0;	4702	return 0;
4590	}	4703	}
4591	pr_warning("failed to guess program type based on ELF section name '%s'\n", name);	4704	pr_warn("failed to guess program type based on ELF section name '%s'\n", name);
4592	type_names = libbpf_get_type_names(false);	4705	type_names = libbpf_get_type_names(false);
4593	if (type_names != NULL) {	4706	if (type_names != NULL) {
4594	pr_info("supported section(type) names are:%s\n", type_names);	4707	pr_info("supported section(type) names are:%s\n", type_names);
4595	free(type_names);	4708	free(type_names);
4596	}	4709	}
4597		4710
4598	return -EINVAL;	4711	return -ESRCH;
4599	}	4712	}
4600		4713
4601	int libbpf_attach_type_by_name(const char *name,	4714	int libbpf_attach_type_by_name(const char *name,
@@ -4615,7 +4728,7 @@ int libbpf_attach_type_by_name(const char *name, · 13 unmodified lines
4615	*attach_type = section_names[i].attach_type;	4728	*attach_type = section_names[i].attach_type;
4616	return 0;	4729	return 0;
4617	}	4730	}
4618	pr_warning("failed to guess attach type based on ELF section name '%s'\n", name);	4731	pr_warn("failed to guess attach type based on ELF section name '%s'\n", name);
4619	type_names = libbpf_get_type_names(true);	4732	type_names = libbpf_get_type_names(true);
4620	if (type_names != NULL) {	4733	if (type_names != NULL) {
4621	pr_info("attachable section(type) names are:%s\n", type_names);	4734	pr_info("attachable section(type) names are:%s\n", type_names);
@@ -4625,15 +4738,6 @@ int libbpf_attach_type_by_name(const char *name, · 3 unmodified lines
4625	return -EINVAL;	4738	return -EINVAL;
4626	}	4739	}
4627		4740
4628	static int
4629	bpf_program__identify_section(struct bpf_program *prog,
4630	enum bpf_prog_type *prog_type,
4631	enum bpf_attach_type *expected_attach_type)
4632	{
4633	return libbpf_prog_type_by_name(prog->section_name, prog_type,
4634	expected_attach_type);
4635	}
4636
4637	int bpf_map__fd(const struct bpf_map *map)	4741	int bpf_map__fd(const struct bpf_map *map)
4638	{	4742	{
4639	return map ? map->fd : -EINVAL;	4743	return map ? map->fd : -EINVAL;
@@ -4698,11 +4802,11 @@ void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) · 58 unmodified lines
4698	int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)	4802	int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
4699	{	4803	{
4700	if (!bpf_map_type__is_map_in_map(map->def.type)) {	4804	if (!bpf_map_type__is_map_in_map(map->def.type)) {
4701	pr_warning("error: unsupported map type\n");	4805	pr_warn("error: unsupported map type\n");
4702	return -EINVAL;	4806	return -EINVAL;
4703	}	4807	}
4704	if (map->inner_map_fd != -1) {	4808	if (map->inner_map_fd != -1) {
4705	pr_warning("error: inner_map_fd already specified\n");	4809	pr_warn("error: inner_map_fd already specified\n");
4706	return -EINVAL;	4810	return -EINVAL;
4707	}	4811	}
4708	map->inner_map_fd = fd;	4812	map->inner_map_fd = fd;
@@ -4722,8 +4826,8 @@ __bpf_map__iter(const struct bpf_map m, const struct bpf_object obj, int i) · 13 unmodified lines
4722	e = obj->maps + obj->nr_maps;	4826	e = obj->maps + obj->nr_maps;
4723		4827
4724	if ((m < s) \|\| (m >= e)) {	4828	if ((m < s) \|\| (m >= e)) {
4725	pr_warning("error in %s: map handler doesn't belong to object\n",	4829	pr_warn("error in %s: map handler doesn't belong to object\n",
4726	__func__);	4830	__func__);
4727	return NULL;	4831	return NULL;
4728	}	4832	}
4729		4833
@@ -4801,8 +4905,6 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, · 71 unmodified lines
4801	{	4905	{
4802	struct bpf_object_open_attr open_attr = {};	4906	struct bpf_object_open_attr open_attr = {};
4803	struct bpf_program prog, first_prog = NULL;	4907	struct bpf_program prog, first_prog = NULL;
4804	enum bpf_attach_type expected_attach_type;
4805	enum bpf_prog_type prog_type;
4806	struct bpf_object *obj;	4908	struct bpf_object *obj;
4807	struct bpf_map *map;	4909	struct bpf_map *map;
4808	int err;	4910	int err;
@@ -4820,26 +4922,27 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, · 11 unmodified lines
4820	return -ENOENT;	4922	return -ENOENT;
4821		4923
4822	bpf_object__for_each_program(prog, obj) {	4924	bpf_object__for_each_program(prog, obj) {
		4925	enum bpf_attach_type attach_type = attr->expected_attach_type;
4823	/*	4926	/*
4824	* If type is not specified, try to guess it based on	4927	* to preserve backwards compatibility, bpf_prog_load treats
4825	* section name.	4928	* attr->prog_type, if specified, as an override to whatever
		4929	* bpf_object__open guessed
4826	*/	4930	*/
4827	prog_type = attr->prog_type;	4931	if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) {
4828	prog->prog_ifindex = attr->ifindex;	4932	bpf_program__set_type(prog, attr->prog_type);
4829	expected_attach_type = attr->expected_attach_type;	4933	bpf_program__set_expected_attach_type(prog,
4830	if (prog_type == BPF_PROG_TYPE_UNSPEC) {	4934	attach_type);
4831	err = bpf_program__identify_section(prog, &prog_type,	4935	}
4832	&expected_attach_type);	4936	if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) {
4833	if (err < 0) {	4937	/*
4834	bpf_object__close(obj);	4938	* we haven't guessed from section name and user
4835	return -EINVAL;	4939	* didn't provide a fallback type, too bad...
4836	}	4940	*/
		4941	bpf_object__close(obj);
		4942	return -EINVAL;
4837	}	4943	}
4838		4944
4839	bpf_program__set_type(prog, prog_type);	4945	prog->prog_ifindex = attr->ifindex;
4840	bpf_program__set_expected_attach_type(prog,
4841	expected_attach_type);
4842
4843	prog->log_level = attr->log_level;	4946	prog->log_level = attr->log_level;
4844	prog->prog_flags = attr->prog_flags;	4947	prog->prog_flags = attr->prog_flags;
4845	if (!first_prog)	4948	if (!first_prog)
@@ -4852,7 +4955,7 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, · 6 unmodified lines
4852	}	4955	}
4853		4956
4854	if (!first_prog) {	4957	if (!first_prog) {
4855	pr_warning("object file doesn't contain bpf program\n");	4958	pr_warn("object file doesn't contain bpf program\n");
4856	bpf_object__close(obj);	4959	bpf_object__close(obj);
4857	return -ENOENT;	4960	return -ENOENT;
4858	}	4961	}
@@ -4911,14 +5014,14 @@ struct bpf_link bpf_program__attach_perf_event(struct bpf_program prog, · 52 unmodified lines
4911	int prog_fd, err;	5014	int prog_fd, err;
4912		5015
4913	if (pfd < 0) {	5016	if (pfd < 0) {
4914	pr_warning("program '%s': invalid perf event FD %d\n",	5017	pr_warn("program '%s': invalid perf event FD %d\n",
4915	bpf_program__title(prog, false), pfd);	5018	bpf_program__title(prog, false), pfd);
4916	return ERR_PTR(-EINVAL);	5019	return ERR_PTR(-EINVAL);
4917	}	5020	}
4918	prog_fd = bpf_program__fd(prog);	5021	prog_fd = bpf_program__fd(prog);
4919	if (prog_fd < 0) {	5022	if (prog_fd < 0) {
4920	pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n",	5023	pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n",
4921	bpf_program__title(prog, false));	5024	bpf_program__title(prog, false));
4922	return ERR_PTR(-EINVAL);	5025	return ERR_PTR(-EINVAL);
4923	}	5026	}
4924		5027
@@ -4931,16 +5034,16 @@ struct bpf_link bpf_program__attach_perf_event(struct bpf_program prog, · 6 unmodified lines
4931	if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {	5034	if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) {
4932	err = -errno;	5035	err = -errno;
4933	free(link);	5036	free(link);
4934	pr_warning("program '%s': failed to attach to pfd %d: %s\n",	5037	pr_warn("program '%s': failed to attach to pfd %d: %s\n",
4935	bpf_program__title(prog, false), pfd,	5038	bpf_program__title(prog, false), pfd,
4936	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));	5039	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
4937	return ERR_PTR(err);	5040	return ERR_PTR(err);
4938	}	5041	}
4939	if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {	5042	if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
4940	err = -errno;	5043	err = -errno;
4941	free(link);	5044	free(link);
4942	pr_warning("program '%s': failed to enable pfd %d: %s\n",	5045	pr_warn("program '%s': failed to enable pfd %d: %s\n",
4943	bpf_program__title(prog, false), pfd,	5046	bpf_program__title(prog, false), pfd,
4944	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));	5047	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
4945	return ERR_PTR(err);	5048	return ERR_PTR(err);
4946	}	5049	}
@@ -5015,9 +5118,9 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 68 unmodified lines
5015	type = uprobe ? determine_uprobe_perf_type()	5118	type = uprobe ? determine_uprobe_perf_type()
5016	: determine_kprobe_perf_type();	5119	: determine_kprobe_perf_type();
5017	if (type < 0) {	5120	if (type < 0) {
5018	pr_warning("failed to determine %s perf type: %s\n",	5121	pr_warn("failed to determine %s perf type: %s\n",
5019	uprobe ? "uprobe" : "kprobe",	5122	uprobe ? "uprobe" : "kprobe",
5020	libbpf_strerror_r(type, errmsg, sizeof(errmsg)));	5123	libbpf_strerror_r(type, errmsg, sizeof(errmsg)));
5021	return type;	5124	return type;
5022	}	5125	}
5023	if (retprobe) {	5126	if (retprobe) {
@@ -5025,10 +5128,9 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 1 unmodified line
5025	: determine_kprobe_retprobe_bit();	5128	: determine_kprobe_retprobe_bit();
5026		5129
5027	if (bit < 0) {	5130	if (bit < 0) {
5028	pr_warning("failed to determine %s retprobe bit: %s\n",	5131	pr_warn("failed to determine %s retprobe bit: %s\n",
5029	uprobe ? "uprobe" : "kprobe",	5132	uprobe ? "uprobe" : "kprobe",
5030	libbpf_strerror_r(bit, errmsg,	5133	libbpf_strerror_r(bit, errmsg, sizeof(errmsg)));
5031	sizeof(errmsg)));
5032	return bit;	5134	return bit;
5033	}	5135	}
5034	attr.config \|= 1 << bit;	5136	attr.config \|= 1 << bit;
@@ -5045,9 +5147,9 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, · 10 unmodified lines
5045	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);	5147	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
5046	if (pfd < 0) {	5148	if (pfd < 0) {
5047	err = -errno;	5149	err = -errno;
5048	pr_warning("%s perf_event_open() failed: %s\n",	5150	pr_warn("%s perf_event_open() failed: %s\n",
5049	uprobe ? "uprobe" : "kprobe",	5151	uprobe ? "uprobe" : "kprobe",
5050	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));	5152	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5051	return err;	5153	return err;
5052	}	5154	}
5053	return pfd;	5155	return pfd;
@@ -5064,20 +5166,20 @@ struct bpf_link bpf_program__attach_kprobe(struct bpf_program prog, · 10 unmodified lines
5064	pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,	5166	pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name,
5065	0 /* offset /, -1 / pid */);	5167	0 /* offset /, -1 / pid */);
5066	if (pfd < 0) {	5168	if (pfd < 0) {
5067	pr_warning("program '%s': failed to create %s '%s' perf event: %s\n",	5169	pr_warn("program '%s': failed to create %s '%s' perf event: %s\n",
5068	bpf_program__title(prog, false),	5170	bpf_program__title(prog, false),
5069	retprobe ? "kretprobe" : "kprobe", func_name,	5171	retprobe ? "kretprobe" : "kprobe", func_name,
5070	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));	5172	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5071	return ERR_PTR(pfd);	5173	return ERR_PTR(pfd);
5072	}	5174	}
5073	link = bpf_program__attach_perf_event(prog, pfd);	5175	link = bpf_program__attach_perf_event(prog, pfd);
5074	if (IS_ERR(link)) {	5176	if (IS_ERR(link)) {
5075	close(pfd);	5177	close(pfd);
5076	err = PTR_ERR(link);	5178	err = PTR_ERR(link);
5077	pr_warning("program '%s': failed to attach to %s '%s': %s\n",	5179	pr_warn("program '%s': failed to attach to %s '%s': %s\n",
5078	bpf_program__title(prog, false),	5180	bpf_program__title(prog, false),
5079	retprobe ? "kretprobe" : "kprobe", func_name,	5181	retprobe ? "kretprobe" : "kprobe", func_name,
5080	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));	5182	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5081	return link;	5183	return link;
5082	}	5184	}
5083	return link;	5185	return link;
@@ -5095,22 +5197,22 @@ struct bpf_link bpf_program__attach_uprobe(struct bpf_program prog, · 11 unmodified lines
5095	pfd = perf_event_open_probe(true /* uprobe */, retprobe,	5197	pfd = perf_event_open_probe(true /* uprobe */, retprobe,
5096	binary_path, func_offset, pid);	5198	binary_path, func_offset, pid);
5097	if (pfd < 0) {	5199	if (pfd < 0) {
5098	pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n",	5200	pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
5099	bpf_program__title(prog, false),	5201	bpf_program__title(prog, false),
5100	retprobe ? "uretprobe" : "uprobe",	5202	retprobe ? "uretprobe" : "uprobe",
5101	binary_path, func_offset,	5203	binary_path, func_offset,
5102	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));	5204	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5103	return ERR_PTR(pfd);	5205	return ERR_PTR(pfd);
5104	}	5206	}
5105	link = bpf_program__attach_perf_event(prog, pfd);	5207	link = bpf_program__attach_perf_event(prog, pfd);
5106	if (IS_ERR(link)) {	5208	if (IS_ERR(link)) {
5107	close(pfd);	5209	close(pfd);
5108	err = PTR_ERR(link);	5210	err = PTR_ERR(link);
5109	pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n",	5211	pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n",
5110	bpf_program__title(prog, false),	5212	bpf_program__title(prog, false),
5111	retprobe ? "uretprobe" : "uprobe",	5213	retprobe ? "uretprobe" : "uprobe",
5112	binary_path, func_offset,	5214	binary_path, func_offset,
5113	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));	5215	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5114	return link;	5216	return link;
5115	}	5217	}
5116	return link;	5218	return link;
@@ -5144,9 +5246,9 @@ static int perf_event_open_tracepoint(const char *tp_category, · 27 unmodified lines
5144		5246
5145	tp_id = determine_tracepoint_id(tp_category, tp_name);	5247	tp_id = determine_tracepoint_id(tp_category, tp_name);
5146	if (tp_id < 0) {	5248	if (tp_id < 0) {
5147	pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n",	5249	pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n",
5148	tp_category, tp_name,	5250	tp_category, tp_name,
5149	libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));	5251	libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg)));
5150	return tp_id;	5252	return tp_id;
5151	}	5253	}
5152		5254
@@ -5158,9 +5260,9 @@ static int perf_event_open_tracepoint(const char *tp_category, · 5 unmodified lines
5158	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);	5260	-1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);
5159	if (pfd < 0) {	5261	if (pfd < 0) {
5160	err = -errno;	5262	err = -errno;
5161	pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n",	5263	pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n",
5162	tp_category, tp_name,	5264	tp_category, tp_name,
5163	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));	5265	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5164	return err;	5266	return err;
5165	}	5267	}
5166	return pfd;	5268	return pfd;
@@ -5176,20 +5278,20 @@ struct bpf_link bpf_program__attach_tracepoint(struct bpf_program prog, · 9 unmodified lines
5176		5278
5177	pfd = perf_event_open_tracepoint(tp_category, tp_name);	5279	pfd = perf_event_open_tracepoint(tp_category, tp_name);
5178	if (pfd < 0) {	5280	if (pfd < 0) {
5179	pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n",	5281	pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
5180	bpf_program__title(prog, false),	5282	bpf_program__title(prog, false),
5181	tp_category, tp_name,	5283	tp_category, tp_name,
5182	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));	5284	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5183	return ERR_PTR(pfd);	5285	return ERR_PTR(pfd);
5184	}	5286	}
5185	link = bpf_program__attach_perf_event(prog, pfd);	5287	link = bpf_program__attach_perf_event(prog, pfd);
5186	if (IS_ERR(link)) {	5288	if (IS_ERR(link)) {
5187	close(pfd);	5289	close(pfd);
5188	err = PTR_ERR(link);	5290	err = PTR_ERR(link);
5189	pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n",	5291	pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n",
5190	bpf_program__title(prog, false),	5292	bpf_program__title(prog, false),
5191	tp_category, tp_name,	5293	tp_category, tp_name,
5192	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));	5294	libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
5193	return link;	5295	return link;
5194	}	5296	}
5195	return link;	5297	return link;
@@ -5211,8 +5313,8 @@ struct bpf_link bpf_program__attach_raw_tracepoint(struct bpf_program prog, · 15 unmodified lines
5211		5313
5212	prog_fd = bpf_program__fd(prog);	5314	prog_fd = bpf_program__fd(prog);
5213	if (prog_fd < 0) {	5315	if (prog_fd < 0) {
5214	pr_warning("program '%s': can't attach before loaded\n",	5316	pr_warn("program '%s': can't attach before loaded\n",
5215	bpf_program__title(prog, false));	5317	bpf_program__title(prog, false));
5216	return ERR_PTR(-EINVAL);	5318	return ERR_PTR(-EINVAL);
5217	}	5319	}
5218		5320
@@ -5225,9 +5327,9 @@ struct bpf_link bpf_program__attach_raw_tracepoint(struct bpf_program prog, · 6 unmodified lines
5225	if (pfd < 0) {	5327	if (pfd < 0) {
5226	pfd = -errno;	5328	pfd = -errno;
5227	free(link);	5329	free(link);
5228	pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n",	5330	pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n",
5229	bpf_program__title(prog, false), tp_name,	5331	bpf_program__title(prog, false), tp_name,
5230	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));	5332	libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
5231	return ERR_PTR(pfd);	5333	return ERR_PTR(pfd);
5232	}	5334	}
5233	link->fd = pfd;	5335	link->fd = pfd;
@@ -5329,7 +5431,7 @@ static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, · 95 unmodified lines
5329	return;	5431	return;
5330	if (cpu_buf->base &&	5432	if (cpu_buf->base &&
5331	munmap(cpu_buf->base, pb->mmap_size + pb->page_size))	5433	munmap(cpu_buf->base, pb->mmap_size + pb->page_size))
5332	pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);	5434	pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);
5333	if (cpu_buf->fd >= 0) {	5435	if (cpu_buf->fd >= 0) {
5334	ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);	5436	ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);
5335	close(cpu_buf->fd);	5437	close(cpu_buf->fd);
@@ -5379,8 +5481,8 @@ perf_buffer__open_cpu_buf(struct perf_buffer pb, struct perf_event_attr attr, · 43 unmodified lines
5379	-1, PERF_FLAG_FD_CLOEXEC);	5481	-1, PERF_FLAG_FD_CLOEXEC);
5380	if (cpu_buf->fd < 0) {	5482	if (cpu_buf->fd < 0) {
5381	err = -errno;	5483	err = -errno;
5382	pr_warning("failed to open perf buffer event on cpu #%d: %s\n",	5484	pr_warn("failed to open perf buffer event on cpu #%d: %s\n",
5383	cpu, libbpf_strerror_r(err, msg, sizeof(msg)));	5485	cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
5384	goto error;	5486	goto error;
5385	}	5487	}
5386		5488
@@ -5390,15 +5492,15 @@ perf_buffer__open_cpu_buf(struct perf_buffer pb, struct perf_event_attr attr, · 3 unmodified lines
5390	if (cpu_buf->base == MAP_FAILED) {	5492	if (cpu_buf->base == MAP_FAILED) {
5391	cpu_buf->base = NULL;	5493	cpu_buf->base = NULL;
5392	err = -errno;	5494	err = -errno;
5393	pr_warning("failed to mmap perf buffer on cpu #%d: %s\n",	5495	pr_warn("failed to mmap perf buffer on cpu #%d: %s\n",
5394	cpu, libbpf_strerror_r(err, msg, sizeof(msg)));	5496	cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
5395	goto error;	5497	goto error;
5396	}	5498	}
5397		5499
5398	if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {	5500	if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
5399	err = -errno;	5501	err = -errno;
5400	pr_warning("failed to enable perf buffer event on cpu #%d: %s\n",	5502	pr_warn("failed to enable perf buffer event on cpu #%d: %s\n",
5401	cpu, libbpf_strerror_r(err, msg, sizeof(msg)));	5503	cpu, libbpf_strerror_r(err, msg, sizeof(msg)));
5402	goto error;	5504	goto error;
5403	}	5505	}
5404		5506
@@ -5458,8 +5560,8 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 53 unmodified lines
5458	int err, i;	5560	int err, i;
5459		5561
5460	if (page_cnt & (page_cnt - 1)) {	5562	if (page_cnt & (page_cnt - 1)) {
5461	pr_warning("page count should be power of two, but is %zu\n",	5563	pr_warn("page count should be power of two, but is %zu\n",
5462	page_cnt);	5564	page_cnt);
5463	return ERR_PTR(-EINVAL);	5565	return ERR_PTR(-EINVAL);
5464	}	5566	}
5465		5567
@@ -5467,14 +5569,14 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 1 unmodified line
5467	err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);	5569	err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len);
5468	if (err) {	5570	if (err) {
5469	err = -errno;	5571	err = -errno;
5470	pr_warning("failed to get map info for map FD %d: %s\n",	5572	pr_warn("failed to get map info for map FD %d: %s\n",
5471	map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));	5573	map_fd, libbpf_strerror_r(err, msg, sizeof(msg)));
5472	return ERR_PTR(err);	5574	return ERR_PTR(err);
5473	}	5575	}
5474		5576
5475	if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {	5577	if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) {
5476	pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",	5578	pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n",
5477	map.name);	5579	map.name);
5478	return ERR_PTR(-EINVAL);	5580	return ERR_PTR(-EINVAL);
5479	}	5581	}
5480		5582
@@ -5494,8 +5596,8 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 13 unmodified lines
5494	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);	5596	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
5495	if (pb->epoll_fd < 0) {	5597	if (pb->epoll_fd < 0) {
5496	err = -errno;	5598	err = -errno;
5497	pr_warning("failed to create epoll instance: %s\n",	5599	pr_warn("failed to create epoll instance: %s\n",
5498	libbpf_strerror_r(err, msg, sizeof(msg)));	5600	libbpf_strerror_r(err, msg, sizeof(msg)));
5499	goto error;	5601	goto error;
5500	}	5602	}
5501		5603
@@ -5514,13 +5616,13 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 12 unmodified lines
5514	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));	5616	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));
5515	if (!pb->events) {	5617	if (!pb->events) {
5516	err = -ENOMEM;	5618	err = -ENOMEM;
5517	pr_warning("failed to allocate events: out of memory\n");	5619	pr_warn("failed to allocate events: out of memory\n");
5518	goto error;	5620	goto error;
5519	}	5621	}
5520	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));	5622	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));
5521	if (!pb->cpu_bufs) {	5623	if (!pb->cpu_bufs) {
5522	err = -ENOMEM;	5624	err = -ENOMEM;
5523	pr_warning("failed to allocate buffers: out of memory\n");	5625	pr_warn("failed to allocate buffers: out of memory\n");
5524	goto error;	5626	goto error;
5525	}	5627	}
5526		5628
@@ -5543,9 +5645,9 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 16 unmodified lines
5543	&cpu_buf->fd, 0);	5645	&cpu_buf->fd, 0);
5544	if (err) {	5646	if (err) {
5545	err = -errno;	5647	err = -errno;
5546	pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n",	5648	pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",
5547	cpu, map_key, cpu_buf->fd,	5649	cpu, map_key, cpu_buf->fd,
5548	libbpf_strerror_r(err, msg, sizeof(msg)));	5650	libbpf_strerror_r(err, msg, sizeof(msg)));
5549	goto error;	5651	goto error;
5550	}	5652	}
5551		5653
@@ -5554,9 +5656,9 @@ static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, · 2 unmodified lines
5554	if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,	5656	if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,
5555	&pb->events[i]) < 0) {	5657	&pb->events[i]) < 0) {
5556	err = -errno;	5658	err = -errno;
5557	pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n",	5659	pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n",
5558	cpu, cpu_buf->fd,	5660	cpu, cpu_buf->fd,
5559	libbpf_strerror_r(err, msg, sizeof(msg)));	5661	libbpf_strerror_r(err, msg, sizeof(msg)));
5560	goto error;	5662	goto error;
5561	}	5663	}
5562	}	5664	}
@@ -5609,7 +5711,7 @@ perf_buffer__process_record(struct perf_event_header e, void ctx) · 46 unmodified lines
5609	break;	5711	break;
5610	}	5712	}
5611	default:	5713	default:
5612	pr_warning("unknown perf sample type %d\n", e->type);	5714	pr_warn("unknown perf sample type %d\n", e->type);
5613	return LIBBPF_PERF_EVENT_ERROR;	5715	return LIBBPF_PERF_EVENT_ERROR;
5614	}	5716	}
5615	return LIBBPF_PERF_EVENT_CONT;	5717	return LIBBPF_PERF_EVENT_CONT;
@@ -5639,7 +5741,7 @@ int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) · 23 unmodified lines
5639		5741
5640	err = perf_buffer__process_records(pb, cpu_buf);	5742	err = perf_buffer__process_records(pb, cpu_buf);
5641	if (err) {	5743	if (err) {
5642	pr_warning("error while processing records: %d\n", err);	5744	pr_warn("error while processing records: %d\n", err);
5643	return err;	5745	return err;
5644	}	5746	}
5645	}	5747	}
@@ -5836,13 +5938,13 @@ bpf_program__get_prog_info_linear(int fd, __u64 arrays) · 190 unmodified lines
5836	v2 = bpf_prog_info_read_offset_u32(&info_linear->info,	5938	v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
5837	desc->count_offset);	5939	desc->count_offset);
5838	if (v1 != v2)	5940	if (v1 != v2)
5839	pr_warning("%s: mismatch in element count\n", __func__);	5941	pr_warn("%s: mismatch in element count\n", __func__);
5840		5942
5841	v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);	5943	v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
5842	v2 = bpf_prog_info_read_offset_u32(&info_linear->info,	5944	v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
5843	desc->size_offset);	5945	desc->size_offset);
5844	if (v1 != v2)	5946	if (v1 != v2)
5845	pr_warning("%s: mismatch in rec size\n", __func__);	5947	pr_warn("%s: mismatch in rec size\n", __func__);
5846	}	5948	}
5847		5949
5848	/* step 7: update info_len and data_len */	5950	/* step 7: update info_len and data_len */
@@ -5910,20 +6012,19 @@ int libbpf_num_possible_cpus(void) · 61 unmodified lines
5910	fd = open(fcpu, O_RDONLY);	6012	fd = open(fcpu, O_RDONLY);
5911	if (fd < 0) {	6013	if (fd < 0) {
5912	error = errno;	6014	error = errno;
5913	pr_warning("Failed to open file %s: %s\n",	6015	pr_warn("Failed to open file %s: %s\n", fcpu, strerror(error));
5914	fcpu, strerror(error));
5915	return -error;	6016	return -error;
5916	}	6017	}
5917	len = read(fd, buf, sizeof(buf));	6018	len = read(fd, buf, sizeof(buf));
5918	close(fd);	6019	close(fd);
5919	if (len <= 0) {	6020	if (len <= 0) {
5920	error = len ? errno : EINVAL;	6021	error = len ? errno : EINVAL;
5921	pr_warning("Failed to read # of possible cpus from %s: %s\n",	6022	pr_warn("Failed to read # of possible cpus from %s: %s\n",
5922	fcpu, strerror(error));	6023	fcpu, strerror(error));
5923	return -error;	6024	return -error;
5924	}	6025	}
5925	if (len == sizeof(buf)) {	6026	if (len == sizeof(buf)) {
5926	pr_warning("File %s size overflow\n", fcpu);	6027	pr_warn("File %s size overflow\n", fcpu);
5927	return -EOVERFLOW;	6028	return -EOVERFLOW;
5928	}	6029	}
5929	buf[len] = '\0';	6030	buf[len] = '\0';
@@ -5934,8 +6035,8 @@ int libbpf_num_possible_cpus(void) · 4 unmodified lines
5934	buf[ir] = '\0';	6035	buf[ir] = '\0';
5935	n = sscanf(&buf[il], "%u-%u", &start, &end);	6036	n = sscanf(&buf[il], "%u-%u", &start, &end);
5936	if (n <= 0) {	6037	if (n <= 0) {
5937	pr_warning("Failed to get # CPUs from %s\n",	6038	pr_warn("Failed to get # CPUs from %s\n",
5938	&buf[il]);	6039	&buf[il]);
5939	return -EINVAL;	6040	return -EINVAL;
5940	} else if (n == 1) {	6041	} else if (n == 1) {
5941	end = start;	6042	end = start;
@@ -5945,7 +6046,7 @@ int libbpf_num_possible_cpus(void) · 3 unmodified lines
5945	}	6046	}
5946	}	6047	}
5947	if (tmp_cpus <= 0) {	6048	if (tmp_cpus <= 0) {
5948	pr_warning("Invalid #CPUs %d from %s\n", tmp_cpus, fcpu);	6049	pr_warn("Invalid #CPUs %d from %s\n", tmp_cpus, fcpu);
5949	return -EINVAL;	6050	return -EINVAL;
5950	}	6051	}
5951		6052

tools/lib/bpf/libbpf.h+20−8

  * have all the padding bytes initialized to zero. It's not guaranteed though,
  * when copying literal, that compiler won't copy garbage in literal's padding
  * bytes, but that's the best way I've found and it seems to work in practice.
+ *
+ * Macro declares opts struct of given type and name, zero-initializes,
+ * including any extra padding, it with memset() and then assigns initial
+ * values provided by users in struct initializer-syntax as varargs.
  */
-#define LIBBPF_OPTS(TYPE, NAME, ...)					    \
+#define DECLARE_LIBBPF_OPTS(TYPE, NAME, ...)				    \
-	struct TYPE NAME;						    \
+	struct TYPE NAME = ({ 						    \
-	memset(&NAME, 0, sizeof(struct TYPE));				    \
+		memset(&NAME, 0, sizeof(struct TYPE));			    \
-	NAME = (struct TYPE) {						    \
+		(struct TYPE) {						    \
-		.sz = sizeof(struct TYPE),				    \
+			.sz = sizeof(struct TYPE),			    \
-		__VA_ARGS__						    \
+			__VA_ARGS__					    \
-	}
+		};							    \
+	})
 struct bpf_object_open_opts {
 	/* size of this struct, for forward/backward compatiblity */
 	const char *object_name;
 	/* parse map definitions non-strictly, allowing extra attributes/data */
 	bool relaxed_maps;
+	/* process CO-RE relocations non-strictly, allowing them to fail */
+	bool relaxed_core_relocs;
 };
-#define bpf_object_open_opts__last_field relaxed_maps
+#define bpf_object_open_opts__last_field relaxed_core_relocs
 LIBBPF_API struct bpf_object *bpf_object__open(const char *path);
 LIBBPF_API struct bpf_object *
 LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog);
 LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog);
 LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog);
+LIBBPF_API enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog);
 LIBBPF_API void bpf_program__set_type(struct bpf_program *prog,
 				      enum bpf_prog_type type);
+LIBBPF_API enum bpf_attach_type
+bpf_program__get_expected_attach_type(struct bpf_program *prog);
 LIBBPF_API void
 bpf_program__set_expected_attach_type(struct bpf_program *prog,
 				      enum bpf_attach_type type);

tools/lib/bpf/libbpf.map+2−0

@@ -195,4 +195,6 @@ LIBBPF_0.0.6 { · 194 unmodified lines
195	global:	195	global:
196	bpf_object__open_file;	196	bpf_object__open_file;
197	bpf_object__open_mem;	197	bpf_object__open_mem;
		198	bpf_program__get_expected_attach_type;
		199	bpf_program__get_type;
198	} LIBBPF_0.0.5;	200	} LIBBPF_0.0.5;

tools/lib/bpf/libbpf_internal.h+22−11

 	libbpf_print(level, "libbpf: " fmt, ##__VA_ARGS__);	\
 } while (0)
-#define pr_warning(fmt, ...)	__pr(LIBBPF_WARN, fmt, ##__VA_ARGS__)
+#define pr_warn(fmt, ...)	__pr(LIBBPF_WARN, fmt, ##__VA_ARGS__)
 #define pr_info(fmt, ...)	__pr(LIBBPF_INFO, fmt, ##__VA_ARGS__)
 #define pr_debug(fmt, ...)	__pr(LIBBPF_DEBUG, fmt, ##__VA_ARGS__)
 					const char *type_name)
 {
 	if (user_sz < sizeof(size_t)) {
-		pr_warning("%s size (%zu) is too small\n", type_name, user_sz);
+		pr_warn("%s size (%zu) is too small\n", type_name, user_sz);
 		return false;
 	}
 	if (user_sz > opts_sz) {
 		for (i = opts_sz; i < user_sz; i++) {
 			if (opts[i]) {
-				pr_warning("%s has non-zero extra bytes",
+				pr_warn("%s has non-zero extra bytes",
-					   type_name);
+					type_name);
 				return false;
 			}
 		}
 	};
 	struct btf_ext_info func_info;
 	struct btf_ext_info line_info;
-	struct btf_ext_info offset_reloc_info;
+	struct btf_ext_info field_reloc_info;
 	__u32 data_size;
 };
 	__u32	line_col;
 };
-/* The minimum bpf_offset_reloc checked by the loader
+/* bpf_field_info_kind encodes which aspect of captured field has to be
+ * adjusted by relocations. Currently supported values are:
+ *   - BPF_FIELD_BYTE_OFFSET: field offset (in bytes);
+ *   - BPF_FIELD_EXISTS: field existence (1, if field exists; 0, otherwise);
+ */
+enum bpf_field_info_kind {
+	BPF_FIELD_BYTE_OFFSET = 0,	/* field byte offset */
+	BPF_FIELD_EXISTS = 2,		/* field existence in target kernel */
+};
+/* The minimum bpf_field_reloc checked by the loader
  *
- * Offset relocation captures the following data:
+ * Field relocation captures the following data:
  * - insn_off - instruction offset (in bytes) within a BPF program that needs
- *   its insn->imm field to be relocated with actual offset;
+ *   its insn->imm field to be relocated with actual field info;
  * - type_id - BTF type ID of the "root" (containing) entity of a relocatable
- *   offset;
+ *   field;
  * - access_str_off - offset into corresponding .BTF string section. String
  *   itself encodes an accessed field using a sequence of field and array
  *   indicies, separated by colon (:). It's conceptually very close to LLVM's
  * bpf_probe_read(&dst, sizeof(dst),
  *		  __builtin_preserve_access_index(&src->a.b.c));
  *
- * In this case Clang will emit offset relocation recording necessary data to
+ * In this case Clang will emit field relocation recording necessary data to
  * be able to find offset of embedded `a.b.c` field within `src` struct.
  *
  *   [0] https://llvm.org/docs/LangRef.html#getelementptr-instruction
  */
-struct bpf_offset_reloc {
+struct bpf_field_reloc {
 	__u32   insn_off;
 	__u32   type_id;
 	__u32   access_str_off;
+	enum bpf_field_info_kind kind;
 };
 #endif /* __LIBBPF_LIBBPF_INTERNAL_H */

tools/lib/bpf/xsk.c+34−12

 	/* This is the C-program:
 	 * SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
 	 * {
-	 *     int index = ctx->rx_queue_index;
+	 *     int ret, index = ctx->rx_queue_index;
 	 *
 	 *     // A set entry here means that the correspnding queue_id
 	 *     // has an active AF_XDP socket bound to it.
+	 *     ret = bpf_redirect_map(&xsks_map, index, XDP_PASS);
+	 *     if (ret > 0)
+	 *         return ret;
+	 *
+	 *     // Fallback for pre-5.3 kernels, not supporting default
+	 *     // action in the flags parameter.
 	 *     if (bpf_map_lookup_elem(&xsks_map, &index))
 	 *         return bpf_redirect_map(&xsks_map, index, 0);
-	 *
 	 *     return XDP_PASS;
 	 * }
 	 */
 	struct bpf_insn prog[] = {
-		/* r1 = *(u32 *)(r1 + 16) */
+		/* r2 = *(u32 *)(r1 + 16) */
-		BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, 16),
+		BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 16),
-		/* *(u32 *)(r10 - 4) = r1 */
+		/* *(u32 *)(r10 - 4) = r2 */
-		BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_1, -4),
+		BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -4),
+		/* r1 = xskmap[] */
+		BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
+		/* r3 = XDP_PASS */
+		BPF_MOV64_IMM(BPF_REG_3, 2),
+		/* call bpf_redirect_map */
+		BPF_EMIT_CALL(BPF_FUNC_redirect_map),
+		/* if w0 != 0 goto pc+13 */
+		BPF_JMP32_IMM(BPF_JSGT, BPF_REG_0, 0, 13),
+		/* r2 = r10 */
 		BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
+		/* r2 += -4 */
 		BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
+		/* r1 = xskmap[] */
 		BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
+		/* call bpf_map_lookup_elem */
 		BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
+		/* r1 = r0 */
 		BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
-		BPF_MOV32_IMM(BPF_REG_0, 2),
+		/* r0 = XDP_PASS */
-		/* if r1 == 0 goto +5 */
+		BPF_MOV64_IMM(BPF_REG_0, 2),
+		/* if r1 == 0 goto pc+5 */
 		BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
 		/* r2 = *(u32 *)(r10 - 4) */
-		BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
 		BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_10, -4),
-		BPF_MOV32_IMM(BPF_REG_3, 0),
+		/* r1 = xskmap[] */
+		BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
+		/* r3 = 0 */
+		BPF_MOV64_IMM(BPF_REG_3, 0),
+		/* call bpf_redirect_map */
 		BPF_EMIT_CALL(BPF_FUNC_redirect_map),
 		/* The jumps are to this instruction */
 		BPF_EXIT_INSN(),
 				   "LGPL-2.1 or BSD-2-Clause", 0, log_buf,
 				   log_buf_size);
 	if (prog_fd < 0) {
-		pr_warning("BPF log buffer:\n%s", log_buf);
+		pr_warn("BPF log buffer:\n%s", log_buf);
 		return prog_fd;
 	}
 		return -EFAULT;
 	if (umem->refcount) {
-		pr_warning("Error: shared umems not supported by libbpf.\n");
+		pr_warn("Error: shared umems not supported by libbpf.\n");
 		return -EBUSY;
 	}

tools/testing/selftests/bpf/.gitignore+3−3

 fixdep
 test_align
 test_dev_cgroup
-test_progs
+/test_progs*
 test_tcpbpf_user
 test_verifier_log
 feature
-test_libbpf_open
 test_sock
 test_sock_addr
 test_sock_fields
 test_libbpf
 test_tcp_check_syncookie_user
 test_sysctl
-alu32
 libbpf.pc
 libbpf.so.*
 test_hashmap
 test_btf_dump
 xdping
+/no_alu32
+/bpf_gcc

tools/testing/selftests/bpf/Makefile+190−161

@@ -2,10 +2,12 @@ · 1 unmodified line
2	include ../../../../scripts/Kbuild.include	2	include ../../../../scripts/Kbuild.include
3	include ../../../scripts/Makefile.arch	3	include ../../../scripts/Makefile.arch
4		4
5	LIBDIR := ../../../lib	5	CURDIR := $(abspath .)
		6	LIBDIR := $(abspath ../../../lib)
6	BPFDIR := $(LIBDIR)/bpf	7	BPFDIR := $(LIBDIR)/bpf
7	APIDIR := ../../../include/uapi	8	TOOLSDIR := $(abspath ../../../include)
8	GENDIR := ../../../../include/generated	9	APIDIR := $(TOOLSDIR)/uapi
		10	GENDIR := $(abspath ../../../../include/generated)
9	GENHDR := $(GENDIR)/autoconf.h	11	GENHDR := $(GENDIR)/autoconf.h
10		12
11	ifneq ($(wildcard $(GENHDR)),)	13	ifneq ($(wildcard $(GENHDR)),)
@@ -16,8 +18,9 @@ CLANG ?= clang · 4 unmodified lines
16	LLC ?= llc	18	LLC ?= llc
17	LLVM_OBJCOPY ?= llvm-objcopy	19	LLVM_OBJCOPY ?= llvm-objcopy
18	BPF_GCC ?= $(shell command -v bpf-gcc;)	20	BPF_GCC ?= $(shell command -v bpf-gcc;)
19	CFLAGS += -g -Wall -O2 -I$(APIDIR) -I$(LIBDIR) -I$(BPFDIR) -I$(GENDIR) $(GENFLAGS) -I../../../include \	21	CFLAGS += -g -Wall -O2 $(GENFLAGS) -I$(APIDIR) -I$(LIBDIR) -I$(BPFDIR) \
20	-Dbpf_prog_load=bpf_prog_test_load \	22	-I$(GENDIR) -I$(TOOLSDIR) -I$(CURDIR) \
		23	-Dbpf_prog_load=bpf_prog_test_load \
21	-Dbpf_load_program=bpf_test_load_program	24	-Dbpf_load_program=bpf_test_load_program
22	LDLIBS += -lcap -lelf -lrt -lpthread	25	LDLIBS += -lcap -lelf -lrt -lpthread
23		26
@@ -25,33 +28,20 @@ LDLIBS += -lcap -lelf -lrt -lpthread · 1 unmodified line
25	TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \	28	TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
26	test_align test_verifier_log test_dev_cgroup test_tcpbpf_user \	29	test_align test_verifier_log test_dev_cgroup test_tcpbpf_user \
27	test_sock test_btf test_sockmap get_cgroup_id_user test_socket_cookie \	30	test_sock test_btf test_sockmap get_cgroup_id_user test_socket_cookie \
28	test_cgroup_storage test_select_reuseport test_section_names \	31	test_cgroup_storage test_select_reuseport \
29	test_netcnt test_tcpnotify_user test_sock_fields test_sysctl test_hashmap \	32	test_netcnt test_tcpnotify_user test_sock_fields test_sysctl test_hashmap \
30	test_cgroup_attach xdping	33	test_cgroup_attach xdping test_progs-no_alu32
31
32	BPF_OBJ_FILES = $(patsubst %.c,%.o, $(notdir $(wildcard progs/*.c)))
33	TEST_GEN_FILES = $(BPF_OBJ_FILES)
34
35	BTF_C_FILES = $(wildcard progs/btf_dump_test_case_*.c)
36	TEST_FILES = $(BTF_C_FILES)
37
38	# Also test sub-register code-gen if LLVM has eBPF v3 processor support which
39	# contains both ALU32 and JMP32 instructions.
40	SUBREG_CODEGEN := $(shell echo "int cal(int a) { return a > 0; }" \| \
41	$(CLANG) -target bpf -O2 -emit-llvm -S -x c - -o - \| \
42	$(LLC) -mattr=+alu32 -mcpu=v3 2>&1 \| \
43	grep 'if w')
44	ifneq ($(SUBREG_CODEGEN),)
45	TEST_GEN_FILES += $(patsubst %.o,alu32/%.o, $(BPF_OBJ_FILES))
46	endif
47		34
		35	# Also test bpf-gcc, if present
48	ifneq ($(BPF_GCC),)	36	ifneq ($(BPF_GCC),)
49	TEST_GEN_FILES += $(patsubst %.o,bpf_gcc/%.o, $(BPF_OBJ_FILES))	37	TEST_GEN_PROGS += test_progs-bpf_gcc
50	endif	38	endif
51		39
		40	TEST_GEN_FILES =
		41	TEST_FILES =
		42
52	# Order correspond to 'make run_tests' order	43	# Order correspond to 'make run_tests' order
53	TEST_PROGS := test_kmod.sh \	44	TEST_PROGS := test_kmod.sh \
54	test_libbpf.sh \
55	test_xdp_redirect.sh \	45	test_xdp_redirect.sh \
56	test_xdp_meta.sh \	46	test_xdp_meta.sh \
57	test_xdp_veth.sh \	47	test_xdp_veth.sh \
@@ -78,27 +68,30 @@ TEST_PROGS_EXTENDED := with_addr.sh \ · 20 unmodified lines
78	test_xdp_vlan.sh	68	test_xdp_vlan.sh
79		69
80	# Compile but not part of 'make run_tests'	70	# Compile but not part of 'make run_tests'
81	TEST_GEN_PROGS_EXTENDED = test_libbpf_open test_sock_addr test_skb_cgroup_id_user \	71	TEST_GEN_PROGS_EXTENDED = test_sock_addr test_skb_cgroup_id_user \
82	flow_dissector_load test_flow_dissector test_tcp_check_syncookie_user \	72	flow_dissector_load test_flow_dissector test_tcp_check_syncookie_user \
83	test_lirc_mode2_user	73	test_lirc_mode2_user
84		74
85	include ../lib.mk	75	TEST_CUSTOM_PROGS = urandom_read
86		76
87	# NOTE: $(OUTPUT) won't get default value if used before lib.mk	77	include ../lib.mk
88	TEST_CUSTOM_PROGS = $(OUTPUT)/urandom_read
89	all: $(TEST_CUSTOM_PROGS)
90		78
91	$(OUTPUT)/urandom_read: $(OUTPUT)/%: %.c	79	# Define simple and short `make test_progs`, `make test_sysctl`, etc targets
		80	# to build individual tests.
		81	# NOTE: Semicolon at the end is critical to override lib.mk's default static
		82	# rule for binaries.
		83	$(notdir $(TEST_GEN_PROGS) \
		84	$(TEST_PROGS) \
		85	$(TEST_PROGS_EXTENDED) \
		86	$(TEST_GEN_PROGS_EXTENDED) \
		87	$(TEST_CUSTOM_PROGS)): %: $(OUTPUT)/% ;
		88
		89	$(OUTPUT)/urandom_read: urandom_read.c
92	$(CC) -o $@ $< -Wl,--build-id	90	$(CC) -o $@ $< -Wl,--build-id
93		91
94	$(OUTPUT)/test_stub.o: test_stub.c
95	$(CC) $(TEST_PROGS_CFLAGS) $(CFLAGS) -c -o $@ $<
96
97	BPFOBJ := $(OUTPUT)/libbpf.a	92	BPFOBJ := $(OUTPUT)/libbpf.a
98		93
99	$(TEST_GEN_PROGS): $(OUTPUT)/test_stub.o $(BPFOBJ)	94	$(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(OUTPUT)/test_stub.o $(BPFOBJ)
100
101	$(TEST_GEN_PROGS_EXTENDED): $(OUTPUT)/test_stub.o $(OUTPUT)/libbpf.a
102		95
103	$(OUTPUT)/test_dev_cgroup: cgroup_helpers.c	96	$(OUTPUT)/test_dev_cgroup: cgroup_helpers.c
104	$(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c	97	$(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c
@@ -108,7 +101,6 @@ $(OUTPUT)/test_socket_cookie: cgroup_helpers.c · 3 unmodified lines
108	$(OUTPUT)/test_sockmap: cgroup_helpers.c	101	$(OUTPUT)/test_sockmap: cgroup_helpers.c
109	$(OUTPUT)/test_tcpbpf_user: cgroup_helpers.c	102	$(OUTPUT)/test_tcpbpf_user: cgroup_helpers.c
110	$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c	103	$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c
111	$(OUTPUT)/test_progs: cgroup_helpers.c trace_helpers.c
112	$(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c	104	$(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c
113	$(OUTPUT)/test_cgroup_storage: cgroup_helpers.c	105	$(OUTPUT)/test_cgroup_storage: cgroup_helpers.c
114	$(OUTPUT)/test_netcnt: cgroup_helpers.c	106	$(OUTPUT)/test_netcnt: cgroup_helpers.c
@@ -124,6 +116,10 @@ force: · 9 unmodified lines
124	$(BPFOBJ): force	116	$(BPFOBJ): force
125	$(MAKE) -C $(BPFDIR) OUTPUT=$(OUTPUT)/	117	$(MAKE) -C $(BPFDIR) OUTPUT=$(OUTPUT)/
126		118
		119	BPF_HELPERS := $(BPFDIR)/bpf_helper_defs.h $(wildcard $(BPFDIR)/bpf_*.h)
		120	$(BPFDIR)/bpf_helper_defs.h:
		121	$(MAKE) -C $(BPFDIR) OUTPUT=$(OUTPUT)/ bpf_helper_defs.h
		122
127	# Get Clang's default includes on this system, as opposed to those seen by	123	# Get Clang's default includes on this system, as opposed to those seen by
128	# '-target bpf'. This fixes "missing" files on some architectures/distros,	124	# '-target bpf'. This fixes "missing" files on some architectures/distros,
129	# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.	125	# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.
@@ -134,10 +130,11 @@ define get_sys_includes · 4 unmodified lines
134	$(shell $(1) -v -E - </dev/null 2>&1 \	130	$(shell $(1) -v -E - </dev/null 2>&1 \
135	\| sed -n '/<...> search starts here:/,/End of search list./{ s\| $/.*$\|-idirafter \1\|p }')	131	\| sed -n '/<...> search starts here:/,/End of search list./{ s\| $/.*$\|-idirafter \1\|p }')
136	endef	132	endef
		133
137	CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG))	134	CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG))
138	BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \	135	BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \
139	-I. -I./include/uapi -I../../../include/uapi \	136	-I. -I./include/uapi -I$(APIDIR) \
140	-I$(BPFDIR) -I$(OUTPUT)/../usr/include	137	-I$(BPFDIR) -I$(abspath $(OUTPUT)/../usr/include)
141		138
142	CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \	139	CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \
143	-Wno-compare-distinct-pointer-types	140	-Wno-compare-distinct-pointer-types
@@ -145,137 +142,169 @@ CLANG_CFLAGS = $(CLANG_SYS_INCLUDES) \ · 1 unmodified line
145	$(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline	142	$(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline
146	$(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline	143	$(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline
147		144
148	$(OUTPUT)/test_queue_map.o: test_queue_stack_map.h
149	$(OUTPUT)/test_stack_map.o: test_queue_stack_map.h
150
151	$(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h	145	$(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h
152	$(OUTPUT)/test_progs.o: flow_dissector_load.h	146
153		147	# Build BPF object using Clang
154	TEST_PROGS_CFLAGS := -I. -I$(OUTPUT)	148	# $1 - input .c file
155	TEST_MAPS_CFLAGS := -I. -I$(OUTPUT)	149	# $2 - output .o file
156	TEST_VERIFIER_CFLAGS := -I. -I$(OUTPUT) -Iverifier	150	# $3 - CFLAGS
157		151	# $4 - LDFLAGS
158	ifneq ($(SUBREG_CODEGEN),)	152	define CLANG_BPF_BUILD_RULE
159	ALU32_BUILD_DIR = $(OUTPUT)/alu32	153	($(CLANG) $3 -O2 -target bpf -emit-llvm \
160	TEST_CUSTOM_PROGS += $(ALU32_BUILD_DIR)/test_progs_32	154	-c $1 -o - \|\| echo "BPF obj compilation failed") \| \
161	$(ALU32_BUILD_DIR):	155	$(LLC) -march=bpf -mcpu=probe $4 -filetype=obj -o $2
162	mkdir -p $@	156	endef
163		157	# Similar to CLANG_BPF_BUILD_RULE, but using native Clang and bpf LLC
164	$(ALU32_BUILD_DIR)/urandom_read: $(OUTPUT)/urandom_read \| $(ALU32_BUILD_DIR)	158	define CLANG_NATIVE_BPF_BUILD_RULE
165	cp $< $@	159	($(CLANG) $3 -O2 -emit-llvm \
166		160	-c $1 -o - \|\| echo "BPF obj compilation failed") \| \
167	$(ALU32_BUILD_DIR)/test_progs_32: test_progs.c $(OUTPUT)/libbpf.a\	161	$(LLC) -march=bpf -mcpu=probe $4 -filetype=obj -o $2
168	$(ALU32_BUILD_DIR)/urandom_read \	162	endef
169	\| $(ALU32_BUILD_DIR)	163	# Build BPF object using GCC
170	$(CC) $(TEST_PROGS_CFLAGS) $(CFLAGS) \	164	define GCC_BPF_BUILD_RULE
171	-o $(ALU32_BUILD_DIR)/test_progs_32 \	165	$(BPF_GCC) $3 $4 -O2 -c $1 -o $2
172	test_progs.c test_stub.c cgroup_helpers.c trace_helpers.c prog_tests/*.c \	166	endef
173	$(OUTPUT)/libbpf.a $(LDLIBS)	167
174		168	# Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on
175	$(ALU32_BUILD_DIR)/test_progs_32: $(PROG_TESTS_H)	169	# $eval()) and pass control to DEFINE_TEST_RUNNER_RULES.
176	$(ALU32_BUILD_DIR)/test_progs_32: prog_tests/*.c	170	# Parameters:
177		171	# $1 - test runner base binary name (e.g., test_progs)
178	$(ALU32_BUILD_DIR)/%.o: progs/%.c $(ALU32_BUILD_DIR)/test_progs_32 \	172	# $2 - test runner extra "flavor" (e.g., no_alu32, gcc-bpf, etc)
179	\| $(ALU32_BUILD_DIR)	173	define DEFINE_TEST_RUNNER
180	($(CLANG) $(BPF_CFLAGS) $(CLANG_CFLAGS) -O2 -target bpf -emit-llvm \	174
181	-c $< -o - \|\| echo "clang failed") \| \	175	TRUNNER_OUTPUT := $(OUTPUT)$(if $2,/)$2
182	$(LLC) -march=bpf -mcpu=probe -mattr=+alu32 $(LLC_FLAGS) \	176	TRUNNER_BINARY := $1$(if $2,-)$2
183	-filetype=obj -o $@	177	TRUNNER_TEST_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.test.o, \
		178	$$(notdir $$(wildcard $(TRUNNER_TESTS_DIR)/*.c)))
		179	TRUNNER_EXTRA_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \
		180	$$(filter %.c,$(TRUNNER_EXTRA_SOURCES)))
		181	TRUNNER_EXTRA_HDRS := $$(filter %.h,$(TRUNNER_EXTRA_SOURCES))
		182	TRUNNER_TESTS_HDR := $(TRUNNER_TESTS_DIR)/tests.h
		183	TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, \
		184	$$(notdir $$(wildcard $(TRUNNER_BPF_PROGS_DIR)/*.c)))
		185
		186	# Evaluate rules now with extra TRUNNER_XXX variables above already defined
		187	$$(eval $$(call DEFINE_TEST_RUNNER_RULES,$1,$2))
		188
		189	endef
		190
		191	# Using TRUNNER_XXX variables, provided by callers of DEFINE_TEST_RUNNER and
		192	# set up by DEFINE_TEST_RUNNER itself, create test runner build rules with:
		193	# $1 - test runner base binary name (e.g., test_progs)
		194	# $2 - test runner extra "flavor" (e.g., no_alu32, gcc-bpf, etc)
		195	define DEFINE_TEST_RUNNER_RULES
		196
		197	ifeq ($($(TRUNNER_OUTPUT)-dir),)
		198	$(TRUNNER_OUTPUT)-dir := y
		199	$(TRUNNER_OUTPUT):
		200	mkdir -p $$@
		201	endif
		202
		203	# ensure we set up BPF objects generation rule just once for a given
		204	# input/output directory combination
		205	ifeq ($($(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs),)
		206	$(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs := y
		207	$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.o: \
		208	$(TRUNNER_BPF_PROGS_DIR)/%.c \
		209	$(TRUNNER_BPF_PROGS_DIR)/*.h \
		210	$$(BPF_HELPERS) \| $(TRUNNER_OUTPUT)
		211	$$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@, \
		212	$(TRUNNER_BPF_CFLAGS), \
		213	$(TRUNNER_BPF_LDFLAGS))
		214	endif
		215
		216	# ensure we set up tests.h header generation rule just once
		217	ifeq ($($(TRUNNER_TESTS_DIR)-tests-hdr),)
		218	$(TRUNNER_TESTS_DIR)-tests-hdr := y
		219	$(TRUNNER_TESTS_HDR): $(TRUNNER_TESTS_DIR)/*.c
		220	$$(shell ( cd $(TRUNNER_TESTS_DIR); \
		221	echo '/* Generated header, do not edit */'; \
		222	ls *.c 2> /dev/null \| \
		223	sed -e 's@$[^\.]*$\.c@DEFINE_TEST(\1)@'; \
		224	) > $$@)
		225	endif
		226
		227	# compile individual test files
		228	# Note: we cd into output directory to ensure embedded BPF object is found
		229	$(TRUNNER_TEST_OBJS): $(TRUNNER_OUTPUT)/%.test.o: \
		230	$(TRUNNER_TESTS_DIR)/%.c \
		231	$(TRUNNER_EXTRA_HDRS) \
		232	$(TRUNNER_BPF_OBJS) \
		233	$$(BPFOBJ) \| $(TRUNNER_OUTPUT)
		234	cd $$(@D) && $$(CC) $$(CFLAGS) -c $(CURDIR)/$$< $$(LDLIBS) -o $$(@F)
		235
		236	$(TRUNNER_EXTRA_OBJS): $(TRUNNER_OUTPUT)/%.o: \
		237	%.c \
		238	$(TRUNNER_EXTRA_HDRS) \
		239	$(TRUNNER_TESTS_HDR) \
		240	$$(BPFOBJ) \| $(TRUNNER_OUTPUT)
		241	$$(CC) $$(CFLAGS) -c $$< $$(LDLIBS) -o $$@
		242
		243	$(TRUNNER_BINARY)-extras: $(TRUNNER_EXTRA_FILES) \| $(TRUNNER_OUTPUT)
		244	ifneq ($2,)
		245	# only copy extra resources if in flavored build
		246	cp -a $$^ $(TRUNNER_OUTPUT)/
184	endif	247	endif
185		248
		249	$(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS) \
		250	$(TRUNNER_EXTRA_OBJS) $$(BPFOBJ) \
		251	\| $(TRUNNER_BINARY)-extras
		252	$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) -o $$@
		253
		254	endef
		255
		256	# Define test_progs test runner.
		257	TRUNNER_TESTS_DIR := prog_tests
		258	TRUNNER_BPF_PROGS_DIR := progs
		259	TRUNNER_EXTRA_SOURCES := test_progs.c cgroup_helpers.c trace_helpers.c \
		260	flow_dissector_load.h
		261	TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \
		262	$(wildcard progs/btf_dump_test_case_*.c)
		263	TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE
		264	TRUNNER_BPF_CFLAGS := -I. -I$(OUTPUT) $(BPF_CFLAGS) $(CLANG_CFLAGS)
		265	TRUNNER_BPF_LDFLAGS := -mattr=+alu32
		266	$(eval $(call DEFINE_TEST_RUNNER,test_progs))
		267
		268	# Define test_progs-no_alu32 test runner.
		269	TRUNNER_BPF_LDFLAGS :=
		270	$(eval $(call DEFINE_TEST_RUNNER,test_progs,no_alu32))
		271
		272	# Define test_progs BPF-GCC-flavored test runner.
186	ifneq ($(BPF_GCC),)	273	ifneq ($(BPF_GCC),)
187	GCC_SYS_INCLUDES = $(call get_sys_includes,gcc)
188	IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null \| \	274	IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null \| \
189	grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')	275	grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
190	ifeq ($(IS_LITTLE_ENDIAN),)	276	MENDIAN=$(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian)
191	MENDIAN=-mbig-endian	277
192	else	278	TRUNNER_BPF_BUILD_RULE := GCC_BPF_BUILD_RULE
193	MENDIAN=-mlittle-endian	279	TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(call get_sys_includes,gcc) $(MENDIAN)
194	endif	280	TRUNNER_BPF_LDFLAGS :=
195	BPF_GCC_CFLAGS = $(GCC_SYS_INCLUDES) $(MENDIAN)	281	$(eval $(call DEFINE_TEST_RUNNER,test_progs,bpf_gcc))
196	BPF_GCC_BUILD_DIR = $(OUTPUT)/bpf_gcc
197	TEST_CUSTOM_PROGS += $(BPF_GCC_BUILD_DIR)/test_progs_bpf_gcc
198	$(BPF_GCC_BUILD_DIR):
199	mkdir -p $@
200
201	$(BPF_GCC_BUILD_DIR)/urandom_read: $(OUTPUT)/urandom_read \| $(BPF_GCC_BUILD_DIR)
202	cp $< $@
203
204	$(BPF_GCC_BUILD_DIR)/test_progs_bpf_gcc: $(OUTPUT)/test_progs \
205	\| $(BPF_GCC_BUILD_DIR)
206	cp $< $@
207
208	$(BPF_GCC_BUILD_DIR)/%.o: progs/%.c $(BPF_GCC_BUILD_DIR)/test_progs_bpf_gcc \
209	\| $(BPF_GCC_BUILD_DIR)
210	$(BPF_GCC) $(BPF_CFLAGS) $(BPF_GCC_CFLAGS) -O2 -c $< -o $@
211	endif	282	endif
212		283
213	# Have one program compiled without "-target bpf" to test whether libbpf loads	284	# Define test_maps test runner.
214	# it successfully	285	TRUNNER_TESTS_DIR := map_tests
215	$(OUTPUT)/test_xdp.o: progs/test_xdp.c	286	TRUNNER_BPF_PROGS_DIR := progs
216	($(CLANG) $(BPF_CFLAGS) $(CLANG_CFLAGS) -O2 -emit-llvm -c $< -o - \|\| \	287	TRUNNER_EXTRA_SOURCES := test_maps.c
217	echo "clang failed") \| \	288	TRUNNER_EXTRA_FILES :=
218	$(LLC) -march=bpf -mcpu=probe $(LLC_FLAGS) -filetype=obj -o $@	289	TRUNNER_BPF_BUILD_RULE := $$(error no BPF objects should be built)
219		290	TRUNNER_BPF_CFLAGS :=
220	# libbpf has to be built before BPF programs due to bpf_helper_defs.h	291	TRUNNER_BPF_LDFLAGS :=
221	$(OUTPUT)/%.o: progs/%.c \| $(BPFOBJ)	292	$(eval $(call DEFINE_TEST_RUNNER,test_maps))
222	($(CLANG) $(BPF_CFLAGS) $(CLANG_CFLAGS) -O2 -target bpf -emit-llvm \	293
223	-c $< -o - \|\| echo "clang failed") \| \	294	# Define test_verifier test runner.
224	$(LLC) -march=bpf -mcpu=probe $(LLC_FLAGS) -filetype=obj -o $@	295	# It is much simpler than test_maps/test_progs and sufficiently different from
225		296	# them (e.g., test.h is using completely pattern), that it's worth just
226	PROG_TESTS_DIR = $(OUTPUT)/prog_tests	297	# explicitly defining all the rules explicitly.
227	$(PROG_TESTS_DIR):	298	verifier/tests.h: verifier/*.c
228	mkdir -p $@
229	PROG_TESTS_H := $(PROG_TESTS_DIR)/tests.h
230	PROG_TESTS_FILES := $(wildcard prog_tests/*.c)
231	test_progs.c: $(PROG_TESTS_H)
232	$(OUTPUT)/test_progs: CFLAGS += $(TEST_PROGS_CFLAGS)
233	$(OUTPUT)/test_progs: test_progs.c $(PROG_TESTS_FILES) \| $(OUTPUT)/test_attach_probe.o $(PROG_TESTS_H)
234	$(PROG_TESTS_H): $(PROG_TESTS_FILES) \| $(PROG_TESTS_DIR)
235	$(shell ( cd prog_tests/; \
236	echo '/* Generated header, do not edit */'; \
237	ls *.c 2> /dev/null \| \
238	sed -e 's@$[^\.]*$\.c@DEFINE_TEST(\1)@'; \
239	) > $(PROG_TESTS_H))
240
241	MAP_TESTS_DIR = $(OUTPUT)/map_tests
242	$(MAP_TESTS_DIR):
243	mkdir -p $@
244	MAP_TESTS_H := $(MAP_TESTS_DIR)/tests.h
245	MAP_TESTS_FILES := $(wildcard map_tests/*.c)
246	test_maps.c: $(MAP_TESTS_H)
247	$(OUTPUT)/test_maps: CFLAGS += $(TEST_MAPS_CFLAGS)
248	$(OUTPUT)/test_maps: test_maps.c $(MAP_TESTS_FILES) \| $(MAP_TESTS_H)
249	$(MAP_TESTS_H): $(MAP_TESTS_FILES) \| $(MAP_TESTS_DIR)
250	$(shell ( cd map_tests/; \
251	echo '/* Generated header, do not edit */'; \
252	echo '#ifdef DECLARE'; \
253	ls *.c 2> /dev/null \| \
254	sed -e 's@$[^\.]*$\.c@extern void test_\1(void);@'; \
255	echo '#endif'; \
256	echo '#ifdef CALL'; \
257	ls *.c 2> /dev/null \| \
258	sed -e 's@$[^\.]*$\.c@test_\1();@'; \
259	echo '#endif' \
260	) > $(MAP_TESTS_H))
261
262	VERIFIER_TESTS_DIR = $(OUTPUT)/verifier
263	$(VERIFIER_TESTS_DIR):
264	mkdir -p $@
265	VERIFIER_TESTS_H := $(VERIFIER_TESTS_DIR)/tests.h
266	VERIFIER_TEST_FILES := $(wildcard verifier/*.c)
267	test_verifier.c: $(VERIFIER_TESTS_H)
268	$(OUTPUT)/test_verifier: CFLAGS += $(TEST_VERIFIER_CFLAGS)
269	$(OUTPUT)/test_verifier: test_verifier.c \| $(VERIFIER_TEST_FILES) $(VERIFIER_TESTS_H)
270	$(VERIFIER_TESTS_H): $(VERIFIER_TEST_FILES) \| $(VERIFIER_TESTS_DIR)
271	$(shell ( cd verifier/; \	299	$(shell ( cd verifier/; \
272	echo '/* Generated header, do not edit */'; \	300	echo '/* Generated header, do not edit */'; \
273	echo '#ifdef FILL_ARRAY'; \	301	echo '#ifdef FILL_ARRAY'; \
274	ls *.c 2> /dev/null \| \	302	ls .c 2> /dev/null \| sed -e 's@$.$@#include \"\1\"@'; \
275	sed -e 's@$.*$@#include \"\1\"@'; \
276	echo '#endif' \	303	echo '#endif' \
277	) > $(VERIFIER_TESTS_H))	304	) > verifier/tests.h)
		305	$(OUTPUT)/test_verifier: test_verifier.c verifier/tests.h $(BPFOBJ) \| $(OUTPUT)
		306	$(CC) $(CFLAGS) $(filter %.a %.o %.c,$^) $(LDLIBS) -o $@
278		307
279	EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(ALU32_BUILD_DIR) $(BPF_GCC_BUILD_DIR) \	308	EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) \
280	$(VERIFIER_TESTS_H) $(PROG_TESTS_H) $(MAP_TESTS_H) \	309	prog_tests/tests.h map_tests/tests.h verifier/tests.h \
281	feature	310	feature $(OUTPUT)/*.o $(OUTPUT)/no_alu32 $(OUTPUT)/bpf_gcc

tools/testing/selftests/bpf/prog_tests/attach_probe.c+1−6

 	const int kprobe_idx = 0, kretprobe_idx = 1;
 	const int uprobe_idx = 2, uretprobe_idx = 3;
 	const char *obj_name = "attach_probe";
-	LIBBPF_OPTS(bpf_object_open_opts, open_opts,
+	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
 		.object_name = obj_name,
 		.relaxed_maps = true,
 	);
 		  "prog '%s' not found\n", uretprobe_name))
 		goto cleanup;
-	bpf_program__set_kprobe(kprobe_prog);
-	bpf_program__set_kprobe(kretprobe_prog);
-	bpf_program__set_kprobe(uprobe_prog);
-	bpf_program__set_kprobe(uretprobe_prog);
 	/* create maps && load programs */
 	err = bpf_object__load(obj);
 	if (CHECK(err, "obj_load", "err %d\n", err))

tools/testing/selftests/bpf/prog_tests/core_reloc.c+76−5

@@ -174,6 +174,21 @@ · 173 unmodified lines
174	.fails = true, \	174	.fails = true, \
175	}	175	}
176		176
		177	#define EXISTENCE_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \
		178	.a = 42, \
		179	}
		180
		181	#define EXISTENCE_CASE_COMMON(name) \
		182	.case_name = #name, \
		183	.bpf_obj_file = "test_core_reloc_existence.o", \
		184	.btf_src_file = "btf__core_reloc_" #name ".o", \
		185	.relaxed_core_relocs = true \
		186
		187	#define EXISTENCE_ERR_CASE(name) { \
		188	EXISTENCE_CASE_COMMON(name), \
		189	.fails = true, \
		190	}
		191
177	struct core_reloc_test_case {	192	struct core_reloc_test_case {
178	const char *case_name;	193	const char *case_name;
179	const char *bpf_obj_file;	194	const char *bpf_obj_file;
@@ -183,6 +198,7 @@ struct core_reloc_test_case { · 3 unmodified lines
183	const char *output;	198	const char *output;
184	int output_len;	199	int output_len;
185	bool fails;	200	bool fails;
		201	bool relaxed_core_relocs;
186	};	202	};
187		203
188	static struct core_reloc_test_case test_cases[] = {	204	static struct core_reloc_test_case test_cases[] = {
@@ -195,8 +211,8 @@ static struct core_reloc_test_case test_cases[] = { · 6 unmodified lines
195	.input_len = 0,	211	.input_len = 0,
196	.output = STRUCT_TO_CHAR_PTR(core_reloc_kernel_output) {	212	.output = STRUCT_TO_CHAR_PTR(core_reloc_kernel_output) {
197	.valid = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, },	213	.valid = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, },
198	.comm = "test_progs\0\0\0\0\0",	214	.comm = "test_progs",
199	.comm_len = 11,	215	.comm_len = sizeof("test_progs"),
200	},	216	},
201	.output_len = sizeof(struct core_reloc_kernel_output),	217	.output_len = sizeof(struct core_reloc_kernel_output),
202	},	218	},
@@ -283,6 +299,59 @@ static struct core_reloc_test_case test_cases[] = { · 80 unmodified lines
283	},	299	},
284	.output_len = sizeof(struct core_reloc_misc_output),	300	.output_len = sizeof(struct core_reloc_misc_output),
285	},	301	},
		302
		303	/* validate field existence checks */
		304	{
		305	EXISTENCE_CASE_COMMON(existence),
		306	.input = STRUCT_TO_CHAR_PTR(core_reloc_existence) {
		307	.a = 1,
		308	.b = 2,
		309	.c = 3,
		310	.arr = { 4 },
		311	.s = { .x = 5 },
		312	},
		313	.input_len = sizeof(struct core_reloc_existence),
		314	.output = STRUCT_TO_CHAR_PTR(core_reloc_existence_output) {
		315	.a_exists = 1,
		316	.b_exists = 1,
		317	.c_exists = 1,
		318	.arr_exists = 1,
		319	.s_exists = 1,
		320	.a_value = 1,
		321	.b_value = 2,
		322	.c_value = 3,
		323	.arr_value = 4,
		324	.s_value = 5,
		325	},
		326	.output_len = sizeof(struct core_reloc_existence_output),
		327	},
		328	{
		329	EXISTENCE_CASE_COMMON(existence___minimal),
		330	.input = STRUCT_TO_CHAR_PTR(core_reloc_existence___minimal) {
		331	.a = 42,
		332	},
		333	.input_len = sizeof(struct core_reloc_existence),
		334	.output = STRUCT_TO_CHAR_PTR(core_reloc_existence_output) {
		335	.a_exists = 1,
		336	.b_exists = 0,
		337	.c_exists = 0,
		338	.arr_exists = 0,
		339	.s_exists = 0,
		340	.a_value = 42,
		341	.b_value = 0xff000002u,
		342	.c_value = 0xff000003u,
		343	.arr_value = 0xff000004u,
		344	.s_value = 0xff000005u,
		345	},
		346	.output_len = sizeof(struct core_reloc_existence_output),
		347	},
		348
		349	EXISTENCE_ERR_CASE(existence__err_int_sz),
		350	EXISTENCE_ERR_CASE(existence__err_int_type),
		351	EXISTENCE_ERR_CASE(existence__err_int_kind),
		352	EXISTENCE_ERR_CASE(existence__err_arr_kind),
		353	EXISTENCE_ERR_CASE(existence__err_arr_value_type),
		354	EXISTENCE_ERR_CASE(existence__err_struct_type),
286	};	355	};
287		356
288	struct data {	357	struct data {
@@ -305,11 +374,14 @@ void test_core_reloc(void) · 16 unmodified lines
305		374
306	for (i = 0; i < ARRAY_SIZE(test_cases); i++) {	375	for (i = 0; i < ARRAY_SIZE(test_cases); i++) {
307	test_case = &test_cases[i];	376	test_case = &test_cases[i];
308
309	if (!test__start_subtest(test_case->case_name))	377	if (!test__start_subtest(test_case->case_name))
310	continue;	378	continue;
311		379
312	obj = bpf_object__open(test_case->bpf_obj_file);	380	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
		381	.relaxed_core_relocs = test_case->relaxed_core_relocs,
		382	);
		383
		384	obj = bpf_object__open_file(test_case->bpf_obj_file, &opts);
313	if (CHECK(IS_ERR_OR_NULL(obj), "obj_open",	385	if (CHECK(IS_ERR_OR_NULL(obj), "obj_open",
314	"failed to open '%s': %ld\n",	386	"failed to open '%s': %ld\n",
315	test_case->bpf_obj_file, PTR_ERR(obj)))	387	test_case->bpf_obj_file, PTR_ERR(obj)))
@@ -319,7 +391,6 @@ void test_core_reloc(void) · 3 unmodified lines
319	if (CHECK(!prog, "find_probe",	391	if (CHECK(!prog, "find_probe",
320	"prog '%s' not found\n", probe_name))	392	"prog '%s' not found\n", probe_name))
321	goto cleanup;	393	goto cleanup;
322	bpf_program__set_type(prog, BPF_PROG_TYPE_RAW_TRACEPOINT);
323		394
324	load_attr.obj = obj;	395	load_attr.obj = obj;
325	load_attr.log_level = 0;	396	load_attr.log_level = 0;

tools/testing/selftests/bpf/prog_tests/flow_dissector_reattach.c+17−4

 void test_flow_dissector_reattach(void)
 {
-	int init_net, err;
+	int init_net, self_net, err;
+	self_net = open("/proc/self/ns/net", O_RDONLY);
+	if (CHECK_FAIL(self_net < 0)) {
+		perror("open(/proc/self/ns/net");
+		return;
+	}
 	init_net = open("/proc/1/ns/net", O_RDONLY);
 	if (CHECK_FAIL(init_net < 0)) {
 		perror("open(/proc/1/ns/net)");
-		return;
+		goto out_close;
 	}
 	err = setns(init_net, CLONE_NEWNET);
 	if (is_attached(init_net)) {
 		test__skip();
 		printf("Can't test with flow dissector attached to init_net\n");
-		return;
+		goto out_setns;
 	}
 	/* First run tests in root network namespace */
 	err = unshare(CLONE_NEWNET);
 	if (CHECK_FAIL(err)) {
 		perror("unshare(CLONE_NEWNET)");
-		goto out_close;
+		goto out_setns;
 	}
 	do_flow_dissector_reattach();
+out_setns:
+	/* Move back to netns we started in. */
+	err = setns(self_net, CLONE_NEWNET);
+	if (CHECK_FAIL(err))
+		perror("setns(/proc/self/ns/net)");
 out_close:
 	close(init_net);
+	close(self_net);
 }

tools/testing/selftests/bpf/prog_tests/kfree_skb.c+89−0

@@ -0,0 +1,89 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <test_progs.h>
	3
	4	static void on_sample(void ctx, int cpu, void data, __u32 size)
	5	{
	6	int ifindex = (int )data, duration = 0;
	7	struct ipv6_packet *pkt_v6 = data + 4;
	8
	9	if (ifindex != 1)
	10	/* spurious kfree_skb not on loopback device */
	11	return;
	12	if (CHECK(size != 76, "check_size", "size %u != 76\n", size))
	13	return;
	14	if (CHECK(pkt_v6->eth.h_proto != 0xdd86, "check_eth",
	15	"h_proto %x\n", pkt_v6->eth.h_proto))
	16	return;
	17	if (CHECK(pkt_v6->iph.nexthdr != 6, "check_ip",
	18	"iph.nexthdr %x\n", pkt_v6->iph.nexthdr))
	19	return;
	20	if (CHECK(pkt_v6->tcp.doff != 5, "check_tcp",
	21	"tcp.doff %x\n", pkt_v6->tcp.doff))
	22	return;
	23
	24	(bool )ctx = true;
	25	}
	26
	27	void test_kfree_skb(void)
	28	{
	29	struct bpf_prog_load_attr attr = {
	30	.file = "./kfree_skb.o",
	31	};
	32
	33	struct bpf_object obj, obj2 = NULL;
	34	struct perf_buffer_opts pb_opts = {};
	35	struct perf_buffer *pb = NULL;
	36	struct bpf_link *link = NULL;
	37	struct bpf_map *perf_buf_map;
	38	struct bpf_program *prog;
	39	__u32 duration, retval;
	40	int err, pkt_fd, kfree_skb_fd;
	41	bool passed = false;
	42
	43	err = bpf_prog_load("./test_pkt_access.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &pkt_fd);
	44	if (CHECK(err, "prog_load sched cls", "err %d errno %d\n", err, errno))
	45	return;
	46
	47	err = bpf_prog_load_xattr(&attr, &obj2, &kfree_skb_fd);
	48	if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno))
	49	goto close_prog;
	50
	51	prog = bpf_object__find_program_by_title(obj2, "tp_btf/kfree_skb");
	52	if (CHECK(!prog, "find_prog", "prog kfree_skb not found\n"))
	53	goto close_prog;
	54	link = bpf_program__attach_raw_tracepoint(prog, NULL);
	55	if (CHECK(IS_ERR(link), "attach_raw_tp", "err %ld\n", PTR_ERR(link)))
	56	goto close_prog;
	57
	58	perf_buf_map = bpf_object__find_map_by_name(obj2, "perf_buf_map");
	59	if (CHECK(!perf_buf_map, "find_perf_buf_map", "not found\n"))
	60	goto close_prog;
	61
	62	/* set up perf buffer */
	63	pb_opts.sample_cb = on_sample;
	64	pb_opts.ctx = &passed;
	65	pb = perf_buffer__new(bpf_map__fd(perf_buf_map), 1, &pb_opts);
	66	if (CHECK(IS_ERR(pb), "perf_buf__new", "err %ld\n", PTR_ERR(pb)))
	67	goto close_prog;
	68
	69	err = bpf_prog_test_run(pkt_fd, 1, &pkt_v6, sizeof(pkt_v6),
	70	NULL, NULL, &retval, &duration);
	71	CHECK(err \|\| retval, "ipv6",
	72	"err %d errno %d retval %d duration %d\n",
	73	err, errno, retval, duration);
	74
	75	/* read perf buffer */
	76	err = perf_buffer__poll(pb, 100);
	77	if (CHECK(err < 0, "perf_buffer__poll", "err %d\n", err))
	78	goto close_prog;
	79	/* make sure kfree_skb program was triggered
	80	* and it sent expected skb into ring buffer
	81	*/
	82	CHECK_FAIL(!passed);
	83	close_prog:
	84	perf_buffer__free(pb);
	85	if (!IS_ERR_OR_NULL(link))
	86	bpf_link__destroy(link);
	87	bpf_object__close(obj);
	88	bpf_object__close(obj2);
	89	}

tools/testing/selftests/bpf/prog_tests/rdonly_maps.c+0−4

 	if (CHECK(IS_ERR(obj), "obj_open", "err %ld\n", PTR_ERR(obj)))
 		return;
-	bpf_object__for_each_program(prog, obj) {
-		bpf_program__set_raw_tracepoint(prog);
-	}
 	err = bpf_object__load(obj);
 	if (CHECK(err, "obj_load", "err %d errno %d\n", err, errno))
 		goto cleanup;

tools/testing/selftests/bpf/prog_tests/reference_tracking.c+3−2

 {
 	const char *file = "test_sk_lookup_kern.o";
 	const char *obj_name = "ref_track";
-	LIBBPF_OPTS(bpf_object_open_opts, open_opts,
+	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
 		.object_name = obj_name,
 		.relaxed_maps = true,
 	);
 		if (strstr(title, ".text") != NULL)
 			continue;
-		bpf_program__set_type(prog, BPF_PROG_TYPE_SCHED_CLS);
+		if (!test__start_subtest(title))
+			continue;
 		/* Expect verifier failure if test name has 'fail' */
 		if (strstr(title, "fail") != NULL) {

tools/testing/selftests/bpf/prog_tests/section_names.crenamed from tools/testing/selftests/bpf/test_section_names.c+30−60

 // SPDX-License-Identifier: GPL-2.0
 // Copyright (c) 2018 Facebook
+#include <test_progs.h>
-#include <err.h>
+static int duration = 0;
-#include <bpf/libbpf.h>
-#include "bpf_util.h"
 struct sec_name_test {
 	const char sec_name[32];
 };
 static struct sec_name_test tests[] = {
-	{"InvAliD", {-EINVAL, 0, 0}, {-EINVAL, 0} },
+	{"InvAliD", {-ESRCH, 0, 0}, {-EINVAL, 0} },
-	{"cgroup", {-EINVAL, 0, 0}, {-EINVAL, 0} },
+	{"cgroup", {-ESRCH, 0, 0}, {-EINVAL, 0} },
 	{"socket", {0, BPF_PROG_TYPE_SOCKET_FILTER, 0}, {-EINVAL, 0} },
 	{"kprobe/", {0, BPF_PROG_TYPE_KPROBE, 0}, {-EINVAL, 0} },
+	{"uprobe/", {0, BPF_PROG_TYPE_KPROBE, 0}, {-EINVAL, 0} },
 	{"kretprobe/", {0, BPF_PROG_TYPE_KPROBE, 0}, {-EINVAL, 0} },
+	{"uretprobe/", {0, BPF_PROG_TYPE_KPROBE, 0}, {-EINVAL, 0} },
 	{"classifier", {0, BPF_PROG_TYPE_SCHED_CLS, 0}, {-EINVAL, 0} },
 	{"action", {0, BPF_PROG_TYPE_SCHED_ACT, 0}, {-EINVAL, 0} },
 	{"tracepoint/", {0, BPF_PROG_TYPE_TRACEPOINT, 0}, {-EINVAL, 0} },
+	{"tp/", {0, BPF_PROG_TYPE_TRACEPOINT, 0}, {-EINVAL, 0} },
 	{
 		"raw_tracepoint/",
 		{0, BPF_PROG_TYPE_RAW_TRACEPOINT, 0},
 		{-EINVAL, 0},
 	},
+	{"raw_tp/", {0, BPF_PROG_TYPE_RAW_TRACEPOINT, 0}, {-EINVAL, 0} },
 	{"xdp", {0, BPF_PROG_TYPE_XDP, 0}, {-EINVAL, 0} },
 	{"perf_event", {0, BPF_PROG_TYPE_PERF_EVENT, 0}, {-EINVAL, 0} },
 	{"lwt_in", {0, BPF_PROG_TYPE_LWT_IN, 0}, {-EINVAL, 0} },
 	},
 };
-static int test_prog_type_by_name(const struct sec_name_test *test)
+static void test_prog_type_by_name(const struct sec_name_test *test)
 {
 	enum bpf_attach_type expected_attach_type;
 	enum bpf_prog_type prog_type;
 	rc = libbpf_prog_type_by_name(test->sec_name, &prog_type,
 				      &expected_attach_type);
-	if (rc != test->expected_load.rc) {
+	CHECK(rc != test->expected_load.rc, "check_code",
-		warnx("prog: unexpected rc=%d for %s", rc, test->sec_name);
+	      "prog: unexpected rc=%d for %s", rc, test->sec_name);
-		return -1;
-	}
 	if (rc)
-		return 0;
+		return;
-	if (prog_type != test->expected_load.prog_type) {
-		warnx("prog: unexpected prog_type=%d for %s", prog_type,
-		      test->sec_name);
-		return -1;
-	}
-	if (expected_attach_type != test->expected_load.expected_attach_type) {
+	CHECK(prog_type != test->expected_load.prog_type, "check_prog_type",
-		warnx("prog: unexpected expected_attach_type=%d for %s",
+	      "prog: unexpected prog_type=%d for %s",
-		      expected_attach_type, test->sec_name);
+	      prog_type, test->sec_name);
-		return -1;
-	}
-	return 0;
+	CHECK(expected_attach_type != test->expected_load.expected_attach_type,
+	      "check_attach_type", "prog: unexpected expected_attach_type=%d for %s",
+	      expected_attach_type, test->sec_name);
 }
-static int test_attach_type_by_name(const struct sec_name_test *test)
+static void test_attach_type_by_name(const struct sec_name_test *test)
 {
 	enum bpf_attach_type attach_type;
 	int rc;
 	rc = libbpf_attach_type_by_name(test->sec_name, &attach_type);
-	if (rc != test->expected_attach.rc) {
+	CHECK(rc != test->expected_attach.rc, "check_ret",
-		warnx("attach: unexpected rc=%d for %s", rc, test->sec_name);
+	      "attach: unexpected rc=%d for %s", rc, test->sec_name);
-		return -1;
-	}
 	if (rc)
-		return 0;
+		return;
-	if (attach_type != test->expected_attach.attach_type) {
-		warnx("attach: unexpected attach_type=%d for %s", attach_type,
-		      test->sec_name);
-		return -1;
-	}
-	return 0;
+	CHECK(attach_type != test->expected_attach.attach_type,
+	      "check_attach_type", "attach: unexpected attach_type=%d for %s",
+	      attach_type, test->sec_name);
 }
-static int run_test_case(const struct sec_name_test *test)
+void test_section_names(void)
 {
-	if (test_prog_type_by_name(test))
-		return -1;
-	if (test_attach_type_by_name(test))
-		return -1;
-	return 0;
-}
-static int run_tests(void)
-{
-	int passes = 0;
-	int fails = 0;
 	int i;
 	for (i = 0; i < ARRAY_SIZE(tests); ++i) {
-		if (run_test_case(&tests[i]))
+		struct sec_name_test *test = &tests[i];
-			++fails;
-		else
-			++passes;
-	}
-	printf("Summary: %d PASSED, %d FAILED\n", passes, fails);
-	return fails ? -1 : 0;
-}
-int main(int argc, char **argv)
+		test_prog_type_by_name(test);
-{
+		test_attach_type_by_name(test);
-	return run_tests();
+	}
 }

tools/testing/selftests/bpf/prog_tests/skb_ctx.c+5−0

 		.cb[3] = 4,
 		.cb[4] = 5,
 		.priority = 6,
+		.tstamp = 7,
 	};
 	struct bpf_prog_test_run_attr tattr = {
 		.data_in = &pkt_v4,
 		   "ctx_out_priority",
 		   "skb->priority == %d, expected %d\n",
 		   skb.priority, 7);
+	CHECK_ATTR(skb.tstamp != 8,
+		   "ctx_out_tstamp",
+		   "skb->tstamp == %lld, expected %d\n",
+		   skb.tstamp, 8);
 }

tools/testing/selftests/bpf/progs/btf__core_reloc_existence.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_kind.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence___err_wrong_arr_kind x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_arr_value_type.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence___err_wrong_arr_value_type x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_kind.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence___err_wrong_int_kind x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_sz.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence___err_wrong_int_sz x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_int_type.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence___err_wrong_int_type x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_existence___err_wrong_struct_type.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence___err_wrong_struct_type x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_existence___minimal.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_existence___minimal x) {}

tools/testing/selftests/bpf/progs/core_reloc_types.h+57−1

@@ -6,7 +6,7 @@ · 5 unmodified lines
6		6
7	struct core_reloc_kernel_output {	7	struct core_reloc_kernel_output {
8	int valid[10];	8	int valid[10];
9	char comm[16];	9	char comm[sizeof("test_progs")];
10	int comm_len;	10	int comm_len;
11	};	11	};
12		12
@@ -674,3 +674,59 @@ struct core_reloc_misc_extensible { · 661 unmodified lines
674	int c;	674	int c;
675	int d;	675	int d;
676	};	676	};
		677
		678	/*
		679	* EXISTENCE
		680	*/
		681	struct core_reloc_existence_output {
		682	int a_exists;
		683	int a_value;
		684	int b_exists;
		685	int b_value;
		686	int c_exists;
		687	int c_value;
		688	int arr_exists;
		689	int arr_value;
		690	int s_exists;
		691	int s_value;
		692	};
		693
		694	struct core_reloc_existence {
		695	int a;
		696	struct {
		697	int b;
		698	};
		699	int c;
		700	int arr[1];
		701	struct {
		702	int x;
		703	} s;
		704	};
		705
		706	struct core_reloc_existence___minimal {
		707	int a;
		708	};
		709
		710	struct core_reloc_existence___err_wrong_int_sz {
		711	short a;
		712	};
		713
		714	struct core_reloc_existence___err_wrong_int_type {
		715	int b[1];
		716	};
		717
		718	struct core_reloc_existence___err_wrong_int_kind {
		719	struct{ int x; } c;
		720	};
		721
		722	struct core_reloc_existence___err_wrong_arr_kind {
		723	int arr;
		724	};
		725
		726	struct core_reloc_existence___err_wrong_arr_value_type {
		727	short arr[1];
		728	};
		729
		730	struct core_reloc_existence___err_wrong_struct_type {
		731	int s;
		732	};

tools/testing/selftests/bpf/progs/kfree_skb.c+103−0

@@ -0,0 +1,103 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	// Copyright (c) 2019 Facebook
	3	#include <linux/bpf.h>
	4	#include "bpf_helpers.h"
	5	#include "bpf_endian.h"
	6
	7	char _license[] SEC("license") = "GPL";
	8	struct {
	9	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
	10	__uint(key_size, sizeof(int));
	11	__uint(value_size, sizeof(int));
	12	} perf_buf_map SEC(".maps");
	13
	14	#define _(P) (__builtin_preserve_access_index(P))
	15
	16	/* define few struct-s that bpf program needs to access */
	17	struct callback_head {
	18	struct callback_head *next;
	19	void (func)(struct callback_head head);
	20	};
	21	struct dev_ifalias {
	22	struct callback_head rcuhead;
	23	};
	24
	25	struct net_device /* same as kernel's struct net_device */ {
	26	int ifindex;
	27	struct dev_ifalias *ifalias;
	28	};
	29
	30	typedef struct {
	31	int counter;
	32	} atomic_t;
	33	typedef struct refcount_struct {
	34	atomic_t refs;
	35	} refcount_t;
	36
	37	struct sk_buff {
	38	/* field names and sizes should match to those in the kernel */
	39	unsigned int len, data_len;
	40	__u16 mac_len, hdr_len, queue_mapping;
	41	struct net_device *dev;
	42	/* order of the fields doesn't matter */
	43	refcount_t users;
	44	unsigned char *data;
	45	char __pkt_type_offset[0];
	46	};
	47
	48	/* copy arguments from
	49	* include/trace/events/skb.h:
	50	* TRACE_EVENT(kfree_skb,
	51	* TP_PROTO(struct sk_buff skb, void location),
	52	*
	53	* into struct below:
	54	*/
	55	struct trace_kfree_skb {
	56	struct sk_buff *skb;
	57	void *location;
	58	};
	59
	60	SEC("tp_btf/kfree_skb")
	61	int trace_kfree_skb(struct trace_kfree_skb *ctx)
	62	{
	63	struct sk_buff *skb = ctx->skb;
	64	struct net_device *dev;
	65	int ifindex;
	66	struct callback_head *ptr;
	67	void *func;
	68	int users;
	69	unsigned char *data;
	70	unsigned short pkt_data;
	71	char pkt_type;
	72
	73	__builtin_preserve_access_index(({
	74	users = skb->users.refs.counter;
	75	data = skb->data;
	76	dev = skb->dev;
	77	ifindex = dev->ifindex;
	78	ptr = dev->ifalias->rcuhead.next;
	79	func = ptr->func;
	80	}));
	81
	82	bpf_probe_read(&pkt_type, sizeof(pkt_type), _(&skb->__pkt_type_offset));
	83	pkt_type &= 7;
	84
	85	/* read eth proto */
	86	bpf_probe_read(&pkt_data, sizeof(pkt_data), data + 12);
	87
	88	bpf_printk("rcuhead.next %llx func %llx\n", ptr, func);
	89	bpf_printk("skb->len %d users %d pkt_type %x\n",
	90	_(skb->len), users, pkt_type);
	91	bpf_printk("skb->queue_mapping %d\n", _(skb->queue_mapping));
	92	bpf_printk("dev->ifindex %d data %llx pkt_data %x\n",
	93	ifindex, data, pkt_data);
	94
	95	if (users != 1 \|\| pkt_data != bpf_htons(0x86dd) \|\| ifindex != 1)
	96	/* raw tp ignores return value */
	97	return 0;
	98
	99	/* send first 72 byte of the packet to user space */
	100	bpf_skb_output(skb, &perf_buf_map, (72ull << 32) \| BPF_F_CURRENT_CPU,
	101	&ifindex, sizeof(ifindex));
	102	return 0;
	103	}

tools/testing/selftests/bpf/progs/test_core_reloc_existence.c+79−0

@@ -0,0 +1,79 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	// Copyright (c) 2019 Facebook
	3
	4	#include <linux/bpf.h>
	5	#include <stdint.h>
	6	#include "bpf_helpers.h"
	7	#include "bpf_core_read.h"
	8
	9	char _license[] SEC("license") = "GPL";
	10
	11	static volatile struct data {
	12	char in[256];
	13	char out[256];
	14	} data;
	15
	16	struct core_reloc_existence_output {
	17	int a_exists;
	18	int a_value;
	19	int b_exists;
	20	int b_value;
	21	int c_exists;
	22	int c_value;
	23	int arr_exists;
	24	int arr_value;
	25	int s_exists;
	26	int s_value;
	27	};
	28
	29	struct core_reloc_existence {
	30	struct {
	31	int x;
	32	} s;
	33	int arr[1];
	34	int a;
	35	struct {
	36	int b;
	37	};
	38	int c;
	39	};
	40
	41	SEC("raw_tracepoint/sys_enter")
	42	int test_core_existence(void *ctx)
	43	{
	44	struct core_reloc_existence in = (void )&data.in;
	45	struct core_reloc_existence_output out = (void )&data.out;
	46
	47	out->a_exists = bpf_core_field_exists(in->a);
	48	if (bpf_core_field_exists(in->a))
	49	out->a_value = BPF_CORE_READ(in, a);
	50	else
	51	out->a_value = 0xff000001u;
	52
	53	out->b_exists = bpf_core_field_exists(in->b);
	54	if (bpf_core_field_exists(in->b))
	55	out->b_value = BPF_CORE_READ(in, b);
	56	else
	57	out->b_value = 0xff000002u;
	58
	59	out->c_exists = bpf_core_field_exists(in->c);
	60	if (bpf_core_field_exists(in->c))
	61	out->c_value = BPF_CORE_READ(in, c);
	62	else
	63	out->c_value = 0xff000003u;
	64
	65	out->arr_exists = bpf_core_field_exists(in->arr);
	66	if (bpf_core_field_exists(in->arr))
	67	out->arr_value = BPF_CORE_READ(in, arr[0]);
	68	else
	69	out->arr_value = 0xff000004u;
	70
	71	out->s_exists = bpf_core_field_exists(in->s);
	72	if (bpf_core_field_exists(in->s))
	73	out->s_value = BPF_CORE_READ(in, s.x);
	74	else
	75	out->s_value = 0xff000005u;
	76
	77	return 0;
	78	}
	79

tools/testing/selftests/bpf/progs/test_core_reloc_kernel.c+2−1

@@ -15,7 +15,8 @@ static volatile struct data { · 14 unmodified lines
15		15
16	struct core_reloc_kernel_output {	16	struct core_reloc_kernel_output {
17	int valid[10];	17	int valid[10];
18	char comm[16];	18	/* we have test_progs[-flavor], so cut flavor part */
		19	char comm[sizeof("test_progs")];
19	int comm_len;	20	int comm_len;
20	};	21	};
21		22

tools/testing/selftests/bpf/progs/test_queue_stack_map.hrenamed from tools/testing/selftests/bpf/test_queue_stack_map.h+0−0

File renamed without changes.

tools/testing/selftests/bpf/progs/test_sk_lookup_kern.c+9−9

@@ -53,7 +53,7 @@ static struct bpf_sock_tuple get_tuple(void data, __u64 nh_off, · 52 unmodified lines
53	return result;	53	return result;
54	}	54	}
55		55
56	SEC("sk_lookup_success")	56	SEC("classifier/sk_lookup_success")
57	int bpf_sk_lookup_test0(struct __sk_buff *skb)	57	int bpf_sk_lookup_test0(struct __sk_buff *skb)
58	{	58	{
59	void data_end = (void )(long)skb->data_end;	59	void data_end = (void )(long)skb->data_end;
@@ -78,7 +78,7 @@ int bpf_sk_lookup_test0(struct __sk_buff *skb) · 18 unmodified lines
78	return sk ? TC_ACT_OK : TC_ACT_UNSPEC;	78	return sk ? TC_ACT_OK : TC_ACT_UNSPEC;
79	}	79	}
80		80
81	SEC("sk_lookup_success_simple")	81	SEC("classifier/sk_lookup_success_simple")
82	int bpf_sk_lookup_test1(struct __sk_buff *skb)	82	int bpf_sk_lookup_test1(struct __sk_buff *skb)
83	{	83	{
84	struct bpf_sock_tuple tuple = {};	84	struct bpf_sock_tuple tuple = {};
@@ -90,7 +90,7 @@ int bpf_sk_lookup_test1(struct __sk_buff *skb) · 5 unmodified lines
90	return 0;	90	return 0;
91	}	91	}
92		92
93	SEC("fail_use_after_free")	93	SEC("classifier/fail_use_after_free")
94	int bpf_sk_lookup_uaf(struct __sk_buff *skb)	94	int bpf_sk_lookup_uaf(struct __sk_buff *skb)
95	{	95	{
96	struct bpf_sock_tuple tuple = {};	96	struct bpf_sock_tuple tuple = {};
@@ -105,7 +105,7 @@ int bpf_sk_lookup_uaf(struct __sk_buff *skb) · 8 unmodified lines
105	return family;	105	return family;
106	}	106	}
107		107
108	SEC("fail_modify_sk_pointer")	108	SEC("classifier/fail_modify_sk_pointer")
109	int bpf_sk_lookup_modptr(struct __sk_buff *skb)	109	int bpf_sk_lookup_modptr(struct __sk_buff *skb)
110	{	110	{
111	struct bpf_sock_tuple tuple = {};	111	struct bpf_sock_tuple tuple = {};
@@ -120,7 +120,7 @@ int bpf_sk_lookup_modptr(struct __sk_buff *skb) · 8 unmodified lines
120	return 0;	120	return 0;
121	}	121	}
122		122
123	SEC("fail_modify_sk_or_null_pointer")	123	SEC("classifier/fail_modify_sk_or_null_pointer")
124	int bpf_sk_lookup_modptr_or_null(struct __sk_buff *skb)	124	int bpf_sk_lookup_modptr_or_null(struct __sk_buff *skb)
125	{	125	{
126	struct bpf_sock_tuple tuple = {};	126	struct bpf_sock_tuple tuple = {};
@@ -134,7 +134,7 @@ int bpf_sk_lookup_modptr_or_null(struct __sk_buff *skb) · 7 unmodified lines
134	return 0;	134	return 0;
135	}	135	}
136		136
137	SEC("fail_no_release")	137	SEC("classifier/fail_no_release")
138	int bpf_sk_lookup_test2(struct __sk_buff *skb)	138	int bpf_sk_lookup_test2(struct __sk_buff *skb)
139	{	139	{
140	struct bpf_sock_tuple tuple = {};	140	struct bpf_sock_tuple tuple = {};
@@ -143,7 +143,7 @@ int bpf_sk_lookup_test2(struct __sk_buff *skb) · 2 unmodified lines
143	return 0;	143	return 0;
144	}	144	}
145		145
146	SEC("fail_release_twice")	146	SEC("classifier/fail_release_twice")
147	int bpf_sk_lookup_test3(struct __sk_buff *skb)	147	int bpf_sk_lookup_test3(struct __sk_buff *skb)
148	{	148	{
149	struct bpf_sock_tuple tuple = {};	149	struct bpf_sock_tuple tuple = {};
@@ -155,7 +155,7 @@ int bpf_sk_lookup_test3(struct __sk_buff *skb) · 5 unmodified lines
155	return 0;	155	return 0;
156	}	156	}
157		157
158	SEC("fail_release_unchecked")	158	SEC("classifier/fail_release_unchecked")
159	int bpf_sk_lookup_test4(struct __sk_buff *skb)	159	int bpf_sk_lookup_test4(struct __sk_buff *skb)
160	{	160	{
161	struct bpf_sock_tuple tuple = {};	161	struct bpf_sock_tuple tuple = {};
@@ -172,7 +172,7 @@ void lookup_no_release(struct __sk_buff *skb) · 10 unmodified lines
172	bpf_sk_lookup_tcp(skb, &tuple, sizeof(tuple), BPF_F_CURRENT_NETNS, 0);	172	bpf_sk_lookup_tcp(skb, &tuple, sizeof(tuple), BPF_F_CURRENT_NETNS, 0);
173	}	173	}
174		174
175	SEC("fail_no_release_subcall")	175	SEC("classifier/fail_no_release_subcall")
176	int bpf_sk_lookup_test5(struct __sk_buff *skb)	176	int bpf_sk_lookup_test5(struct __sk_buff *skb)
177	{	177	{
178	lookup_no_release(skb);	178	lookup_no_release(skb);

tools/testing/selftests/bpf/progs/test_skb_ctx.c+1−0

@@ -16,6 +16,7 @@ int process(struct __sk_buff *skb) · 15 unmodified lines
16	skb->cb[i]++;	16	skb->cb[i]++;
17	}	17	}
18	skb->priority++;	18	skb->priority++;
		19	skb->tstamp++;
19		20
20	return 0;	21	return 0;
21	}	22	}

tools/testing/selftests/bpf/test_libbpf.sh+0−43

@@ -1,43 +0,0 @@
1	#!/bin/sh
2	# SPDX-License-Identifier: GPL-2.0
3
4	export TESTNAME=test_libbpf
5
6	# Determine selftest success via shell exit code
7	exit_handler()
8	{
9	if [ $? -eq 0 ]; then
10	echo "selftests: $TESTNAME [PASS]";
11	else
12	echo "$TESTNAME: failed at file $LAST_LOADED" 1>&2
13	echo "selftests: $TESTNAME [FAILED]";
14	fi
15	}
16
17	libbpf_open_file()
18	{
19	LAST_LOADED=$1
20	if [ -n "$VERBOSE" ]; then
21	./test_libbpf_open $1
22	else
23	./test_libbpf_open --quiet $1
24	fi
25	}
26
27	# Exit script immediately (well catched by trap handler) if any
28	# program/thing exits with a non-zero status.
29	set -e
30
31	# (Use 'trap -l' to list meaning of numbers)
32	trap exit_handler 0 2 3 6 9
33
34	libbpf_open_file test_l4lb.o
35
36	# Load a program with BPF-to-BPF calls
37	libbpf_open_file test_l4lb_noinline.o
38
39	# Load a program compiled without the "-target bpf" flag
40	libbpf_open_file test_xdp.o
41
42	# Success
43	exit 0

tools/testing/selftests/bpf/test_libbpf_open.c+0−144

@@ -1,144 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0
2	* Copyright (c) 2018 Jesper Dangaard Brouer, Red Hat Inc.
3	*/
4	static const char *__doc__ =
5	"Libbpf test program for loading BPF ELF object files";
6
7	#include <stdlib.h>
8	#include <stdio.h>
9	#include <string.h>
10	#include <stdarg.h>
11	#include <bpf/libbpf.h>
12	#include <getopt.h>
13
14	#include "bpf_rlimit.h"
15
16	static const struct option long_options[] = {
17	{"help", no_argument, NULL, 'h' },
18	{"debug", no_argument, NULL, 'D' },
19	{"quiet", no_argument, NULL, 'q' },
20	{0, 0, NULL, 0 }
21	};
22
23	static void usage(char *argv[])
24	{
25	int i;
26
27	printf("\nDOCUMENTATION:\n%s\n\n", __doc__);
28	printf(" Usage: %s (options-see-below) BPF_FILE\n", argv[0]);
29	printf(" Listing options:\n");
30	for (i = 0; long_options[i].name != 0; i++) {
31	printf(" --%-12s", long_options[i].name);
32	printf(" short-option: -%c",
33	long_options[i].val);
34	printf("\n");
35	}
36	printf("\n");
37	}
38
39	static bool debug = 0;
40	static int libbpf_debug_print(enum libbpf_print_level level,
41	const char *fmt, va_list args)
42	{
43	if (level == LIBBPF_DEBUG && !debug)
44	return 0;
45
46	fprintf(stderr, "[%d] ", level);
47	return vfprintf(stderr, fmt, args);
48	}
49
50	#define EXIT_FAIL_LIBBPF EXIT_FAILURE
51	#define EXIT_FAIL_OPTION 2
52
53	int test_walk_progs(struct bpf_object *obj, bool verbose)
54	{
55	struct bpf_program *prog;
56	int cnt = 0;
57
58	bpf_object__for_each_program(prog, obj) {
59	cnt++;
60	if (verbose)
61	printf("Prog (count:%d) section_name: %s\n", cnt,
62	bpf_program__title(prog, false));
63	}
64	return 0;
65	}
66
67	int test_walk_maps(struct bpf_object *obj, bool verbose)
68	{
69	struct bpf_map *map;
70	int cnt = 0;
71
72	bpf_object__for_each_map(map, obj) {
73	cnt++;
74	if (verbose)
75	printf("Map (count:%d) name: %s\n", cnt,
76	bpf_map__name(map));
77	}
78	return 0;
79	}
80
81	int test_open_file(char *filename, bool verbose)
82	{
83	struct bpf_object *bpfobj = NULL;
84	long err;
85
86	if (verbose)
87	printf("Open BPF ELF-file with libbpf: %s\n", filename);
88
89	/* Load BPF ELF object file and check for errors */
90	bpfobj = bpf_object__open(filename);
91	err = libbpf_get_error(bpfobj);
92	if (err) {
93	char err_buf[128];
94	libbpf_strerror(err, err_buf, sizeof(err_buf));
95	if (verbose)
96	printf("Unable to load eBPF objects in file '%s': %s\n",
97	filename, err_buf);
98	return EXIT_FAIL_LIBBPF;
99	}
100	test_walk_progs(bpfobj, verbose);
101	test_walk_maps(bpfobj, verbose);
102
103	if (verbose)
104	printf("Close BPF ELF-file with libbpf: %s\n",
105	bpf_object__name(bpfobj));
106	bpf_object__close(bpfobj);
107
108	return 0;
109	}
110
111	int main(int argc, char **argv)
112	{
113	char filename[1024] = { 0 };
114	bool verbose = 1;
115	int longindex = 0;
116	int opt;
117
118	libbpf_set_print(libbpf_debug_print);
119
120	/* Parse commands line args */
121	while ((opt = getopt_long(argc, argv, "hDq",
122	long_options, &longindex)) != -1) {
123	switch (opt) {
124	case 'D':
125	debug = 1;
126	break;
127	case 'q': /* Use in scripting mode */
128	verbose = 0;
129	break;
130	case 'h':
131	default:
132	usage(argv);
133	return EXIT_FAIL_OPTION;
134	}
135	}
136	if (optind >= argc) {
137	usage(argv);
138	printf("ERROR: Expected BPF_FILE argument after options\n");
139	return EXIT_FAIL_OPTION;
140	}
141	snprintf(filename, sizeof(filename), "%s", argv[optind]);
142
143	return test_open_file(filename, verbose);
144	}

tools/testing/selftests/bpf/test_maps.c+4−8

 #define MAPINMAP_PROG "./test_map_in_map.o"
 static void test_map_in_map(void)
 {
-	struct bpf_program *prog;
 	struct bpf_object *obj;
 	struct bpf_map *map;
 	int mim_fd, fd, err;
 		goto out_map_in_map;
 	}
-	bpf_object__for_each_program(prog, obj) {
-		bpf_program__set_xdp(prog);
-	}
 	bpf_object__load(obj);
 	map = bpf_object__find_map_by_name(obj, "mim_array");
 	test_map_in_map();
 }
-#define DECLARE
+#define DEFINE_TEST(name) extern void test_##name(void);
 #include <map_tests/tests.h>
-#undef DECLARE
+#undef DEFINE_TEST
 int main(void)
 {
 	map_flags = BPF_F_NO_PREALLOC;
 	run_all_tests();
-#define CALL
+#define DEFINE_TEST(name) test_##name();
 #include <map_tests/tests.h>
-#undef CALL
+#undef DEFINE_TEST
 	printf("test_maps: OK, %d SKIPPED\n", skips);
 	return 0;

tools/testing/selftests/bpf/test_offload.py+1−1

@@ -312,7 +312,7 @@ class DebugfsDir: · 311 unmodified lines
312	if f == "ports":	312	if f == "ports":
313	continue	313	continue
314	p = os.path.join(path, f)	314	p = os.path.join(path, f)
315	if os.path.isfile(p):	315	if os.path.isfile(p) and os.access(p, os.R_OK):
316	_, out = cmd('cat %s/%s' % (path, f))	316	_, out = cmd('cat %s/%s' % (path, f))
317	dfs[f] = out.strip()	317	dfs[f] = out.strip()
318	elif os.path.isdir(p):	318	elif os.path.isdir(p):

tools/testing/selftests/bpf/test_progs.c+44−6

 	bool tested;
 	bool need_cgroup_cleanup;
-	const char *subtest_name;
+	char *subtest_name;
 	int subtest_num;
 	/* store counts before subtest started */
 	fprintf(env.stdout, "#%d/%d %s:%s\n",
 	       test->test_num, test->subtest_num,
 	       test->subtest_name, sub_error_cnt ? "FAIL" : "OK");
+	free(test->subtest_name);
+	test->subtest_name = NULL;
 }
 bool test__start_subtest(const char *name)
 {
 	struct prog_test_def *test = env.test;
-	if (test->subtest_name) {
+	if (test->subtest_name)
 		test__end_subtest();
-		test->subtest_name = NULL;
-	}
 	test->subtest_num++;
 	if (!should_run(&env.subtest_selector, test->subtest_num, name))
 		return false;
-	test->subtest_name = name;
+	test->subtest_name = strdup(name);
+	if (!test->subtest_name) {
+		fprintf(env.stderr,
+			"Subtest #%d: failed to copy subtest name!\n",
+			test->subtest_num);
+		return false;
+	}
 	env.test->old_error_cnt = env.test->error_cnt;
 	return true;
 }
 /* extern declarations for test funcs */
-#define DEFINE_TEST(name) extern void test_##name();
+#define DEFINE_TEST(name) extern void test_##name(void);
 #include <prog_tests/tests.h>
 #undef DEFINE_TEST
 #endif
 }
+/*
+ * Determine if test_progs is running as a "flavored" test runner and switch
+ * into corresponding sub-directory to load correct BPF objects.
+ *
+ * This is done by looking at executable name. If it contains "-flavor"
+ * suffix, then we are running as a flavored test runner.
+ */
+int cd_flavor_subdir(const char *exec_name)
+{
+	/* General form of argv[0] passed here is:
+	 * some/path/to/test_progs[-flavor], where -flavor part is optional.
+	 * First cut out "test_progs[-flavor]" part, then extract "flavor"
+	 * part, if it's there.
+	 */
+	const char *flavor = strrchr(exec_name, '/');
+	if (!flavor)
+		return 0;
+	flavor++;
+	flavor = strrchr(flavor, '-');
+	if (!flavor)
+		return 0;
+	flavor++;
+	printf("Switching to flavor '%s' subdirectory...\n", flavor);
+	return chdir(flavor);
+}
 int main(int argc, char **argv)
 {
 	static const struct argp argp = {
 	if (err)
 		return err;
+	err = cd_flavor_subdir(argv[0]);
+	if (err)
+		return err;
 	libbpf_set_print(libbpf_print_fn);
 	srand(time(NULL));

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code