Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2020-10-12 The main changes are: 1) The BPF verifier improvements to track register allocation pattern, from Alexei and Yonghong. 2) libbpf relocation support for different size load/store, from Andrii. 3) bpf_redirect_peer() helper and support for inner map array with different max_entries, from Daniel. 4) BPF support for per-cpu variables, form Hao. 5) sockmap improvements, from John. ====================

Jakub Kicinski committed 6 years ago

Commit ccdf7fa2 parents a308283 376dcfe73 files · +4339 −772Browse files

Unified · Split

73 files

Documentation/bpf/bpf_devel_QA.rst +2 −2
MAINTAINERS +1 −1
drivers/net/veth.c +9 −0
include/linux/bpf.h +7 −1
include/linux/bpf_verifier.h +7 −0
include/linux/btf.h +26 −0
include/linux/netdevice.h +4 −0
include/linux/skmsg.h +2 −0
include/net/tcp.h +0 −33
include/uapi/linux/bpf.h +87 −14
kernel/bpf/arraymap.c +11 −6
kernel/bpf/btf.c +0 −25
kernel/bpf/hashtab.c +3 −3
kernel/bpf/helpers.c +32 −0
kernel/bpf/percpu_freelist.c +96 −5
kernel/bpf/percpu_freelist.h +1 −0
kernel/bpf/syscall.c +3 −1
kernel/bpf/verifier.c +251 −19
kernel/trace/bpf_trace.c +6 −0
net/core/dev.c +12 −3
net/core/filter.c +99 −8
net/core/skmsg.c +123 −38
net/core/sock_map.c +23 −14
net/ipv4/tcp_minisocks.c +0 −1
net/xdp/xsk_buff_pool.c +0 −3
net/xdp/xsk_queue.h +4 −0
net/xdp/xskmap.c +1 −1
samples/bpf/Makefile +12 −3
samples/bpf/hbm.c +2 −1
samples/bpf/xdp_monitor_kern.c +30 −30
samples/bpf/xdp_monitor_user.c +120 −39
samples/bpf/xdp_redirect_cpu_user.c +72 −81
samples/bpf/xdp_sample_pkts_kern.c +6 −8
samples/bpf/xdp_sample_pkts_user.c +0 −1
samples/bpf/xdpsock_user.c +290 −64
tools/include/uapi/linux/bpf.h +87 −14
tools/lib/bpf/libbpf.c +289 −59
tools/lib/bpf/xsk.c +6 −1
tools/testing/selftests/bpf/README.rst +38 −0
tools/testing/selftests/bpf/prog_tests/align.c +8 −8
tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c +29 −10
tools/testing/selftests/bpf/prog_tests/core_autosize.c +225 −0
tools/testing/selftests/bpf/prog_tests/ksyms.c +13 −25
tools/testing/selftests/bpf/prog_tests/ksyms_btf.c +88 −0
tools/testing/selftests/bpf/prog_tests/pinning.c +48 −1
tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +1 −1
tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c +12 −0
tools/testing/selftests/bpf/prog_tests/test_profiler.c +72 −0
tools/testing/selftests/bpf/prog_tests/xdp_noinline.c +1 −1
tools/testing/selftests/bpf/progs/connect4_prog.c +19 −0
tools/testing/selftests/bpf/progs/profiler.h +177 −0
tools/testing/selftests/bpf/progs/profiler.inc.h +969 −0
tools/testing/selftests/bpf/progs/profiler1.c +6 −0
tools/testing/selftests/bpf/progs/profiler2.c +6 −0
tools/testing/selftests/bpf/progs/profiler3.c +6 −0
tools/testing/selftests/bpf/progs/test_btf_map_in_map.c +43 −0
tools/testing/selftests/bpf/progs/test_core_autosize.c +172 −0
tools/testing/selftests/bpf/progs/test_ksyms_btf.c +55 −0
tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c +2 −2
tools/testing/selftests/bpf/progs/test_sockmap_kern.h +25 −9
tools/testing/selftests/bpf/progs/test_tc_neigh.c +22 −18
tools/testing/selftests/bpf/progs/test_tc_peer.c +45 −0
tools/testing/selftests/bpf/progs/test_tcp_hdr_options.c +5 −2
tools/testing/selftests/bpf/test_sockmap.c +45 −36
tools/testing/selftests/bpf/test_tc_neigh.sh +0 −168
tools/testing/selftests/bpf/test_tc_redirect.sh +204 −0
tools/testing/selftests/bpf/test_tcp_hdr_options.h +3 −2
tools/testing/selftests/bpf/trace_helpers.c +27 −0
tools/testing/selftests/bpf/trace_helpers.h +4 −0
tools/testing/selftests/bpf/verifier/basic.c +1 −1
tools/testing/selftests/bpf/verifier/direct_packet_access.c +1 −1
tools/testing/selftests/bpf/verifier/ld_imm64.c +0 −8
tools/testing/selftests/bpf/verifier/regalloc.c +243 −0

Documentation/bpf/bpf_devel_QA.rst+2−2

@@ -60,13 +60,13 @@ Q: Where can I find patches currently under discussion for BPF subsystem? · 59 unmodified lines
60	60	A: All patches that are Cc'ed to netdev are queued for review under netdev
61	61	patchwork project:
62	62
63		− http://patchwork.ozlabs.org/project/netdev/list/
	63	+ https://patchwork.kernel.org/project/netdevbpf/list/
64	64
65	65	Those patches which target BPF, are assigned to a 'bpf' delegate for
66	66	further processing from BPF maintainers. The current queue with
67	67	patches under review can be found at:
68	68
69		− https://patchwork.ozlabs.org/project/netdev/list/?delegate=77147
	69	+ https://patchwork.kernel.org/project/netdevbpf/list/?delegate=121173
70	70
71	71	Once the patches have been reviewed by the BPF community as a whole
72	72	and approved by the BPF maintainers, their status in patchwork will be

MAINTAINERS+1−1

@@ -3263,7 +3263,7 @@ M: Daniel Borkmann <daniel@iogearbox.net> · 3262 unmodified lines
3263	3263	R: Martin KaFai Lau <kafai@fb.com>
3264	3264	R: Song Liu <songliubraving@fb.com>
3265	3265	R: Yonghong Song <yhs@fb.com>
3266		−R: Andrii Nakryiko <andriin@fb.com>
	3266	+R: Andrii Nakryiko <andrii@kernel.org>
3267	3267	R: John Fastabend <john.fastabend@gmail.com>
3268	3268	R: KP Singh <kpsingh@chromium.org>
3269	3269	L: netdev@vger.kernel.org

drivers/net/veth.c+9−0

@@ -420,6 +420,14 @@ static int veth_select_rxq(struct net_device *dev) · 419 unmodified lines
420	420	return smp_processor_id() % dev->real_num_rx_queues;
421	421	}
422	422
	423	+static struct net_device veth_peer_dev(struct net_device dev)
	424	+{
	425	+ struct veth_priv *priv = netdev_priv(dev);
	426	+
	427	+ /* Callers must be under RCU read side. */
	428	+ return rcu_dereference(priv->peer);
	429	+}
	430	+
423	431	static int veth_xdp_xmit(struct net_device *dev, int n,
424	432	struct xdp_frame **frames,
425	433	u32 flags, bool ndo_xmit)
@@ -1224,6 +1232,7 @@ static const struct net_device_ops veth_netdev_ops = { · 798 unmodified lines
1224	1232	.ndo_set_rx_headroom = veth_set_rx_headroom,
1225	1233	.ndo_bpf = veth_xdp,
1226	1234	.ndo_xdp_xmit = veth_ndo_xdp_xmit,
	1235	+ .ndo_get_peer_dev = veth_peer_dev,
1227	1236	};
1228	1237
1229	1238	#define VETH_FEATURES (NETIF_F_SG \| NETIF_F_FRAGLIST \| NETIF_F_HW_CSUM \| \

include/linux/bpf.h+7−1

@@ -82,7 +82,7 @@ struct bpf_map_ops { · 81 unmodified lines
82	82	void (map_fd_get_ptr)(struct bpf_map map, struct file map_file,
83	83	int fd);
84	84	void (map_fd_put_ptr)(void ptr);
85		− u32 (map_gen_lookup)(struct bpf_map map, struct bpf_insn *insn_buf);
	85	+ int (map_gen_lookup)(struct bpf_map map, struct bpf_insn *insn_buf);
86	86	u32 (map_fd_sys_lookup_elem)(void ptr);
87	87	void (map_seq_show_elem)(struct bpf_map map, void *key,
88	88	struct seq_file *m);
@@ -293,6 +293,7 @@ enum bpf_arg_type { · 204 unmodified lines
293	293	ARG_PTR_TO_ALLOC_MEM_OR_NULL, /* pointer to dynamically allocated memory or NULL */
294	294	ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */
295	295	ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
	296	+ ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */
296	297	__BPF_ARG_TYPE_MAX,
297	298	};
298	299
@@ -307,6 +308,8 @@ enum bpf_return_type { · 8 unmodified lines
307	308	RET_PTR_TO_SOCK_COMMON_OR_NULL, /* returns a pointer to a sock_common or NULL */
308	309	RET_PTR_TO_ALLOC_MEM_OR_NULL, /* returns a pointer to dynamically allocated memory or NULL */
309	310	RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */
	311	+ RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
	312	+ RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
310	313	};
311	314
312	315	/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
@@ -405,6 +408,7 @@ enum bpf_reg_type { · 92 unmodified lines
405	408	PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
406	409	PTR_TO_RDWR_BUF, /* reg points to a read/write buffer */
407	410	PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
	411	+ PTR_TO_PERCPU_BTF_ID, /* reg points to a percpu kernel variable */
408	412	};
409	413
410	414	/* The information passed from prog-specific *_is_valid_access
@@ -1828,6 +1832,8 @@ extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; · 1417 unmodified lines
1828	1832	extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
1829	1833	extern const struct bpf_func_proto bpf_copy_from_user_proto;
1830	1834	extern const struct bpf_func_proto bpf_snprintf_btf_proto;
	1835	+extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
	1836	+extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
1831	1837
1832	1838	const struct bpf_func_proto *bpf_tracing_func_proto(
1833	1839	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_verifier.h+7−0

@@ -308,6 +308,13 @@ struct bpf_insn_aux_data { · 307 unmodified lines
308	308	u32 map_index; /* index into used_maps[] */
309	309	u32 map_off; /* offset from value base address */
310	310	};
	311	+ struct {
	312	+ enum bpf_reg_type reg_type; /* type of pseudo_btf_id */
	313	+ union {
	314	+ u32 btf_id; /* btf_id for struct typed var */
	315	+ u32 mem_size; /* mem_size for non-struct typed var */
	316	+ };
	317	+ } btf_var;
311	318	};
312	319	u64 map_key_state; /* constant (32 bit) key tracking for maps */
313	320	int ctx_field_size; /* the ctx field size for load insn, maybe 0 */

include/linux/btf.h+26−0

@@ -110,6 +110,11 @@ btf_resolve_size(const struct btf btf, const struct btf_type type, · 109 unmodified lines
110	110	i < btf_type_vlen(struct_type); \
111	111	i++, member++)
112	112
	113	+#define for_each_vsi(i, datasec_type, member) \
	114	+ for (i = 0, member = btf_type_var_secinfo(datasec_type); \
	115	+ i < btf_type_vlen(datasec_type); \
	116	+ i++, member++)
	117	+
113	118	static inline bool btf_type_is_ptr(const struct btf_type *t)
114	119	{
115	120	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
@@ -145,6 +150,21 @@ static inline bool btf_type_is_func_proto(const struct btf_type *t) · 29 unmodified lines
145	150	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
146	151	}
147	152
	153	+static inline bool btf_type_is_var(const struct btf_type *t)
	154	+{
	155	+ return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
	156	+}
	157	+
	158	+/* union is only a special case of struct:
	159	+ * all its offsetof(member) == 0
	160	+ */
	161	+static inline bool btf_type_is_struct(const struct btf_type *t)
	162	+{
	163	+ u8 kind = BTF_INFO_KIND(t->info);
	164	+
	165	+ return kind == BTF_KIND_STRUCT \|\| kind == BTF_KIND_UNION;
	166	+}
	167	+
148	168	static inline u16 btf_type_vlen(const struct btf_type *t)
149	169	{
150	170	return BTF_INFO_VLEN(t->info);
@@ -179,6 +199,12 @@ static inline const struct btf_member btf_type_member(const struct btf_type t) · 28 unmodified lines
179	199	return (const struct btf_member *)(t + 1);
180	200	}
181	201
	202	+static inline const struct btf_var_secinfo *btf_type_var_secinfo(
	203	+ const struct btf_type *t)
	204	+{
	205	+ return (const struct btf_var_secinfo *)(t + 1);
	206	+}
	207	+
182	208	#ifdef CONFIG_BPF_SYSCALL
183	209	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);
184	210	const char btf_name_by_offset(const struct btf btf, u32 offset);

include/linux/netdevice.h+4−0

@@ -1276,6 +1276,9 @@ struct netdev_net_notifier { · 1275 unmodified lines
1276	1276	* int (ndo_tunnel_ctl)(struct net_device dev, struct ip_tunnel_parm *p,
1277	1277	* int cmd);
1278	1278	* Add, change, delete or get information on an IPv4 tunnel.
	1279	+ * struct net_device (ndo_get_peer_dev)(struct net_device *dev);
	1280	+ * If a device is paired with a peer device, return the peer instance.
	1281	+ * The caller must be under RCU read context.
1279	1282	*/
1280	1283	struct net_device_ops {
1281	1284	int (ndo_init)(struct net_device dev);
@@ -1483,6 +1486,7 @@ struct net_device_ops { · 201 unmodified lines
1483	1486	struct devlink_port * (ndo_get_devlink_port)(struct net_device dev);
1484	1487	int (ndo_tunnel_ctl)(struct net_device dev,
1485	1488	struct ip_tunnel_parm *p, int cmd);
	1489	+ struct net_device * (ndo_get_peer_dev)(struct net_device dev);
1486	1490	};
1487	1491
1488	1492	/**

include/linux/skmsg.h+2−0

@@ -308,6 +308,8 @@ struct sk_psock sk_psock_init(struct sock sk, int node); · 307 unmodified lines
308	308	int sk_psock_init_strp(struct sock sk, struct sk_psock psock);
309	309	void sk_psock_start_strp(struct sock sk, struct sk_psock psock);
310	310	void sk_psock_stop_strp(struct sock sk, struct sk_psock psock);
	311	+void sk_psock_start_verdict(struct sock sk, struct sk_psock psock);
	312	+void sk_psock_stop_verdict(struct sock sk, struct sk_psock psock);
311	313
312	314	int sk_psock_msg_verdict(struct sock sk, struct sk_psock psock,
313	315	struct sk_msg *msg);

include/net/tcp.h+0−33

@@ -2228,34 +2228,6 @@ int __tcp_bpf_recvmsg(struct sock sk, struct sk_psock psock, · 2227 unmodified lines
2228	2228	#endif /* CONFIG_NET_SOCK_MSG */
2229	2229
2230	2230	#ifdef CONFIG_CGROUP_BPF
2231		−/* Copy the listen sk's HDR_OPT_CB flags to its child.
2232		− *
2233		− * During 3-Way-HandShake, the synack is usually sent from
2234		− * the listen sk with the HDR_OPT_CB flags set so that
2235		− * bpf-prog will be called to write the BPF hdr option.
2236		− *
2237		− * In fastopen, the child sk is used to send synack instead
2238		− * of the listen sk. Thus, inheriting the HDR_OPT_CB flags
2239		− * from the listen sk gives the bpf-prog a chance to write
2240		− * BPF hdr option in the synack pkt during fastopen.
2241		− *
2242		− * Both fastopen and non-fastopen child will inherit the
2243		− * HDR_OPT_CB flags to keep the bpf-prog having a consistent
2244		− * behavior when deciding to clear this cb flags (or not)
2245		− * during the PASSIVE_ESTABLISHED_CB.
2246		− *
2247		− * In the future, other cb flags could be inherited here also.
2248		− */
2249		−static inline void bpf_skops_init_child(const struct sock *sk,
2250		− struct sock *child)
2251		−{
2252		− tcp_sk(child)->bpf_sock_ops_cb_flags =
2253		− tcp_sk(sk)->bpf_sock_ops_cb_flags &
2254		− (BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG \|
2255		− BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG \|
2256		− BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG);
2257		−}
2258		−
2259	2231	static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2260	2232	struct sk_buff *skb,
2261	2233	unsigned int end_offset)
@@ -2264,11 +2236,6 @@ static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, · 2 unmodified lines
2264	2236	skops->skb_data_end = skb->data + end_offset;
2265	2237	}
2266	2238	#else
2267		−static inline void bpf_skops_init_child(const struct sock *sk,
2268		− struct sock *child)
2269		−{
2270		−}
2271		−
2272	2239	static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2273	2240	struct sk_buff *skb,
2274	2241	unsigned int end_offset)

include/uapi/linux/bpf.h+87−14

@@ -356,18 +356,36 @@ enum bpf_link_type { · 355 unmodified lines
356	356	#define BPF_F_SLEEPABLE (1U << 4)
357	357
358	358	/* When BPF ldimm64's insn[0].src_reg != 0 then this can have
359		− * two extensions:
360		− *
361		− * insn[0].src_reg: BPF_PSEUDO_MAP_FD BPF_PSEUDO_MAP_VALUE
362		− * insn[0].imm: map fd map fd
363		− * insn[1].imm: 0 offset into value
364		− * insn[0].off: 0 0
365		− * insn[1].off: 0 0
366		− * ldimm64 rewrite: address of map address of map[0]+offset
367		− * verifier type: CONST_PTR_TO_MAP PTR_TO_MAP_VALUE
	359	+ * the following extensions:
	360	+ *
	361	+ * insn[0].src_reg: BPF_PSEUDO_MAP_FD
	362	+ * insn[0].imm: map fd
	363	+ * insn[1].imm: 0
	364	+ * insn[0].off: 0
	365	+ * insn[1].off: 0
	366	+ * ldimm64 rewrite: address of map
	367	+ * verifier type: CONST_PTR_TO_MAP
368	368	*/
369	369	#define BPF_PSEUDO_MAP_FD 1
	370	+/* insn[0].src_reg: BPF_PSEUDO_MAP_VALUE
	371	+ * insn[0].imm: map fd
	372	+ * insn[1].imm: offset into value
	373	+ * insn[0].off: 0
	374	+ * insn[1].off: 0
	375	+ * ldimm64 rewrite: address of map[0]+offset
	376	+ * verifier type: PTR_TO_MAP_VALUE
	377	+ */
370	378	#define BPF_PSEUDO_MAP_VALUE 2
	379	+/* insn[0].src_reg: BPF_PSEUDO_BTF_ID
	380	+ * insn[0].imm: kernel btd id of VAR
	381	+ * insn[1].imm: 0
	382	+ * insn[0].off: 0
	383	+ * insn[1].off: 0
	384	+ * ldimm64 rewrite: address of the kernel variable
	385	+ * verifier type: PTR_TO_BTF_ID or PTR_TO_MEM, depending on whether the var
	386	+ * is struct/union.
	387	+ */
	388	+#define BPF_PSEUDO_BTF_ID 3
371	389
372	390	/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
373	391	* offset to another bpf function
@@ -417,6 +435,9 @@ enum { · 43 unmodified lines
417	435
418	436	/* Share perf_event among processes */
419	437	BPF_F_PRESERVE_ELEMS = (1U << 11),
	438	+
	439	+/* Create a map that is suitable to be an inner map with dynamic max entries */
	440	+ BPF_F_INNER_MAP = (1U << 12),
420	441	};
421	442
422	443	/* Flags for BPF_PROG_QUERY. */
@@ -1680,7 +1701,7 @@ union bpf_attr { · 1257 unmodified lines
1680	1701	* TCP_CONGESTION, TCP_BPF_IW,
1681	1702	* TCP_BPF_SNDCWND_CLAMP, TCP_SAVE_SYN,
1682	1703	* TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT,
1683		− * TCP_SYNCNT, TCP_USER_TIMEOUT.
	1704	+ * TCP_SYNCNT, TCP_USER_TIMEOUT, TCP_NOTSENT_LOWAT.
1684	1705	* * IPPROTO_IP, which supports optname IP_TOS.
1685	1706	* * IPPROTO_IPV6, which supports optname IPV6_TCLASS.
1686	1707	* Return
@@ -2235,7 +2256,7 @@ union bpf_attr { · 548 unmodified lines
2235	2256	* Description
2236	2257	* This helper is used in programs implementing policies at the
2237	2258	* skb socket level. If the sk_buff skb is allowed to pass (i.e.
2238		− * if the verdeict eBPF program returns SK_PASS), redirect it
	2259	+ * if the verdict eBPF program returns SK_PASS), redirect it
2239	2260	* to the socket referenced by map (of type
2240	2261	* BPF_MAP_TYPE_SOCKHASH) using hash key. Both ingress and
2241	2262	* egress interfaces can be used for redirection. The
@@ -3661,10 +3682,59 @@ union bpf_attr { · 1419 unmodified lines
3661	3682	* Redirect the packet to another net device of index ifindex
3662	3683	* and fill in L2 addresses from neighboring subsystem. This helper
3663	3684	* is somewhat similar to bpf_redirect\ (), except that it
3664		− * fills in e.g. MAC addresses based on the L3 information from
3665		− * the packet. This helper is supported for IPv4 and IPv6 protocols.
	3685	+ * populates L2 addresses as well, meaning, internally, the helper
	3686	+ * performs a FIB lookup based on the skb's networking header to
	3687	+ * get the address of the next hop and then relies on the neighbor
	3688	+ * lookup for the L2 address of the nexthop.
	3689	+ *
	3690	+ * The flags argument is reserved and must be 0. The helper is
	3691	+ * currently only supported for tc BPF program types, and enabled
	3692	+ * for IPv4 and IPv6 protocols.
	3693	+ * Return
	3694	+ * The helper returns TC_ACT_REDIRECT on success or
	3695	+ * TC_ACT_SHOT on error.
	3696	+ *
	3697	+ * void bpf_per_cpu_ptr(const void percpu_ptr, u32 cpu)
	3698	+ * Description
	3699	+ * Take a pointer to a percpu ksym, percpu_ptr, and return a
	3700	+ * pointer to the percpu kernel variable on cpu. A ksym is an
	3701	+ * extern variable decorated with '__ksym'. For ksym, there is a
	3702	+ * global var (either static or global) defined of the same name
	3703	+ * in the kernel. The ksym is percpu if the global var is percpu.
	3704	+ * The returned pointer points to the global percpu var on cpu.
	3705	+ *
	3706	+ * bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the
	3707	+ * kernel, except that bpf_per_cpu_ptr() may return NULL. This
	3708	+ * happens if cpu is larger than nr_cpu_ids. The caller of
	3709	+ * bpf_per_cpu_ptr() must check the returned value.
	3710	+ * Return
	3711	+ * A pointer pointing to the kernel percpu variable on cpu, or
	3712	+ * NULL, if cpu is invalid.
	3713	+ *
	3714	+ * void bpf_this_cpu_ptr(const void percpu_ptr)
	3715	+ * Description
	3716	+ * Take a pointer to a percpu ksym, percpu_ptr, and return a
	3717	+ * pointer to the percpu kernel variable on this cpu. See the
	3718	+ * description of 'ksym' in bpf_per_cpu_ptr\ ().
	3719	+ *
	3720	+ * bpf_this_cpu_ptr() has the same semantic as this_cpu_ptr() in
	3721	+ * the kernel. Different from bpf_per_cpu_ptr\ (), it would
	3722	+ * never return NULL.
	3723	+ * Return
	3724	+ * A pointer pointing to the kernel percpu variable on this cpu.
	3725	+ *
	3726	+ * long bpf_redirect_peer(u32 ifindex, u64 flags)
	3727	+ * Description
	3728	+ * Redirect the packet to another net device of index ifindex.
	3729	+ * This helper is somewhat similar to bpf_redirect\ (), except
	3730	+ * that the redirection happens to the ifindex' peer device and
	3731	+ * the netns switch takes place from ingress to ingress without
	3732	+ * going through the CPU's backlog queue.
	3733	+ *
3666	3734	* The flags argument is reserved and must be 0. The helper is
3667		− * currently only supported for tc BPF program types.
	3735	+ * currently only supported for tc BPF program types at the ingress
	3736	+ * hook and for veth device types. The peer device must reside in a
	3737	+ * different network namespace.
3668	3738	* Return
3669	3739	* The helper returns TC_ACT_REDIRECT on success or
3670	3740	* TC_ACT_SHOT on error.
@@ -3823,6 +3893,9 @@ union bpf_attr { · 152 unmodified lines
3823	3893	FN(seq_printf_btf), \
3824	3894	FN(skb_cgroup_classid), \
3825	3895	FN(redirect_neigh), \
	3896	+ FN(bpf_per_cpu_ptr), \
	3897	+ FN(bpf_this_cpu_ptr), \
	3898	+ FN(redirect_peer), \
3826	3899	/* */
3827	3900
3828	3901	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/arraymap.c+11−6

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	16
17	17	#define ARRAY_CREATE_FLAG_MASK \
18	18	(BPF_F_NUMA_NODE \| BPF_F_MMAPABLE \| BPF_F_ACCESS_MASK \| \
19		− BPF_F_PRESERVE_ELEMS)
	19	+ BPF_F_PRESERVE_ELEMS \| BPF_F_INNER_MAP)
20	20
21	21	static void bpf_array_free_percpu(struct bpf_array *array)
22	22	{
@@ -62,7 +62,7 @@ int array_map_alloc_check(union bpf_attr *attr) · 39 unmodified lines
62	62	return -EINVAL;
63	63
64	64	if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
65		− attr->map_flags & BPF_F_MMAPABLE)
	65	+ attr->map_flags & (BPF_F_MMAPABLE \| BPF_F_INNER_MAP))
66	66	return -EINVAL;
67	67
68	68	if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
@@ -214,7 +214,7 @@ static int array_map_direct_value_meta(const struct bpf_map *map, u64 imm, · 145 unmodified lines
214	214	}
215	215
216	216	/* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
217		−static u32 array_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
	217	+static int array_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
218	218	{
219	219	struct bpf_array *array = container_of(map, struct bpf_array, map);
220	220	struct bpf_insn *insn = insn_buf;
@@ -223,6 +223,9 @@ static u32 array_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf) · 2 unmodified lines
223	223	const int map_ptr = BPF_REG_1;
224	224	const int index = BPF_REG_2;
225	225
	226	+ if (map->map_flags & BPF_F_INNER_MAP)
	227	+ return -EOPNOTSUPP;
	228	+
226	229	*insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
227	230	*insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
228	231	if (!map->bypass_spec_v1) {
@@ -496,8 +499,10 @@ static int array_map_mmap(struct bpf_map map, struct vm_area_struct vma) · 267 unmodified lines
496	499	static bool array_map_meta_equal(const struct bpf_map *meta0,
497	500	const struct bpf_map *meta1)
498	501	{
499		− return meta0->max_entries == meta1->max_entries &&
500		− bpf_map_meta_equal(meta0, meta1);
	502	+ if (!bpf_map_meta_equal(meta0, meta1))
	503	+ return false;
	504	+ return meta0->map_flags & BPF_F_INNER_MAP ? true :
	505	+ meta0->max_entries == meta1->max_entries;
501	506	}
502	507
503	508	struct bpf_iter_seq_array_map_info {
@@ -1251,7 +1256,7 @@ static void array_of_map_lookup_elem(struct bpf_map map, void *key) · 747 unmodified lines
1251	1256	return READ_ONCE(*inner_map);
1252	1257	}
1253	1258
1254		−static u32 array_of_map_gen_lookup(struct bpf_map *map,
	1259	+static int array_of_map_gen_lookup(struct bpf_map *map,
1255	1260	struct bpf_insn *insn_buf)
1256	1261	{
1257	1262	struct bpf_array *array = container_of(map, struct bpf_array, map);

kernel/bpf/btf.c+0−25

@@ -188,11 +188,6 @@ · 187 unmodified lines
188	188	i < btf_type_vlen(struct_type); \
189	189	i++, member++)
190	190
191		−#define for_each_vsi(i, struct_type, member) \
192		− for (i = 0, member = btf_type_var_secinfo(struct_type); \
193		− i < btf_type_vlen(struct_type); \
194		− i++, member++)
195		−
196	191	#define for_each_vsi_from(i, from, struct_type, member) \
197	192	for (i = from, member = btf_type_var_secinfo(struct_type) + from; \
198	193	i < btf_type_vlen(struct_type); \
@@ -440,16 +435,6 @@ static bool btf_type_nosize_or_null(const struct btf_type *t) · 241 unmodified lines
440	435	return !t \|\| btf_type_nosize(t);
441	436	}
442	437
443		−/* union is only a special case of struct:
444		− * all its offsetof(member) == 0
445		− */
446		−static bool btf_type_is_struct(const struct btf_type *t)
447		−{
448		− u8 kind = BTF_INFO_KIND(t->info);
449		−
450		− return kind == BTF_KIND_STRUCT \|\| kind == BTF_KIND_UNION;
451		−}
452		−
453	438	static bool __btf_type_is_struct(const struct btf_type *t)
454	439	{
455	440	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
@@ -460,11 +445,6 @@ static bool btf_type_is_array(const struct btf_type *t) · 4 unmodified lines
460	445	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
461	446	}
462	447
463		−static bool btf_type_is_var(const struct btf_type *t)
464		−{
465		− return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
466		−}
467		−
468	448	static bool btf_type_is_datasec(const struct btf_type *t)
469	449	{
470	450	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
@@ -613,11 +593,6 @@ static const struct btf_var btf_type_var(const struct btf_type t) · 142 unmodified lines
613	593	return (const struct btf_var *)(t + 1);
614	594	}
615	595
616		−static const struct btf_var_secinfo btf_type_var_secinfo(const struct btf_type t)
617		−{
618		− return (const struct btf_var_secinfo *)(t + 1);
619		−}
620		−
621	596	static const struct btf_kind_operations btf_type_ops(const struct btf_type t)
622	597	{
623	598	return kind_ops[BTF_INFO_KIND(t->info)];

kernel/bpf/hashtab.c+3−3

@@ -612,7 +612,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void *key) · 611 unmodified lines
612	612	* bpf_prog
613	613	* __htab_map_lookup_elem
614	614	*/
615		−static u32 htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
	615	+static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
616	616	{
617	617	struct bpf_insn *insn = insn_buf;
618	618	const int ret = BPF_REG_0;
@@ -651,7 +651,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void *key) · 32 unmodified lines
651	651	return __htab_lru_map_lookup_elem(map, key, false);
652	652	}
653	653
654		−static u32 htab_lru_map_gen_lookup(struct bpf_map *map,
	654	+static int htab_lru_map_gen_lookup(struct bpf_map *map,
655	655	struct bpf_insn *insn_buf)
656	656	{
657	657	struct bpf_insn *insn = insn_buf;
@@ -2070,7 +2070,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void *key) · 1412 unmodified lines
2070	2070	return READ_ONCE(*inner_map);
2071	2071	}
2072	2072
2073		−static u32 htab_of_map_gen_lookup(struct bpf_map *map,
	2073	+static int htab_of_map_gen_lookup(struct bpf_map *map,
2074	2074	struct bpf_insn *insn_buf)
2075	2075	{
2076	2076	struct bpf_insn *insn = insn_buf;

kernel/bpf/helpers.c+32−0

@@ -623,6 +623,34 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { · 622 unmodified lines
623	623	.arg3_type = ARG_ANYTHING,
624	624	};
625	625
	626	+BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
	627	+{
	628	+ if (cpu >= nr_cpu_ids)
	629	+ return (unsigned long)NULL;
	630	+
	631	+ return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
	632	+}
	633	+
	634	+const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
	635	+ .func = bpf_per_cpu_ptr,
	636	+ .gpl_only = false,
	637	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
	638	+ .arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
	639	+ .arg2_type = ARG_ANYTHING,
	640	+};
	641	+
	642	+BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
	643	+{
	644	+ return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);
	645	+}
	646	+
	647	+const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
	648	+ .func = bpf_this_cpu_ptr,
	649	+ .gpl_only = false,
	650	+ .ret_type = RET_PTR_TO_MEM_OR_BTF_ID,
	651	+ .arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
	652	+};
	653	+
626	654	const struct bpf_func_proto bpf_get_current_task_proto __weak;
627	655	const struct bpf_func_proto bpf_probe_read_user_proto __weak;
628	656	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;
@@ -689,6 +717,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 60 unmodified lines
689	717	return &bpf_snprintf_btf_proto;
690	718	case BPF_FUNC_jiffies64:
691	719	return &bpf_jiffies64_proto;
	720	+ case BPF_FUNC_bpf_per_cpu_ptr:
	721	+ return &bpf_per_cpu_ptr_proto;
	722	+ case BPF_FUNC_bpf_this_cpu_ptr:
	723	+ return &bpf_this_cpu_ptr_proto;
692	724	default:
693	725	break;
694	726	}

kernel/bpf/percpu_freelist.c+96−5

@@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) · 16 unmodified lines
17	17	raw_spin_lock_init(&head->lock);
18	18	head->first = NULL;
19	19	}
	20	+ raw_spin_lock_init(&s->extralist.lock);
	21	+ s->extralist.first = NULL;
20	22	return 0;
21	23	}
22	24
@@ -40,12 +42,50 @@ static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, · 17 unmodified lines
40	42	raw_spin_unlock(&head->lock);
41	43	}
42	44
	45	+static inline bool pcpu_freelist_try_push_extra(struct pcpu_freelist *s,
	46	+ struct pcpu_freelist_node *node)
	47	+{
	48	+ if (!raw_spin_trylock(&s->extralist.lock))
	49	+ return false;
	50	+
	51	+ pcpu_freelist_push_node(&s->extralist, node);
	52	+ raw_spin_unlock(&s->extralist.lock);
	53	+ return true;
	54	+}
	55	+
	56	+static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s,
	57	+ struct pcpu_freelist_node *node)
	58	+{
	59	+ int cpu, orig_cpu;
	60	+
	61	+ orig_cpu = cpu = raw_smp_processor_id();
	62	+ while (1) {
	63	+ struct pcpu_freelist_head *head;
	64	+
	65	+ head = per_cpu_ptr(s->freelist, cpu);
	66	+ if (raw_spin_trylock(&head->lock)) {
	67	+ pcpu_freelist_push_node(head, node);
	68	+ raw_spin_unlock(&head->lock);
	69	+ return;
	70	+ }
	71	+ cpu = cpumask_next(cpu, cpu_possible_mask);
	72	+ if (cpu >= nr_cpu_ids)
	73	+ cpu = 0;
	74	+
	75	+ /* cannot lock any per cpu lock, try extralist */
	76	+ if (cpu == orig_cpu &&
	77	+ pcpu_freelist_try_push_extra(s, node))
	78	+ return;
	79	+ }
	80	+}
	81	+
43	82	void __pcpu_freelist_push(struct pcpu_freelist *s,
44	83	struct pcpu_freelist_node *node)
45	84	{
46		− struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);
47		−
48		− ___pcpu_freelist_push(head, node);
	85	+ if (in_nmi())
	86	+ ___pcpu_freelist_push_nmi(s, node);
	87	+ else
	88	+ ___pcpu_freelist_push(this_cpu_ptr(s->freelist), node);
49	89	}
50	90
51	91	void pcpu_freelist_push(struct pcpu_freelist *s,
@@ -81,7 +121,7 @@ again: · 29 unmodified lines
81	121	}
82	122	}
83	123
84		−struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)
	124	+static struct pcpu_freelist_node ___pcpu_freelist_pop(struct pcpu_freelist s)
85	125	{
86	126	struct pcpu_freelist_head *head;
87	127	struct pcpu_freelist_node *node;
@@ -102,8 +142,59 @@ struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s) · 14 unmodified lines
102	142	if (cpu >= nr_cpu_ids)
103	143	cpu = 0;
104	144	if (cpu == orig_cpu)
105		− return NULL;
	145	+ break;
	146	+ }
	147	+
	148	+ /* per cpu lists are all empty, try extralist */
	149	+ raw_spin_lock(&s->extralist.lock);
	150	+ node = s->extralist.first;
	151	+ if (node)
	152	+ s->extralist.first = node->next;
	153	+ raw_spin_unlock(&s->extralist.lock);
	154	+ return node;
	155	+}
	156	+
	157	+static struct pcpu_freelist_node *
	158	+___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
	159	+{
	160	+ struct pcpu_freelist_head *head;
	161	+ struct pcpu_freelist_node *node;
	162	+ int orig_cpu, cpu;
	163	+
	164	+ orig_cpu = cpu = raw_smp_processor_id();
	165	+ while (1) {
	166	+ head = per_cpu_ptr(s->freelist, cpu);
	167	+ if (raw_spin_trylock(&head->lock)) {
	168	+ node = head->first;
	169	+ if (node) {
	170	+ head->first = node->next;
	171	+ raw_spin_unlock(&head->lock);
	172	+ return node;
	173	+ }
	174	+ raw_spin_unlock(&head->lock);
	175	+ }
	176	+ cpu = cpumask_next(cpu, cpu_possible_mask);
	177	+ if (cpu >= nr_cpu_ids)
	178	+ cpu = 0;
	179	+ if (cpu == orig_cpu)
	180	+ break;
106	181	}
	182	+
	183	+ /* cannot pop from per cpu lists, try extralist */
	184	+ if (!raw_spin_trylock(&s->extralist.lock))
	185	+ return NULL;
	186	+ node = s->extralist.first;
	187	+ if (node)
	188	+ s->extralist.first = node->next;
	189	+ raw_spin_unlock(&s->extralist.lock);
	190	+ return node;
	191	+}
	192	+
	193	+struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)
	194	+{
	195	+ if (in_nmi())
	196	+ return ___pcpu_freelist_pop_nmi(s);
	197	+ return ___pcpu_freelist_pop(s);
107	198	}
108	199
109	200	struct pcpu_freelist_node pcpu_freelist_pop(struct pcpu_freelist s)

kernel/bpf/percpu_freelist.h+1−0

@@ -13,6 +13,7 @@ struct pcpu_freelist_head { · 12 unmodified lines
13	13
14	14	struct pcpu_freelist {
15	15	struct pcpu_freelist_head __percpu *freelist;
	16	+ struct pcpu_freelist_head extralist;
16	17	};
17	18
18	19	struct pcpu_freelist_node {

kernel/bpf/syscall.c+3−1

@@ -4323,8 +4323,10 @@ static int bpf_prog_bind_map(union bpf_attr *attr) · 4322 unmodified lines
4323	4323	used_maps_old = prog->aux->used_maps;
4324	4324
4325	4325	for (i = 0; i < prog->aux->used_map_cnt; i++)
4326		− if (used_maps_old[i] == map)
	4326	+ if (used_maps_old[i] == map) {
	4327	+ bpf_map_put(map);
4327	4328	goto out_unlock;
	4329	+ }
4328	4330
4329	4331	used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
4330	4332	sizeof(used_maps_new[0]),

kernel/bpf/verifier.c+251−19

@@ -238,6 +238,8 @@ struct bpf_call_arg_meta { · 237 unmodified lines
238	238	u64 msize_max_value;
239	239	int ref_obj_id;
240	240	int func_id;
	241	+ u32 btf_id;
	242	+ u32 ret_btf_id;
241	243	};
242	244
243	245	struct btf *btf_vmlinux;
@@ -517,6 +519,7 @@ static const char * const reg_type_str[] = { · 273 unmodified lines
517	519	[PTR_TO_XDP_SOCK] = "xdp_sock",
518	520	[PTR_TO_BTF_ID] = "ptr_",
519	521	[PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
	522	+ [PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
520	523	[PTR_TO_MEM] = "mem",
521	524	[PTR_TO_MEM_OR_NULL] = "mem_or_null",
522	525	[PTR_TO_RDONLY_BUF] = "rdonly_buf",
@@ -583,7 +586,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 60 unmodified lines
583	586	/* reg->off should be 0 for SCALAR_VALUE */
584	587	verbose(env, "%lld", reg->var_off.value + reg->off);
585	588	} else {
586		− if (t == PTR_TO_BTF_ID \|\| t == PTR_TO_BTF_ID_OR_NULL)
	589	+ if (t == PTR_TO_BTF_ID \|\|
	590	+ t == PTR_TO_BTF_ID_OR_NULL \|\|
	591	+ t == PTR_TO_PERCPU_BTF_ID)
587	592	verbose(env, "%s", kernel_type_name(reg->btf_id));
588	593	verbose(env, "(id=%d", reg->id);
589	594	if (reg_type_may_be_refcounted_or_null(t))
@@ -2204,6 +2209,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 1614 unmodified lines
2204	2209	case PTR_TO_RDONLY_BUF_OR_NULL:
2205	2210	case PTR_TO_RDWR_BUF:
2206	2211	case PTR_TO_RDWR_BUF_OR_NULL:
	2212	+ case PTR_TO_PERCPU_BTF_ID:
2207	2213	return true;
2208	2214	default:
2209	2215	return false;
@@ -2221,6 +2227,20 @@ static bool register_is_const(struct bpf_reg_state *reg) · 11 unmodified lines
2221	2227	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
2222	2228	}
2223	2229
	2230	+static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
	2231	+{
	2232	+ return tnum_is_unknown(reg->var_off) &&
	2233	+ reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
	2234	+ reg->umin_value == 0 && reg->umax_value == U64_MAX &&
	2235	+ reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
	2236	+ reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
	2237	+}
	2238	+
	2239	+static bool register_is_bounded(struct bpf_reg_state *reg)
	2240	+{
	2241	+ return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
	2242	+}
	2243	+
2224	2244	static bool __is_pointer_value(bool allow_ptr_leaks,
2225	2245	const struct bpf_reg_state *reg)
2226	2246	{
@@ -2272,7 +2292,7 @@ static int check_stack_write(struct bpf_verifier_env *env, · 45 unmodified lines
2272	2292	if (value_regno >= 0)
2273	2293	reg = &cur->regs[value_regno];
2274	2294
2275		− if (reg && size == BPF_REG_SIZE && register_is_const(reg) &&
	2295	+ if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&
2276	2296	!register_is_null(reg) && env->bpf_capable) {
2277	2297	if (dst_reg != BPF_REG_FP) {
2278	2298	/* The backtracking logic can only recognize explicit
@@ -2667,7 +2687,7 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, · 388 unmodified lines
2667	2687	case BPF_PROG_TYPE_CGROUP_SKB:
2668	2688	if (t == BPF_WRITE)
2669	2689	return false;
2670		− /* fallthrough */
	2690	+ fallthrough;
2671	2691
2672	2692	/* Program types with direct read + write access go here! */
2673	2693	case BPF_PROG_TYPE_SCHED_CLS:
@@ -3978,6 +3998,7 @@ static const struct bpf_reg_types sock_types = { · 1304 unmodified lines
3978	3998	},
3979	3999	};
3980	4000
	4001	+#ifdef CONFIG_NET
3981	4002	static const struct bpf_reg_types btf_id_sock_common_types = {
3982	4003	.types = {
3983	4004	PTR_TO_SOCK_COMMON,
@@ -3988,6 +4009,7 @@ static const struct bpf_reg_types btf_id_sock_common_types = { · 4 unmodified lines
3988	4009	},
3989	4010	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
3990	4011	};
	4012	+#endif
3991	4013
3992	4014	static const struct bpf_reg_types mem_types = {
3993	4015	.types = {
@@ -4017,6 +4039,7 @@ static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; · 23 unmodified lines
4017	4039	static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
4018	4040	static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
4019	4041	static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
	4042	+static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } };
4020	4043
4021	4044	static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
4022	4045	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
@@ -4030,7 +4053,9 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 7 unmodified lines
4030	4053	[ARG_PTR_TO_CTX] = &context_types,
4031	4054	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,
4032	4055	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
	4056	+#ifdef CONFIG_NET
4033	4057	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
	4058	+#endif
4034	4059	[ARG_PTR_TO_SOCKET] = &fullsock_types,
4035	4060	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
4036	4061	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
@@ -4042,6 +4067,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 5 unmodified lines
4042	4067	[ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
4043	4068	[ARG_PTR_TO_INT] = &int_ptr_types,
4044	4069	[ARG_PTR_TO_LONG] = &int_ptr_types,
	4070	+ [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
4045	4071	};
4046	4072
4047	4073	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
@@ -4205,6 +4231,12 @@ skip_type_check: · 157 unmodified lines
4205	4231	err = check_helper_mem_access(env, regno,
4206	4232	meta->map_ptr->value_size, false,
4207	4233	meta);
	4234	+ } else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) {
	4235	+ if (!reg->btf_id) {
	4236	+ verbose(env, "Helper has invalid btf_id in R%d\n", regno);
	4237	+ return -EACCES;
	4238	+ }
	4239	+ meta->ret_btf_id = reg->btf_id;
4208	4240	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
4209	4241	if (meta->func_id == BPF_FUNC_spin_lock) {
4210	4242	if (process_spin_lock(env, regno, true))
@@ -5114,6 +5146,35 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn · 903 unmodified lines
5114	5146	regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
5115	5147	regs[BPF_REG_0].id = ++env->id_gen;
5116	5148	regs[BPF_REG_0].mem_size = meta.mem_size;
	5149	+ } else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL \|\|
	5150	+ fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
	5151	+ const struct btf_type *t;
	5152	+
	5153	+ mark_reg_known_zero(env, regs, BPF_REG_0);
	5154	+ t = btf_type_skip_modifiers(btf_vmlinux, meta.ret_btf_id, NULL);
	5155	+ if (!btf_type_is_struct(t)) {
	5156	+ u32 tsize;
	5157	+ const struct btf_type *ret;
	5158	+ const char *tname;
	5159	+
	5160	+ /* resolve the type size of ksym. */
	5161	+ ret = btf_resolve_size(btf_vmlinux, t, &tsize);
	5162	+ if (IS_ERR(ret)) {
	5163	+ tname = btf_name_by_offset(btf_vmlinux, t->name_off);
	5164	+ verbose(env, "unable to resolve the size of type '%s': %ld\n",
	5165	+ tname, PTR_ERR(ret));
	5166	+ return -EINVAL;
	5167	+ }
	5168	+ regs[BPF_REG_0].type =
	5169	+ fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
	5170	+ PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
	5171	+ regs[BPF_REG_0].mem_size = tsize;
	5172	+ } else {
	5173	+ regs[BPF_REG_0].type =
	5174	+ fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
	5175	+ PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
	5176	+ regs[BPF_REG_0].btf_id = meta.ret_btf_id;
	5177	+ }
5117	5178	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {
5118	5179	int ret_btf_id;
5119	5180
@@ -5432,7 +5493,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 312 unmodified lines
5432	5493	/* smin_val represents the known value */
5433	5494	if (known && smin_val == 0 && opcode == BPF_ADD)
5434	5495	break;
5435		− /* fall-through */
	5496	+ fallthrough;
5436	5497	case PTR_TO_PACKET_END:
5437	5498	case PTR_TO_SOCKET:
5438	5499	case PTR_TO_SOCKET_OR_NULL:
@@ -6389,6 +6450,11 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 950 unmodified lines
6389	6450	src_reg = NULL;
6390	6451	if (dst_reg->type != SCALAR_VALUE)
6391	6452	ptr_reg = dst_reg;
	6453	+ else
	6454	+ /* Make sure ID is cleared otherwise dst_reg min/max could be
	6455	+ * incorrectly propagated into other registers by find_equal_scalars()
	6456	+ */
	6457	+ dst_reg->id = 0;
6392	6458	if (BPF_SRC(insn->code) == BPF_X) {
6393	6459	src_reg = &regs[insn->src_reg];
6394	6460	if (src_reg->type != SCALAR_VALUE) {
@@ -6522,6 +6588,12 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 127 unmodified lines
6522	6588	/* case: R1 = R2
6523	6589	* copy register state to dest reg
6524	6590	*/
	6591	+ if (src_reg->type == SCALAR_VALUE && !src_reg->id)
	6592	+ /* Assign src and dst registers the same ID
	6593	+ * that will be used by find_equal_scalars()
	6594	+ * to propagate min/max range.
	6595	+ */
	6596	+ src_reg->id = ++env->id_gen;
6525	6597	dst_reg = src_reg;
6526	6598	dst_reg->live \|= REG_LIVE_WRITTEN;
6527	6599	dst_reg->subreg_def = DEF_NOT_SUBREG;
@@ -6534,6 +6606,11 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 6 unmodified lines
6534	6606	return -EACCES;
6535	6607	} else if (src_reg->type == SCALAR_VALUE) {
6536	6608	dst_reg = src_reg;
	6609	+ /* Make sure ID is cleared otherwise
	6610	+ * dst_reg min/max could be incorrectly
	6611	+ * propagated into src_reg by find_equal_scalars()
	6612	+ */
	6613	+ dst_reg->id = 0;
6537	6614	dst_reg->live \|= REG_LIVE_WRITTEN;
6538	6615	dst_reg->subreg_def = env->insn_idx + 1;
6539	6616	} else {
@@ -7322,6 +7399,30 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn, · 782 unmodified lines
7322	7399	return true;
7323	7400	}
7324	7401
	7402	+static void find_equal_scalars(struct bpf_verifier_state *vstate,
	7403	+ struct bpf_reg_state *known_reg)
	7404	+{
	7405	+ struct bpf_func_state *state;
	7406	+ struct bpf_reg_state *reg;
	7407	+ int i, j;
	7408	+
	7409	+ for (i = 0; i <= vstate->curframe; i++) {
	7410	+ state = vstate->frame[i];
	7411	+ for (j = 0; j < MAX_BPF_REG; j++) {
	7412	+ reg = &state->regs[j];
	7413	+ if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
	7414	+ reg = known_reg;
	7415	+ }
	7416	+
	7417	+ bpf_for_each_spilled_reg(j, state, reg) {
	7418	+ if (!reg)
	7419	+ continue;
	7420	+ if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
	7421	+ reg = known_reg;
	7422	+ }
	7423	+ }
	7424	+}
	7425	+
7325	7426	static int check_cond_jmp_op(struct bpf_verifier_env *env,
7326	7427	struct bpf_insn insn, int insn_idx)
7327	7428	{
@@ -7450,6 +7551,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 122 unmodified lines
7450	7551	reg_combine_min_max(&other_branch_regs[insn->src_reg],
7451	7552	&other_branch_regs[insn->dst_reg],
7452	7553	src_reg, dst_reg, opcode);
	7554	+ if (src_reg->id) {
	7555	+ find_equal_scalars(this_branch, src_reg);
	7556	+ find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
	7557	+ }
	7558	+
7453	7559	}
7454	7560	} else if (dst_reg->type == SCALAR_VALUE) {
7455	7561	reg_set_min_max(&other_branch_regs[insn->dst_reg],
@@ -7457,6 +7563,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 1 unmodified line
7457	7563	opcode, is_jmp32);
7458	7564	}
7459	7565
	7566	+ if (dst_reg->type == SCALAR_VALUE && dst_reg->id) {
	7567	+ find_equal_scalars(this_branch, dst_reg);
	7568	+ find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
	7569	+ }
	7570	+
7460	7571	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
7461	7572	* NOTE: these optimizations below are related with pointer comparison
7462	7573	* which will never be JMP32.
@@ -7488,6 +7599,7 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 25 unmodified lines
7488	7599	{
7489	7600	struct bpf_insn_aux_data *aux = cur_aux(env);
7490	7601	struct bpf_reg_state *regs = cur_regs(env);
	7602	+ struct bpf_reg_state *dst_reg;
7491	7603	struct bpf_map *map;
7492	7604	int err;
7493	7605
@@ -7504,25 +7616,45 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 10 unmodified lines
7504	7616	if (err)
7505	7617	return err;
7506	7618
	7619	+ dst_reg = &regs[insn->dst_reg];
7507	7620	if (insn->src_reg == 0) {
7508	7621	u64 imm = ((u64)(insn + 1)->imm << 32) \| (u32)insn->imm;
7509	7622
7510		− regs[insn->dst_reg].type = SCALAR_VALUE;
	7623	+ dst_reg->type = SCALAR_VALUE;
7511	7624	__mark_reg_known(&regs[insn->dst_reg], imm);
7512	7625	return 0;
7513	7626	}
7514	7627
	7628	+ if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
	7629	+ mark_reg_known_zero(env, regs, insn->dst_reg);
	7630	+
	7631	+ dst_reg->type = aux->btf_var.reg_type;
	7632	+ switch (dst_reg->type) {
	7633	+ case PTR_TO_MEM:
	7634	+ dst_reg->mem_size = aux->btf_var.mem_size;
	7635	+ break;
	7636	+ case PTR_TO_BTF_ID:
	7637	+ case PTR_TO_PERCPU_BTF_ID:
	7638	+ dst_reg->btf_id = aux->btf_var.btf_id;
	7639	+ break;
	7640	+ default:
	7641	+ verbose(env, "bpf verifier is misconfigured\n");
	7642	+ return -EFAULT;
	7643	+ }
	7644	+ return 0;
	7645	+ }
	7646	+
7515	7647	map = env->used_maps[aux->map_index];
7516	7648	mark_reg_known_zero(env, regs, insn->dst_reg);
7517		− regs[insn->dst_reg].map_ptr = map;
	7649	+ dst_reg->map_ptr = map;
7518	7650
7519	7651	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {
7520		− regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;
7521		− regs[insn->dst_reg].off = aux->map_off;
	7652	+ dst_reg->type = PTR_TO_MAP_VALUE;
	7653	+ dst_reg->off = aux->map_off;
7522	7654	if (map_value_has_spin_lock(map))
7523		− regs[insn->dst_reg].id = ++env->id_gen;
	7655	+ dst_reg->id = ++env->id_gen;
7524	7656	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {
7525		− regs[insn->dst_reg].type = CONST_PTR_TO_MAP;
	7657	+ dst_reg->type = CONST_PTR_TO_MAP;
7526	7658	} else {
7527	7659	verbose(env, "bpf verifier is misconfigured\n");
7528	7660	return -EINVAL;
@@ -9424,6 +9556,92 @@ process_bpf_exit: · 1895 unmodified lines
9424	9556	return 0;
9425	9557	}
9426	9558
	9559	+/* replace pseudo btf_id with kernel symbol address */
	9560	+static int check_pseudo_btf_id(struct bpf_verifier_env *env,
	9561	+ struct bpf_insn *insn,
	9562	+ struct bpf_insn_aux_data *aux)
	9563	+{
	9564	+ u32 datasec_id, type, id = insn->imm;
	9565	+ const struct btf_var_secinfo *vsi;
	9566	+ const struct btf_type *datasec;
	9567	+ const struct btf_type *t;
	9568	+ const char *sym_name;
	9569	+ bool percpu = false;
	9570	+ u64 addr;
	9571	+ int i;
	9572	+
	9573	+ if (!btf_vmlinux) {
	9574	+ verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
	9575	+ return -EINVAL;
	9576	+ }
	9577	+
	9578	+ if (insn[1].imm != 0) {
	9579	+ verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n");
	9580	+ return -EINVAL;
	9581	+ }
	9582	+
	9583	+ t = btf_type_by_id(btf_vmlinux, id);
	9584	+ if (!t) {
	9585	+ verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
	9586	+ return -ENOENT;
	9587	+ }
	9588	+
	9589	+ if (!btf_type_is_var(t)) {
	9590	+ verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n",
	9591	+ id);
	9592	+ return -EINVAL;
	9593	+ }
	9594	+
	9595	+ sym_name = btf_name_by_offset(btf_vmlinux, t->name_off);
	9596	+ addr = kallsyms_lookup_name(sym_name);
	9597	+ if (!addr) {
	9598	+ verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
	9599	+ sym_name);
	9600	+ return -ENOENT;
	9601	+ }
	9602	+
	9603	+ datasec_id = btf_find_by_name_kind(btf_vmlinux, ".data..percpu",
	9604	+ BTF_KIND_DATASEC);
	9605	+ if (datasec_id > 0) {
	9606	+ datasec = btf_type_by_id(btf_vmlinux, datasec_id);
	9607	+ for_each_vsi(i, datasec, vsi) {
	9608	+ if (vsi->type == id) {
	9609	+ percpu = true;
	9610	+ break;
	9611	+ }
	9612	+ }
	9613	+ }
	9614	+
	9615	+ insn[0].imm = (u32)addr;
	9616	+ insn[1].imm = addr >> 32;
	9617	+
	9618	+ type = t->type;
	9619	+ t = btf_type_skip_modifiers(btf_vmlinux, type, NULL);
	9620	+ if (percpu) {
	9621	+ aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
	9622	+ aux->btf_var.btf_id = type;
	9623	+ } else if (!btf_type_is_struct(t)) {
	9624	+ const struct btf_type *ret;
	9625	+ const char *tname;
	9626	+ u32 tsize;
	9627	+
	9628	+ /* resolve the type size of ksym. */
	9629	+ ret = btf_resolve_size(btf_vmlinux, t, &tsize);
	9630	+ if (IS_ERR(ret)) {
	9631	+ tname = btf_name_by_offset(btf_vmlinux, t->name_off);
	9632	+ verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
	9633	+ tname, PTR_ERR(ret));
	9634	+ return -EINVAL;
	9635	+ }
	9636	+ aux->btf_var.reg_type = PTR_TO_MEM;
	9637	+ aux->btf_var.mem_size = tsize;
	9638	+ } else {
	9639	+ aux->btf_var.reg_type = PTR_TO_BTF_ID;
	9640	+ aux->btf_var.btf_id = type;
	9641	+ }
	9642	+ return 0;
	9643	+}
	9644	+
9427	9645	static int check_map_prealloc(struct bpf_map *map)
9428	9646	{
9429	9647	return (map->map_type != BPF_MAP_TYPE_HASH &&
@@ -9534,10 +9752,14 @@ static bool bpf_map_is_cgroup_storage(struct bpf_map *map) · 104 unmodified lines
9534	9752	map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
9535	9753	}
9536	9754
9537		−/* look for pseudo eBPF instructions that access map FDs and
9538		− * replace them with actual map pointers
	9755	+/* find and rewrite pseudo imm in ld_imm64 instructions:
	9756	+ *
	9757	+ * 1. if it accesses map FD, replace it with actual map pointer.
	9758	+ * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
	9759	+ *
	9760	+ * NOTE: btf_vmlinux is required for converting pseudo btf_id.
9539	9761	*/
9540		−static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)
	9762	+static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
9541	9763	{
9542	9764	struct bpf_insn *insn = env->prog->insnsi;
9543	9765	int insn_cnt = env->prog->len;
@@ -9578,6 +9800,14 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) · 34 unmodified lines
9578	9800	/* valid generic load 64-bit imm */
9579	9801	goto next_insn;
9580	9802
	9803	+ if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
	9804	+ aux = &env->insn_aux_data[i];
	9805	+ err = check_pseudo_btf_id(env, insn, aux);
	9806	+ if (err)
	9807	+ return err;
	9808	+ goto next_insn;
	9809	+ }
	9810	+
9581	9811	/* In final convert_pseudo_ld_imm64() step, this is
9582	9812	* converted into regular 64-bit imm load insn.
9583	9813	*/
@@ -10819,7 +11049,9 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 1235 unmodified lines
10819	11049	if (insn->imm == BPF_FUNC_map_lookup_elem &&
10820	11050	ops->map_gen_lookup) {
10821	11051	cnt = ops->map_gen_lookup(map_ptr, insn_buf);
10822		− if (cnt == 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {
	11052	+ if (cnt == -EOPNOTSUPP)
	11053	+ goto patch_map_ops_generic;
	11054	+ if (cnt <= 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {
10823	11055	verbose(env, "bpf verifier is misconfigured\n");
10824	11056	return -EINVAL;
10825	11057	}
@@ -10849,7 +11081,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 23 unmodified lines
10849	11081	(int ()(struct bpf_map map, void *value))NULL));
10850	11082	BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
10851	11083	(int ()(struct bpf_map map, void *value))NULL));
10852		−
	11084	+patch_map_ops_generic:
10853	11085	switch (insn->imm) {
10854	11086	case BPF_FUNC_map_lookup_elem:
10855	11087	insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
@@ -11633,10 +11865,6 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 777 unmodified lines
11633	11865	if (is_priv)
11634	11866	env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
11635	11867
11636		− ret = replace_map_fd_with_map_ptr(env);
11637		− if (ret < 0)
11638		− goto skip_full_check;
11639		−
11640	11868	if (bpf_prog_is_dev_bound(env->prog->aux)) {
11641	11869	ret = bpf_prog_offload_verifier_prep(env->prog);
11642	11870	if (ret)
@@ -11662,6 +11890,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 19 unmodified lines
11662	11890	if (ret)
11663	11891	goto skip_full_check;
11664	11892
	11893	+ ret = resolve_pseudo_ldimm64(env);
	11894	+ if (ret < 0)
	11895	+ goto skip_full_check;
	11896	+
11665	11897	ret = check_cfg(env);
11666	11898	if (ret < 0)
11667	11899	goto skip_full_check;

kernel/trace/bpf_trace.c+6−0

@@ -1327,6 +1327,10 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 1326 unmodified lines
1327	1327	return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
1328	1328	case BPF_FUNC_snprintf_btf:
1329	1329	return &bpf_snprintf_btf_proto;
	1330	+ case BPF_FUNC_bpf_per_cpu_ptr:
	1331	+ return &bpf_per_cpu_ptr_proto;
	1332	+ case BPF_FUNC_bpf_this_cpu_ptr:
	1333	+ return &bpf_this_cpu_ptr_proto;
1330	1334	default:
1331	1335	return NULL;
1332	1336	}
@@ -1776,7 +1780,9 @@ const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { · 443 unmodified lines
1776	1780	};
1777	1781
1778	1782	const struct bpf_prog_ops raw_tracepoint_prog_ops = {
	1783	+#ifdef CONFIG_NET
1779	1784	.test_run = bpf_prog_test_run_raw_tp,
	1785	+#endif
1780	1786	};
1781	1787
1782	1788	const struct bpf_verifier_ops tracing_verifier_ops = {

net/core/dev.c+12−3

@@ -4930,7 +4930,7 @@ EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); · 4929 unmodified lines
4930	4930
4931	4931	static inline struct sk_buff *
4932	4932	sch_handle_ingress(struct sk_buff skb, struct packet_type pt_prev, int ret,
4933		− struct net_device *orig_dev)
	4933	+ struct net_device orig_dev, bool another)
4934	4934	{
4935	4935	#ifdef CONFIG_NET_CLS_ACT
4936	4936	struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
@@ -4974,7 +4974,11 @@ sch_handle_ingress(struct sk_buff skb, struct packet_type pt_prev, int ret, · 37 unmodified lines
4974	4974	* redirecting to another netdev
4975	4975	*/
4976	4976	__skb_push(skb, skb->mac_len);
4977		− skb_do_redirect(skb);
	4977	+ if (skb_do_redirect(skb) == -EAGAIN) {
	4978	+ __skb_pull(skb, skb->mac_len);
	4979	+ *another = true;
	4980	+ break;
	4981	+ }
4978	4982	return NULL;
4979	4983	case TC_ACT_CONSUMED:
4980	4984	return NULL;
@@ -5163,7 +5167,12 @@ another_round: · 182 unmodified lines
5163	5167	skip_taps:
5164	5168	#ifdef CONFIG_NET_INGRESS
5165	5169	if (static_branch_unlikely(&ingress_needed_key)) {
5166		− skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
	5170	+ bool another = false;
	5171	+
	5172	+ skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
	5173	+ &another);
	5174	+ if (another)
	5175	+ goto another_round;
5167	5176	if (!skb)
5168	5177	goto out;
5169	5178

net/core/filter.c+99−8

@@ -76,6 +76,7 @@ · 75 unmodified lines
76	76	#include <net/bpf_sk_storage.h>
77	77	#include <net/transp_v6.h>
78	78	#include <linux/btf_ids.h>
	79	+#include <net/tls.h>
79	80
80	81	static const struct bpf_func_proto *
81	82	bpf_sk_base_func_proto(enum bpf_func_id func_id);
@@ -2379,8 +2380,9 @@ out: · 2297 unmodified lines
2379	2380
2380	2381	/* Internal, non-exposed redirect flags. */
2381	2382	enum {
2382		− BPF_F_NEIGH = (1ULL << 1),
2383		−#define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH)
	2383	+ BPF_F_NEIGH = (1ULL << 1),
	2384	+ BPF_F_PEER = (1ULL << 2),
	2385	+#define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH \| BPF_F_PEER)
2384	2386	};
2385	2387
2386	2388	BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
@@ -2429,19 +2431,35 @@ EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info); · 42 unmodified lines
2429	2431	int skb_do_redirect(struct sk_buff *skb)
2430	2432	{
2431	2433	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
	2434	+ struct net *net = dev_net(skb->dev);
2432	2435	struct net_device *dev;
2433	2436	u32 flags = ri->flags;
2434	2437
2435		− dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->tgt_index);
	2438	+ dev = dev_get_by_index_rcu(net, ri->tgt_index);
2436	2439	ri->tgt_index = 0;
2437		− if (unlikely(!dev)) {
2438		− kfree_skb(skb);
2439		− return -EINVAL;
	2440	+ ri->flags = 0;
	2441	+ if (unlikely(!dev))
	2442	+ goto out_drop;
	2443	+ if (flags & BPF_F_PEER) {
	2444	+ const struct net_device_ops *ops = dev->netdev_ops;
	2445	+
	2446	+ if (unlikely(!ops->ndo_get_peer_dev \|\|
	2447	+ !skb_at_tc_ingress(skb)))
	2448	+ goto out_drop;
	2449	+ dev = ops->ndo_get_peer_dev(dev);
	2450	+ if (unlikely(!dev \|\|
	2451	+ !is_skb_forwardable(dev, skb) \|\|
	2452	+ net_eq(net, dev_net(dev))))
	2453	+ goto out_drop;
	2454	+ skb->dev = dev;
	2455	+ return -EAGAIN;
2440	2456	}
2441		−
2442	2457	return flags & BPF_F_NEIGH ?
2443	2458	__bpf_redirect_neigh(skb, dev) :
2444	2459	__bpf_redirect(skb, dev, flags);
	2460	+out_drop:
	2461	+ kfree_skb(skb);
	2462	+ return -EINVAL;
2445	2463	}
2446	2464
2447	2465	BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
@@ -2465,6 +2483,27 @@ static const struct bpf_func_proto bpf_redirect_proto = { · 17 unmodified lines
2465	2483	.arg2_type = ARG_ANYTHING,
2466	2484	};
2467	2485
	2486	+BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
	2487	+{
	2488	+ struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
	2489	+
	2490	+ if (unlikely(flags))
	2491	+ return TC_ACT_SHOT;
	2492	+
	2493	+ ri->flags = BPF_F_PEER;
	2494	+ ri->tgt_index = ifindex;
	2495	+
	2496	+ return TC_ACT_REDIRECT;
	2497	+}
	2498	+
	2499	+static const struct bpf_func_proto bpf_redirect_peer_proto = {
	2500	+ .func = bpf_redirect_peer,
	2501	+ .gpl_only = false,
	2502	+ .ret_type = RET_INTEGER,
	2503	+ .arg1_type = ARG_ANYTHING,
	2504	+ .arg2_type = ARG_ANYTHING,
	2505	+};
	2506	+
2468	2507	BPF_CALL_2(bpf_redirect_neigh, u32, ifindex, u64, flags)
2469	2508	{
2470	2509	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
@@ -3479,6 +3518,48 @@ static u32 __bpf_skb_max_len(const struct sk_buff *skb) · 1008 unmodified lines
3479	3518	SKB_MAX_ALLOC;
3480	3519	}
3481	3520
	3521	+BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
	3522	+ u32, mode, u64, flags)
	3523	+{
	3524	+ u32 len_diff_abs = abs(len_diff);
	3525	+ bool shrink = len_diff < 0;
	3526	+ int ret = 0;
	3527	+
	3528	+ if (unlikely(flags \|\| mode))
	3529	+ return -EINVAL;
	3530	+ if (unlikely(len_diff_abs > 0xfffU))
	3531	+ return -EFAULT;
	3532	+
	3533	+ if (!shrink) {
	3534	+ ret = skb_cow(skb, len_diff);
	3535	+ if (unlikely(ret < 0))
	3536	+ return ret;
	3537	+ __skb_push(skb, len_diff_abs);
	3538	+ memset(skb->data, 0, len_diff_abs);
	3539	+ } else {
	3540	+ if (unlikely(!pskb_may_pull(skb, len_diff_abs)))
	3541	+ return -ENOMEM;
	3542	+ __skb_pull(skb, len_diff_abs);
	3543	+ }
	3544	+ bpf_compute_data_end_sk_skb(skb);
	3545	+ if (tls_sw_has_ctx_rx(skb->sk)) {
	3546	+ struct strp_msg *rxm = strp_msg(skb);
	3547	+
	3548	+ rxm->full_len += len_diff;
	3549	+ }
	3550	+ return ret;
	3551	+}
	3552	+
	3553	+static const struct bpf_func_proto sk_skb_adjust_room_proto = {
	3554	+ .func = sk_skb_adjust_room,
	3555	+ .gpl_only = false,
	3556	+ .ret_type = RET_INTEGER,
	3557	+ .arg1_type = ARG_PTR_TO_CTX,
	3558	+ .arg2_type = ARG_ANYTHING,
	3559	+ .arg3_type = ARG_ANYTHING,
	3560	+ .arg4_type = ARG_ANYTHING,
	3561	+};
	3562	+
3482	3563	BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3483	3564	u32, mode, u64, flags)
3484	3565	{
@@ -4784,6 +4865,10 @@ static int _bpf_setsockopt(struct sock *sk, int level, int optname, · 1299 unmodified lines
4784	4865	else
4785	4866	icsk->icsk_user_timeout = val;
4786	4867	break;
	4868	+ case TCP_NOTSENT_LOWAT:
	4869	+ tp->notsent_lowat = val;
	4870	+ sk->sk_write_space(sk);
	4871	+ break;
4787	4872	default:
4788	4873	ret = -EINVAL;
4789	4874	}
@@ -5149,7 +5234,6 @@ static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params, · 359 unmodified lines
5149	5234	memcpy(params->smac, dev->dev_addr, ETH_ALEN);
5150	5235	params->h_vlan_TCI = 0;
5151	5236	params->h_vlan_proto = 0;
5152		− params->ifindex = dev->ifindex;
5153	5237
5154	5238	return 0;
5155	5239	}
@@ -5246,6 +5330,7 @@ static int bpf_ipv4_fib_lookup(struct net net, struct bpf_fib_lookup params, · 90 unmodified lines
5246	5330	dev = nhc->nhc_dev;
5247	5331
5248	5332	params->rt_metric = res.fi->fib_priority;
	5333	+ params->ifindex = dev->ifindex;
5249	5334
5250	5335	/* xdp and cls_bpf programs are run in RCU-bh so
5251	5336	* rcu_read_lock_bh is not needed here
@@ -5371,6 +5456,7 @@ static int bpf_ipv6_fib_lookup(struct net net, struct bpf_fib_lookup params, · 119 unmodified lines
5371	5456
5372	5457	dev = res.nh->fib_nh_dev;
5373	5458	params->rt_metric = res.f6i->fib6_metric;
	5459	+ params->ifindex = dev->ifindex;
5374	5460
5375	5461	/* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is
5376	5462	* not needed here.
@@ -6745,6 +6831,7 @@ bool bpf_helper_changes_pkt_data(void *func) · 1368 unmodified lines
6745	6831	func == bpf_skb_change_tail \|\|
6746	6832	func == sk_skb_change_tail \|\|
6747	6833	func == bpf_skb_adjust_room \|\|
	6834	+ func == sk_skb_adjust_room \|\|
6748	6835	func == bpf_skb_pull_data \|\|
6749	6836	func == sk_skb_pull_data \|\|
6750	6837	func == bpf_clone_redirect \|\|
@@ -7005,6 +7092,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 254 unmodified lines
7005	7092	return &bpf_redirect_proto;
7006	7093	case BPF_FUNC_redirect_neigh:
7007	7094	return &bpf_redirect_neigh_proto;
	7095	+ case BPF_FUNC_redirect_peer:
	7096	+ return &bpf_redirect_peer_proto;
7008	7097	case BPF_FUNC_get_route_realm:
7009	7098	return &bpf_get_route_realm_proto;
7010	7099	case BPF_FUNC_get_hash_recalc:
@@ -7218,6 +7307,8 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 207 unmodified lines
7218	7307	return &sk_skb_change_tail_proto;
7219	7308	case BPF_FUNC_skb_change_head:
7220	7309	return &sk_skb_change_head_proto;
	7310	+ case BPF_FUNC_skb_adjust_room:
	7311	+ return &sk_skb_adjust_room_proto;
7221	7312	case BPF_FUNC_get_socket_cookie:
7222	7313	return &bpf_get_socket_cookie_proto;
7223	7314	case BPF_FUNC_get_socket_uid:

net/core/skmsg.c+123−38

@@ -433,10 +433,12 @@ static int sk_psock_skb_ingress(struct sk_psock psock, struct sk_buff skb) · 432 unmodified lines
433	433	static int sk_psock_handle_skb(struct sk_psock psock, struct sk_buff skb,
434	434	u32 off, u32 len, bool ingress)
435	435	{
436		− if (ingress)
437		− return sk_psock_skb_ingress(psock, skb);
438		− else
	436	+ if (!ingress) {
	437	+ if (!sock_writeable(psock->sk))
	438	+ return -EAGAIN;
439	439	return skb_send_sock_locked(psock->sk, skb, off, len);
	440	+ }
	441	+ return sk_psock_skb_ingress(psock, skb);
440	442	}
441	443
442	444	static void sk_psock_backlog(struct work_struct *work)
@@ -625,6 +627,8 @@ void sk_psock_drop(struct sock sk, struct sk_psock psock) · 182 unmodified lines
625	627	rcu_assign_sk_user_data(sk, NULL);
626	628	if (psock->progs.skb_parser)
627	629	sk_psock_stop_strp(sk, psock);
	630	+ else if (psock->progs.skb_verdict)
	631	+ sk_psock_stop_verdict(sk, psock);
628	632	write_unlock_bh(&sk->sk_callback_lock);
629	633	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
630	634
@@ -682,19 +686,8 @@ EXPORT_SYMBOL_GPL(sk_psock_msg_verdict); · 51 unmodified lines
682	686	static int sk_psock_bpf_run(struct sk_psock psock, struct bpf_prog prog,
683	687	struct sk_buff *skb)
684	688	{
685		− int ret;
686		−
687		− skb->sk = psock->sk;
688	689	bpf_compute_data_end_sk_skb(skb);
689		− ret = bpf_prog_run_pin_on_cpu(prog, skb);
690		− /* strparser clones the skb before handing it to a upper layer,
691		− * meaning skb_orphan has been called. We NULL sk on the way out
692		− * to ensure we don't trigger a BUG_ON() in skb/sk operations
693		− * later and because we are not charging the memory of this skb
694		− * to any socket yet.
695		− */
696		− skb->sk = NULL;
697		− return ret;
	690	+ return bpf_prog_run_pin_on_cpu(prog, skb);
698	691	}
699	692
700	693	static struct sk_psock sk_psock_from_strp(struct strparser strp)
@@ -709,38 +702,35 @@ static void sk_psock_skb_redirect(struct sk_buff *skb) · 8 unmodified lines
709	702	{
710	703	struct sk_psock *psock_other;
711	704	struct sock *sk_other;
712		− bool ingress;
713	705
714	706	sk_other = tcp_skb_bpf_redirect_fetch(skb);
	707	+ /* This error is a buggy BPF program, it returned a redirect
	708	+ * return code, but then didn't set a redirect interface.
	709	+ */
715	710	if (unlikely(!sk_other)) {
716	711	kfree_skb(skb);
717	712	return;
718	713	}
719	714	psock_other = sk_psock(sk_other);
	715	+ /* This error indicates the socket is being torn down or had another
	716	+ * error that caused the pipe to break. We can't send a packet on
	717	+ * a socket that is in this state so we drop the skb.
	718	+ */
720	719	if (!psock_other \|\| sock_flag(sk_other, SOCK_DEAD) \|\|
721	720	!sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
722	721	kfree_skb(skb);
723	722	return;
724	723	}
725	724
726		− ingress = tcp_skb_bpf_ingress(skb);
727		− if ((!ingress && sock_writeable(sk_other)) \|\|
728		− (ingress &&
729		− atomic_read(&sk_other->sk_rmem_alloc) <=
730		− sk_other->sk_rcvbuf)) {
731		− if (!ingress)
732		− skb_set_owner_w(skb, sk_other);
733		− skb_queue_tail(&psock_other->ingress_skb, skb);
734		− schedule_work(&psock_other->work);
735		− } else {
736		− kfree_skb(skb);
737		− }
	725	+ skb_queue_tail(&psock_other->ingress_skb, skb);
	726	+ schedule_work(&psock_other->work);
738	727	}
739	728
740		−static void sk_psock_tls_verdict_apply(struct sk_buff *skb, int verdict)
	729	+static void sk_psock_tls_verdict_apply(struct sk_buff skb, struct sock sk, int verdict)
741	730	{
742	731	switch (verdict) {
743	732	case __SK_REDIRECT:
	733	+ skb_set_owner_r(skb, sk);
744	734	sk_psock_skb_redirect(skb);
745	735	break;
746	736	case __SK_PASS:
@@ -758,11 +748,17 @@ int sk_psock_tls_strp_read(struct sk_psock psock, struct sk_buff skb) · 11 unmodified lines
758	748	rcu_read_lock();
759	749	prog = READ_ONCE(psock->progs.skb_verdict);
760	750	if (likely(prog)) {
	751	+ /* We skip full set_owner_r here because if we do a SK_PASS
	752	+ * or SK_DROP we can skip skb memory accounting and use the
	753	+ * TLS context.
	754	+ */
	755	+ skb->sk = psock->sk;
761	756	tcp_skb_bpf_redirect_clear(skb);
762	757	ret = sk_psock_bpf_run(psock, prog, skb);
763	758	ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb));
	759	+ skb->sk = NULL;
764	760	}
765		− sk_psock_tls_verdict_apply(skb, ret);
	761	+ sk_psock_tls_verdict_apply(skb, psock->sk, ret);
766	762	rcu_read_unlock();
767	763	return ret;
768	764	}
@@ -771,7 +767,9 @@ EXPORT_SYMBOL_GPL(sk_psock_tls_strp_read); · 2 unmodified lines
771	767	static void sk_psock_verdict_apply(struct sk_psock *psock,
772	768	struct sk_buff *skb, int verdict)
773	769	{
	770	+ struct tcp_skb_cb *tcp;
774	771	struct sock *sk_other;
	772	+ int err = -EIO;
775	773
776	774	switch (verdict) {
777	775	case __SK_PASS:
@@ -780,16 +778,24 @@ static void sk_psock_verdict_apply(struct sk_psock *psock, · 2 unmodified lines
780	778	!sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
781	779	goto out_free;
782	780	}
783		− if (atomic_read(&sk_other->sk_rmem_alloc) <=
784		− sk_other->sk_rcvbuf) {
785		− struct tcp_skb_cb *tcp = TCP_SKB_CB(skb);
786	781
787		− tcp->bpf.flags \|= BPF_F_INGRESS;
	782	+ tcp = TCP_SKB_CB(skb);
	783	+ tcp->bpf.flags \|= BPF_F_INGRESS;
	784	+
	785	+ /* If the queue is empty then we can submit directly
	786	+ * into the msg queue. If its not empty we have to
	787	+ * queue work otherwise we may get OOO data. Otherwise,
	788	+ * if sk_psock_skb_ingress errors will be handled by
	789	+ * retrying later from workqueue.
	790	+ */
	791	+ if (skb_queue_empty(&psock->ingress_skb)) {
	792	+ err = sk_psock_skb_ingress(psock, skb);
	793	+ }
	794	+ if (err < 0) {
788	795	skb_queue_tail(&psock->ingress_skb, skb);
789	796	schedule_work(&psock->work);
790		− break;
791	797	}
792		− goto out_free;
	798	+ break;
793	799	case __SK_REDIRECT:
794	800	sk_psock_skb_redirect(skb);
795	801	break;
@@ -814,9 +820,9 @@ static void sk_psock_strp_read(struct strparser strp, struct sk_buff skb) · 18 unmodified lines
814	820	kfree_skb(skb);
815	821	goto out;
816	822	}
	823	+ skb_set_owner_r(skb, sk);
817	824	prog = READ_ONCE(psock->progs.skb_verdict);
818	825	if (likely(prog)) {
819		− skb_orphan(skb);
820	826	tcp_skb_bpf_redirect_clear(skb);
821	827	ret = sk_psock_bpf_run(psock, prog, skb);
822	828	ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb));
@@ -839,8 +845,11 @@ static int sk_psock_strp_parse(struct strparser strp, struct sk_buff skb) · 16 unmodified lines
839	845
840	846	rcu_read_lock();
841	847	prog = READ_ONCE(psock->progs.skb_parser);
842		− if (likely(prog))
	848	+ if (likely(prog)) {
	849	+ skb->sk = psock->sk;
843	850	ret = sk_psock_bpf_run(psock, prog, skb);
	851	+ skb->sk = NULL;
	852	+ }
844	853	rcu_read_unlock();
845	854	return ret;
846	855	}
@@ -864,6 +873,57 @@ static void sk_psock_strp_data_ready(struct sock *sk) · 17 unmodified lines
864	873	rcu_read_unlock();
865	874	}
866	875
	876	+static int sk_psock_verdict_recv(read_descriptor_t desc, struct sk_buff skb,
	877	+ unsigned int offset, size_t orig_len)
	878	+{
	879	+ struct sock sk = (struct sock )desc->arg.data;
	880	+ struct sk_psock *psock;
	881	+ struct bpf_prog *prog;
	882	+ int ret = __SK_DROP;
	883	+ int len = skb->len;
	884	+
	885	+ /* clone here so sk_eat_skb() in tcp_read_sock does not drop our data */
	886	+ skb = skb_clone(skb, GFP_ATOMIC);
	887	+ if (!skb) {
	888	+ desc->error = -ENOMEM;
	889	+ return 0;
	890	+ }
	891	+
	892	+ rcu_read_lock();
	893	+ psock = sk_psock(sk);
	894	+ if (unlikely(!psock)) {
	895	+ len = 0;
	896	+ kfree_skb(skb);
	897	+ goto out;
	898	+ }
	899	+ skb_set_owner_r(skb, sk);
	900	+ prog = READ_ONCE(psock->progs.skb_verdict);
	901	+ if (likely(prog)) {
	902	+ tcp_skb_bpf_redirect_clear(skb);
	903	+ ret = sk_psock_bpf_run(psock, prog, skb);
	904	+ ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb));
	905	+ }
	906	+ sk_psock_verdict_apply(psock, skb, ret);
	907	+out:
	908	+ rcu_read_unlock();
	909	+ return len;
	910	+}
	911	+
	912	+static void sk_psock_verdict_data_ready(struct sock *sk)
	913	+{
	914	+ struct socket *sock = sk->sk_socket;
	915	+ read_descriptor_t desc;
	916	+
	917	+ if (unlikely(!sock \|\| !sock->ops \|\| !sock->ops->read_sock))
	918	+ return;
	919	+
	920	+ desc.arg.data = sk;
	921	+ desc.error = 0;
	922	+ desc.count = 1;
	923	+
	924	+ sock->ops->read_sock(sk, &desc, sk_psock_verdict_recv);
	925	+}
	926	+
867	927	static void sk_psock_write_space(struct sock *sk)
868	928	{
869	929	struct sk_psock *psock;
@@ -893,6 +953,19 @@ int sk_psock_init_strp(struct sock sk, struct sk_psock psock) · 23 unmodified lines
893	953	return strp_init(&psock->parser.strp, sk, &cb);
894	954	}
895	955
	956	+void sk_psock_start_verdict(struct sock sk, struct sk_psock psock)
	957	+{
	958	+ struct sk_psock_parser *parser = &psock->parser;
	959	+
	960	+ if (parser->enabled)
	961	+ return;
	962	+
	963	+ parser->saved_data_ready = sk->sk_data_ready;
	964	+ sk->sk_data_ready = sk_psock_verdict_data_ready;
	965	+ sk->sk_write_space = sk_psock_write_space;
	966	+ parser->enabled = true;
	967	+}
	968	+
896	969	void sk_psock_start_strp(struct sock sk, struct sk_psock psock)
897	970	{
898	971	struct sk_psock_parser *parser = &psock->parser;
@@ -918,3 +991,15 @@ void sk_psock_stop_strp(struct sock sk, struct sk_psock psock) · 19 unmodified lines
918	991	strp_stop(&parser->strp);
919	992	parser->enabled = false;
920	993	}
	994	+
	995	+void sk_psock_stop_verdict(struct sock sk, struct sk_psock psock)
	996	+{
	997	+ struct sk_psock_parser *parser = &psock->parser;
	998	+
	999	+ if (!parser->enabled)
	1000	+ return;
	1001	+
	1002	+ sk->sk_data_ready = parser->saved_data_ready;
	1003	+ parser->saved_data_ready = NULL;
	1004	+ parser->enabled = false;
	1005	+}

net/core/sock_map.c+23−14

@@ -148,8 +148,8 @@ static void sock_map_add_link(struct sk_psock *psock, · 147 unmodified lines
148	148	static void sock_map_del_link(struct sock *sk,
149	149	struct sk_psock psock, void link_raw)
150	150	{
	151	+ bool strp_stop = false, verdict_stop = false;
151	152	struct sk_psock_link link, tmp;
152		− bool strp_stop = false;
153	153
154	154	spin_lock_bh(&psock->link_lock);
155	155	list_for_each_entry_safe(link, tmp, &psock->link, list) {
@@ -159,14 +159,19 @@ static void sock_map_del_link(struct sock *sk, · 3 unmodified lines
159	159	map);
160	160	if (psock->parser.enabled && stab->progs.skb_parser)
161	161	strp_stop = true;
	162	+ if (psock->parser.enabled && stab->progs.skb_verdict)
	163	+ verdict_stop = true;
162	164	list_del(&link->list);
163	165	sk_psock_free_link(link);
164	166	}
165	167	}
166	168	spin_unlock_bh(&psock->link_lock);
167		− if (strp_stop) {
	169	+ if (strp_stop \|\| verdict_stop) {
168	170	write_lock_bh(&sk->sk_callback_lock);
169		− sk_psock_stop_strp(sk, psock);
	171	+ if (strp_stop)
	172	+ sk_psock_stop_strp(sk, psock);
	173	+ else
	174	+ sk_psock_stop_verdict(sk, psock);
170	175	write_unlock_bh(&sk->sk_callback_lock);
171	176	}
172	177	}
@@ -230,16 +235,16 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 57 unmodified lines
230	235	{
231	236	struct bpf_prog msg_parser, skb_parser, *skb_verdict;
232	237	struct sk_psock *psock;
233		− bool skb_progs;
234	238	int ret;
235	239
236	240	skb_verdict = READ_ONCE(progs->skb_verdict);
237	241	skb_parser = READ_ONCE(progs->skb_parser);
238		− skb_progs = skb_parser && skb_verdict;
239		− if (skb_progs) {
	242	+ if (skb_verdict) {
240	243	skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
241	244	if (IS_ERR(skb_verdict))
242	245	return PTR_ERR(skb_verdict);
	246	+ }
	247	+ if (skb_parser) {
243	248	skb_parser = bpf_prog_inc_not_zero(skb_parser);
244	249	if (IS_ERR(skb_parser)) {
245	250	bpf_prog_put(skb_verdict);
@@ -264,7 +269,8 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 18 unmodified lines
264	269
265	270	if (psock) {
266	271	if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) \|\|
267		− (skb_progs && READ_ONCE(psock->progs.skb_parser))) {
	272	+ (skb_parser && READ_ONCE(psock->progs.skb_parser)) \|\|
	273	+ (skb_verdict && READ_ONCE(psock->progs.skb_verdict))) {
268	274	sk_psock_put(sk, psock);
269	275	ret = -EBUSY;
270	276	goto out_progs;
@@ -285,28 +291,31 @@ static int sock_map_link(struct bpf_map map, struct sk_psock_progs progs, · 14 unmodified lines
285	291	goto out_drop;
286	292
287	293	write_lock_bh(&sk->sk_callback_lock);
288		− if (skb_progs && !psock->parser.enabled) {
	294	+ if (skb_parser && skb_verdict && !psock->parser.enabled) {
289	295	ret = sk_psock_init_strp(sk, psock);
290		− if (ret) {
291		− write_unlock_bh(&sk->sk_callback_lock);
292		− goto out_drop;
293		− }
	296	+ if (ret)
	297	+ goto out_unlock_drop;
294	298	psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
295	299	psock_set_prog(&psock->progs.skb_parser, skb_parser);
296	300	sk_psock_start_strp(sk, psock);
	301	+ } else if (!skb_parser && skb_verdict && !psock->parser.enabled) {
	302	+ psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
	303	+ sk_psock_start_verdict(sk,psock);
297	304	}
298	305	write_unlock_bh(&sk->sk_callback_lock);
299	306	return 0;
	307	+out_unlock_drop:
	308	+ write_unlock_bh(&sk->sk_callback_lock);
300	309	out_drop:
301	310	sk_psock_put(sk, psock);
302	311	out_progs:
303	312	if (msg_parser)
304	313	bpf_prog_put(msg_parser);
305	314	out:
306		− if (skb_progs) {
	315	+ if (skb_verdict)
307	316	bpf_prog_put(skb_verdict);
	317	+ if (skb_parser)
308	318	bpf_prog_put(skb_parser);
309		− }
310	319	return ret;
311	320	}
312	321

net/ipv4/tcp_minisocks.c+0−1

@@ -548,7 +548,6 @@ struct sock tcp_create_openreq_child(const struct sock sk, · 547 unmodified lines
548	548	newtp->fastopen_req = NULL;
549	549	RCU_INIT_POINTER(newtp->fastopen_rsk, NULL);
550	550
551		− bpf_skops_init_child(sk, newsk);
552	551	tcp_bpf_clone(sk, newsk);
553	552
554	553	__TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS);

net/xdp/xsk_buff_pool.c+0−3

@@ -3,9 +3,6 @@ · 2 unmodified lines
3	3	#include <net/xsk_buff_pool.h>
4	4	#include <net/xdp_sock.h>
5	5	#include <net/xdp_sock_drv.h>
6		−#include <linux/dma-direct.h>
7		−#include <linux/dma-noncoherent.h>
8		−#include <linux/swiotlb.h>
9	6
10	7	#include "xsk_queue.h"
11	8	#include "xdp_umem.h"

net/xdp/xsk_queue.h+4−0

@@ -15,6 +15,10 @@ · 14 unmodified lines
15	15
16	16	struct xdp_ring {
17	17	u32 producer ____cacheline_aligned_in_smp;
	18	+ /* Hinder the adjacent cache prefetcher to prefetch the consumer
	19	+ * pointer if the producer pointer is touched and vice versa.
	20	+ */
	21	+ u32 pad ____cacheline_aligned_in_smp;
18	22	u32 consumer ____cacheline_aligned_in_smp;
19	23	u32 flags;
20	24	};

net/xdp/xskmap.c+1−1

@@ -132,7 +132,7 @@ static int xsk_map_get_next_key(struct bpf_map map, void key, void *next_key) · 131 unmodified lines
132	132	return 0;
133	133	}
134	134
135		−static u32 xsk_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
	135	+static int xsk_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
136	136	{
137	137	const int ret = BPF_REG_0, mp = BPF_REG_1, index = BPF_REG_2;
138	138	struct bpf_insn *insn = insn_buf;

samples/bpf/Makefile+12−3

@@ -98,8 +98,8 @@ test_map_in_map-objs := test_map_in_map_user.o · 97 unmodified lines
98	98	per_socket_stats_example-objs := cookie_uid_helper_example.o
99	99	xdp_redirect-objs := xdp_redirect_user.o
100	100	xdp_redirect_map-objs := xdp_redirect_map_user.o
101		−xdp_redirect_cpu-objs := bpf_load.o xdp_redirect_cpu_user.o
102		−xdp_monitor-objs := bpf_load.o xdp_monitor_user.o
	101	+xdp_redirect_cpu-objs := xdp_redirect_cpu_user.o
	102	+xdp_monitor-objs := xdp_monitor_user.o
103	103	xdp_rxq_info-objs := xdp_rxq_info_user.o
104	104	syscall_tp-objs := syscall_tp_user.o
105	105	cpustat-objs := cpustat_user.o
@@ -211,6 +211,8 @@ TPROGLDLIBS_xsk_fwd += -pthread · 105 unmodified lines
211	211	# make M=samples/bpf/ LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang
212	212	LLC ?= llc
213	213	CLANG ?= clang
	214	+OPT ?= opt
	215	+LLVM_DIS ?= llvm-dis
214	216	LLVM_OBJCOPY ?= llvm-objcopy
215	217	BTF_PAHOLE ?= pahole
216	218
@@ -303,6 +305,11 @@ $(obj)/hbm_edt_kern.o: $(src)/hbm.h $(src)/hbm_kern.h · 86 unmodified lines
303	305	# asm/sysreg.h - inline assembly used by it is incompatible with llvm.
304	306	# But, there is no easy way to fix it, so just exclude it since it is
305	307	# useless for BPF samples.
	308	+# below we use long chain of commands, clang \| opt \| llvm-dis \| llc,
	309	+# to generate final object file. 'clang' compiles the source into IR
	310	+# with native target, e.g., x64, arm64, etc. 'opt' does bpf CORE IR builtin
	311	+# processing (llvm12) and IR optimizations. 'llvm-dis' converts
	312	+# 'opt' output to IR, and finally 'llc' generates bpf byte code.
306	313	$(obj)/%.o: $(src)/%.c
307	314	@echo " CLANG-bpf " $@
308	315	$(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(BPF_EXTRA_CFLAGS) \
@@ -314,7 +321,9 @@ $(obj)/%.o: $(src)/%.c · 5 unmodified lines
314	321	-Wno-address-of-packed-member -Wno-tautological-compare \
315	322	-Wno-unknown-warning-option $(CLANG_ARCH_ARGS) \
316	323	-I$(srctree)/samples/bpf/ -include asm_goto_workaround.h \
317		− -O2 -emit-llvm -c $< -o -\| $(LLC) -march=bpf $(LLC_FLAGS) -filetype=obj -o $@
	324	+ -O2 -emit-llvm -Xclang -disable-llvm-passes -c $< -o - \| \
	325	+ $(OPT) -O2 -mtriple=bpf-pc-linux \| $(LLVM_DIS) \| \
	326	+ $(LLC) -march=bpf $(LLC_FLAGS) -filetype=obj -o $@
318	327	ifeq ($(DWARF2BTF),y)
319	328	$(BTF_PAHOLE) -J $@
320	329	endif

samples/bpf/hbm.c+2−1

@@ -40,6 +40,7 @@ · 39 unmodified lines
40	40	#include <errno.h>
41	41	#include <fcntl.h>
42	42	#include <linux/unistd.h>
	43	+#include <linux/compiler.h>
43	44
44	45	#include <linux/bpf.h>
45	46	#include <bpf/bpf.h>
@@ -483,7 +484,7 @@ int main(int argc, char **argv) · 437 unmodified lines
483	484	"Option -%c requires an argument.\n\n",
484	485	optopt);
485	486	case 'h':
486		− fallthrough;
	487	+ __fallthrough;
487	488	default:
488	489	Usage();
489	490	return 0;

samples/bpf/xdp_monitor_kern.c+30−30

@@ -6,21 +6,21 @@ · 5 unmodified lines
6	6	#include <uapi/linux/bpf.h>
7	7	#include <bpf/bpf_helpers.h>
8	8
9		−struct bpf_map_def SEC("maps") redirect_err_cnt = {
10		− .type = BPF_MAP_TYPE_PERCPU_ARRAY,
11		− .key_size = sizeof(u32),
12		− .value_size = sizeof(u64),
13		− .max_entries = 2,
	9	+struct {
	10	+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	11	+ __type(key, u32);
	12	+ __type(value, u64);
	13	+ __uint(max_entries, 2);
14	14	/* TODO: have entries for all possible errno's */
15		−};
	15	+} redirect_err_cnt SEC(".maps");
16	16
17	17	#define XDP_UNKNOWN XDP_REDIRECT + 1
18		−struct bpf_map_def SEC("maps") exception_cnt = {
19		− .type = BPF_MAP_TYPE_PERCPU_ARRAY,
20		− .key_size = sizeof(u32),
21		− .value_size = sizeof(u64),
22		− .max_entries = XDP_UNKNOWN + 1,
23		−};
	18	+struct {
	19	+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	20	+ __type(key, u32);
	21	+ __type(value, u64);
	22	+ __uint(max_entries, XDP_UNKNOWN + 1);
	23	+} exception_cnt SEC(".maps");
24	24
25	25	/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_redirect/format
26	26	* Code in: kernel/include/trace/events/xdp.h
@@ -129,19 +129,19 @@ struct datarec { · 102 unmodified lines
129	129	};
130	130	#define MAX_CPUS 64
131	131
132		−struct bpf_map_def SEC("maps") cpumap_enqueue_cnt = {
133		− .type = BPF_MAP_TYPE_PERCPU_ARRAY,
134		− .key_size = sizeof(u32),
135		− .value_size = sizeof(struct datarec),
136		− .max_entries = MAX_CPUS,
137		−};
	132	+struct {
	133	+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	134	+ __type(key, u32);
	135	+ __type(value, struct datarec);
	136	+ __uint(max_entries, MAX_CPUS);
	137	+} cpumap_enqueue_cnt SEC(".maps");
138	138
139		−struct bpf_map_def SEC("maps") cpumap_kthread_cnt = {
140		− .type = BPF_MAP_TYPE_PERCPU_ARRAY,
141		− .key_size = sizeof(u32),
142		− .value_size = sizeof(struct datarec),
143		− .max_entries = 1,
144		−};
	139	+struct {
	140	+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	141	+ __type(key, u32);
	142	+ __type(value, struct datarec);
	143	+ __uint(max_entries, 1);
	144	+} cpumap_kthread_cnt SEC(".maps");
145	145
146	146	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_enqueue/format
147	147	* Code in: kernel/include/trace/events/xdp.h
@@ -210,12 +210,12 @@ int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx) · 62 unmodified lines
210	210	return 0;
211	211	}
212	212
213		−struct bpf_map_def SEC("maps") devmap_xmit_cnt = {
214		− .type = BPF_MAP_TYPE_PERCPU_ARRAY,
215		− .key_size = sizeof(u32),
216		− .value_size = sizeof(struct datarec),
217		− .max_entries = 1,
218		−};
	213	+struct {
	214	+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	215	+ __type(key, u32);
	216	+ __type(value, struct datarec);
	217	+ __uint(max_entries, 1);
	218	+} devmap_xmit_cnt SEC(".maps");
219	219
220	220	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_devmap_xmit/format
221	221	* Code in: kernel/include/trace/events/xdp.h

samples/bpf/xdp_monitor_user.c+120−39

@@ -26,12 +26,37 @@ static const char *__doc_err_only__= · 25 unmodified lines
26	26	#include <net/if.h>
27	27	#include <time.h>
28	28
	29	+#include <signal.h>
29	30	#include <bpf/bpf.h>
30		−#include "bpf_load.h"
	31	+#include <bpf/libbpf.h>
31	32	#include "bpf_util.h"
32	33
	34	+enum map_type {
	35	+ REDIRECT_ERR_CNT,
	36	+ EXCEPTION_CNT,
	37	+ CPUMAP_ENQUEUE_CNT,
	38	+ CPUMAP_KTHREAD_CNT,
	39	+ DEVMAP_XMIT_CNT,
	40	+};
	41	+
	42	+static const char *const map_type_strings[] = {
	43	+ [REDIRECT_ERR_CNT] = "redirect_err_cnt",
	44	+ [EXCEPTION_CNT] = "exception_cnt",
	45	+ [CPUMAP_ENQUEUE_CNT] = "cpumap_enqueue_cnt",
	46	+ [CPUMAP_KTHREAD_CNT] = "cpumap_kthread_cnt",
	47	+ [DEVMAP_XMIT_CNT] = "devmap_xmit_cnt",
	48	+};
	49	+
	50	+#define NUM_MAP 5
	51	+#define NUM_TP 8
	52	+
	53	+static int tp_cnt;
	54	+static int map_cnt;
33	55	static int verbose = 1;
34	56	static bool debug = false;
	57	+struct bpf_map *map_data[NUM_MAP] = {};
	58	+struct bpf_link *tp_links[NUM_TP] = {};
	59	+struct bpf_object *obj;
35	60
36	61	static const struct option long_options[] = {
37	62	{"help", no_argument, NULL, 'h' },
@@ -41,6 +66,16 @@ static const struct option long_options[] = { · 3 unmodified lines
41	66	{0, 0, NULL, 0 }
42	67	};
43	68
	69	+static void int_exit(int sig)
	70	+{
	71	+ /* Detach tracepoints */
	72	+ while (tp_cnt)
	73	+ bpf_link__destroy(tp_links[--tp_cnt]);
	74	+
	75	+ bpf_object__close(obj);
	76	+ exit(0);
	77	+}
	78	+
44	79	/* C standard specifies two constants, EXIT_SUCCESS(0) and EXIT_FAILURE(1) */
45	80	#define EXIT_FAIL_MEM 5
46	81
@@ -483,23 +518,23 @@ static bool stats_collect(struct stats_record *rec) · 436 unmodified lines
483	518	* this can happen by someone running perf-record -e
484	519	*/
485	520
486		− fd = map_data[0].fd; /* map0: redirect_err_cnt */
	521	+ fd = bpf_map__fd(map_data[REDIRECT_ERR_CNT]);
487	522	for (i = 0; i < REDIR_RES_MAX; i++)
488	523	map_collect_record_u64(fd, i, &rec->xdp_redirect[i]);
489	524
490		− fd = map_data[1].fd; /* map1: exception_cnt */
	525	+ fd = bpf_map__fd(map_data[EXCEPTION_CNT]);
491	526	for (i = 0; i < XDP_ACTION_MAX; i++) {
492	527	map_collect_record_u64(fd, i, &rec->xdp_exception[i]);
493	528	}
494	529
495		− fd = map_data[2].fd; /* map2: cpumap_enqueue_cnt */
	530	+ fd = bpf_map__fd(map_data[CPUMAP_ENQUEUE_CNT]);
496	531	for (i = 0; i < MAX_CPUS; i++)
497	532	map_collect_record(fd, i, &rec->xdp_cpumap_enqueue[i]);
498	533
499		− fd = map_data[3].fd; /* map3: cpumap_kthread_cnt */
	534	+ fd = bpf_map__fd(map_data[CPUMAP_KTHREAD_CNT]);
500	535	map_collect_record(fd, 0, &rec->xdp_cpumap_kthread);
501	536
502		− fd = map_data[4].fd; /* map4: devmap_xmit_cnt */
	537	+ fd = bpf_map__fd(map_data[DEVMAP_XMIT_CNT]);
503	538	map_collect_record(fd, 0, &rec->xdp_devmap_xmit);
504	539
505	540	return true;
@@ -598,8 +633,8 @@ static void stats_poll(int interval, bool err_only) · 92 unmodified lines
598	633
599	634	/* TODO Need more advanced stats on error types */
600	635	if (verbose) {
601		− printf(" - Stats map0: %s\n", map_data[0].name);
602		− printf(" - Stats map1: %s\n", map_data[1].name);
	636	+ printf(" - Stats map0: %s\n", bpf_map__name(map_data[0]));
	637	+ printf(" - Stats map1: %s\n", bpf_map__name(map_data[1]));
603	638	printf("\n");
604	639	}
605	640	fflush(stdout);
@@ -618,44 +653,51 @@ static void stats_poll(int interval, bool err_only) · 12 unmodified lines
618	653
619	654	static void print_bpf_prog_info(void)
620	655	{
621		− int i;
	656	+ struct bpf_program *prog;
	657	+ struct bpf_map *map;
	658	+ int i = 0;
622	659
623	660	/* Prog info */
624		− printf("Loaded BPF prog have %d bpf program(s)\n", prog_cnt);
625		− for (i = 0; i < prog_cnt; i++) {
626		− printf(" - prog_fd[%d] = fd(%d)\n", i, prog_fd[i]);
	661	+ printf("Loaded BPF prog have %d bpf program(s)\n", tp_cnt);
	662	+ bpf_object__for_each_program(prog, obj) {
	663	+ printf(" - prog_fd[%d] = fd(%d)\n", i, bpf_program__fd(prog));
	664	+ i++;
627	665	}
628	666
	667	+ i = 0;
629	668	/* Maps info */
630		− printf("Loaded BPF prog have %d map(s)\n", map_data_count);
631		− for (i = 0; i < map_data_count; i++) {
632		− char *name = map_data[i].name;
633		− int fd = map_data[i].fd;
	669	+ printf("Loaded BPF prog have %d map(s)\n", map_cnt);
	670	+ bpf_object__for_each_map(map, obj) {
	671	+ const char *name = bpf_map__name(map);
	672	+ int fd = bpf_map__fd(map);
634	673
635	674	printf(" - map_data[%d] = fd(%d) name:%s\n", i, fd, name);
	675	+ i++;
636	676	}
637	677
638	678	/* Event info */
639		− printf("Searching for (max:%d) event file descriptor(s)\n", prog_cnt);
640		− for (i = 0; i < prog_cnt; i++) {
641		− if (event_fd[i] != -1)
642		− printf(" - event_fd[%d] = fd(%d)\n", i, event_fd[i]);
	679	+ printf("Searching for (max:%d) event file descriptor(s)\n", tp_cnt);
	680	+ for (i = 0; i < tp_cnt; i++) {
	681	+ int fd = bpf_link__fd(tp_links[i]);
	682	+
	683	+ if (fd != -1)
	684	+ printf(" - event_fd[%d] = fd(%d)\n", i, fd);
643	685	}
644	686	}
645	687
646	688	int main(int argc, char **argv)
647	689	{
648	690	struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
	691	+ struct bpf_program *prog;
649	692	int longindex = 0, opt;
650		− int ret = EXIT_SUCCESS;
651		− char bpf_obj_file[256];
	693	+ int ret = EXIT_FAILURE;
	694	+ enum map_type type;
	695	+ char filename[256];
652	696
653	697	/* Default settings: */
654	698	bool errors_only = true;
655	699	int interval = 2;
656	700
657		− snprintf(bpf_obj_file, sizeof(bpf_obj_file), "%s_kern.o", argv[0]);
658		−
659	701	/* Parse commands line args */
660	702	while ((opt = getopt_long(argc, argv, "hDSs:",
661	703	long_options, &longindex)) != -1) {
@@ -672,40 +714,79 @@ int main(int argc, char **argv) · 10 unmodified lines
672	714	case 'h':
673	715	default:
674	716	usage(argv);
675		− return EXIT_FAILURE;
	717	+ return ret;
676	718	}
677	719	}
678	720
	721	+ snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
679	722	if (setrlimit(RLIMIT_MEMLOCK, &r)) {
680	723	perror("setrlimit(RLIMIT_MEMLOCK)");
681		− return EXIT_FAILURE;
	724	+ return ret;
682	725	}
683	726
684		− if (load_bpf_file(bpf_obj_file)) {
685		− printf("ERROR - bpf_log_buf: %s", bpf_log_buf);
686		− return EXIT_FAILURE;
	727	+ /* Remove tracepoint program when program is interrupted or killed */
	728	+ signal(SIGINT, int_exit);
	729	+ signal(SIGTERM, int_exit);
	730	+
	731	+ obj = bpf_object__open_file(filename, NULL);
	732	+ if (libbpf_get_error(obj)) {
	733	+ printf("ERROR: opening BPF object file failed\n");
	734	+ obj = NULL;
	735	+ goto cleanup;
	736	+ }
	737	+
	738	+ /* load BPF program */
	739	+ if (bpf_object__load(obj)) {
	740	+ printf("ERROR: loading BPF object file failed\n");
	741	+ goto cleanup;
	742	+ }
	743	+
	744	+ for (type = 0; type < NUM_MAP; type++) {
	745	+ map_data[type] =
	746	+ bpf_object__find_map_by_name(obj, map_type_strings[type]);
	747	+
	748	+ if (libbpf_get_error(map_data[type])) {
	749	+ printf("ERROR: finding a map in obj file failed\n");
	750	+ goto cleanup;
	751	+ }
	752	+ map_cnt++;
687	753	}
688		− if (!prog_fd[0]) {
689		− printf("ERROR - load_bpf_file: %s\n", strerror(errno));
690		− return EXIT_FAILURE;
	754	+
	755	+ bpf_object__for_each_program(prog, obj) {
	756	+ tp_links[tp_cnt] = bpf_program__attach(prog);
	757	+ if (libbpf_get_error(tp_links[tp_cnt])) {
	758	+ printf("ERROR: bpf_program__attach failed\n");
	759	+ tp_links[tp_cnt] = NULL;
	760	+ goto cleanup;
	761	+ }
	762	+ tp_cnt++;
691	763	}
692	764
693	765	if (debug) {
694	766	print_bpf_prog_info();
695	767	}
696	768
697		− /* Unload/stop tracepoint event by closing fd's */
	769	+ /* Unload/stop tracepoint event by closing bpf_link's */
698	770	if (errors_only) {
699		− /* The prog_fd[i] and event_fd[i] depend on the
700		− * order the functions was defined in _kern.c
	771	+ /* The bpf_link[i] depend on the order of
	772	+ * the functions was defined in _kern.c
701	773	*/
702		− close(event_fd[2]); /* tracepoint/xdp/xdp_redirect */
703		− close(prog_fd[2]); /* func: trace_xdp_redirect */
704		− close(event_fd[3]); /* tracepoint/xdp/xdp_redirect_map */
705		− close(prog_fd[3]); /* func: trace_xdp_redirect_map */
	774	+ bpf_link__destroy(tp_links[2]); /* tracepoint/xdp/xdp_redirect */
	775	+ tp_links[2] = NULL;
	776	+
	777	+ bpf_link__destroy(tp_links[3]); /* tracepoint/xdp/xdp_redirect_map */
	778	+ tp_links[3] = NULL;
706	779	}
707	780
708	781	stats_poll(interval, errors_only);
709	782
	783	+ ret = EXIT_SUCCESS;
	784	+
	785	+cleanup:
	786	+ /* Detach tracepoints */
	787	+ while (tp_cnt)
	788	+ bpf_link__destroy(tp_links[--tp_cnt]);
	789	+
	790	+ bpf_object__close(obj);
710	791	return ret;
711	792	}

samples/bpf/xdp_redirect_cpu_user.c+72−81

@@ -37,18 +37,35 @@ static __u32 prog_id; · 36 unmodified lines
37	37
38	38	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
39	39	static int n_cpus;
40		−static int cpu_map_fd;
41		−static int rx_cnt_map_fd;
42		−static int redirect_err_cnt_map_fd;
43		−static int cpumap_enqueue_cnt_map_fd;
44		−static int cpumap_kthread_cnt_map_fd;
45		−static int cpus_available_map_fd;
46		−static int cpus_count_map_fd;
47		−static int cpus_iterator_map_fd;
48		−static int exception_cnt_map_fd;
	40	+
	41	+enum map_type {
	42	+ CPU_MAP,
	43	+ RX_CNT,
	44	+ REDIRECT_ERR_CNT,
	45	+ CPUMAP_ENQUEUE_CNT,
	46	+ CPUMAP_KTHREAD_CNT,
	47	+ CPUS_AVAILABLE,
	48	+ CPUS_COUNT,
	49	+ CPUS_ITERATOR,
	50	+ EXCEPTION_CNT,
	51	+};
	52	+
	53	+static const char *const map_type_strings[] = {
	54	+ [CPU_MAP] = "cpu_map",
	55	+ [RX_CNT] = "rx_cnt",
	56	+ [REDIRECT_ERR_CNT] = "redirect_err_cnt",
	57	+ [CPUMAP_ENQUEUE_CNT] = "cpumap_enqueue_cnt",
	58	+ [CPUMAP_KTHREAD_CNT] = "cpumap_kthread_cnt",
	59	+ [CPUS_AVAILABLE] = "cpus_available",
	60	+ [CPUS_COUNT] = "cpus_count",
	61	+ [CPUS_ITERATOR] = "cpus_iterator",
	62	+ [EXCEPTION_CNT] = "exception_cnt",
	63	+};
49	64
50	65	#define NUM_TP 5
51		−struct bpf_link *tp_links[NUM_TP] = { 0 };
	66	+#define NUM_MAP 9
	67	+struct bpf_link *tp_links[NUM_TP] = {};
	68	+static int map_fds[NUM_MAP];
52	69	static int tp_cnt = 0;
53	70
54	71	/* Exit return codes */
@@ -527,20 +544,20 @@ static void stats_collect(struct stats_record *rec) · 472 unmodified lines
527	544	{
528	545	int fd, i;
529	546
530		− fd = rx_cnt_map_fd;
	547	+ fd = map_fds[RX_CNT];
531	548	map_collect_percpu(fd, 0, &rec->rx_cnt);
532	549
533		− fd = redirect_err_cnt_map_fd;
	550	+ fd = map_fds[REDIRECT_ERR_CNT];
534	551	map_collect_percpu(fd, 1, &rec->redir_err);
535	552
536		− fd = cpumap_enqueue_cnt_map_fd;
	553	+ fd = map_fds[CPUMAP_ENQUEUE_CNT];
537	554	for (i = 0; i < n_cpus; i++)
538	555	map_collect_percpu(fd, i, &rec->enq[i]);
539	556
540		− fd = cpumap_kthread_cnt_map_fd;
	557	+ fd = map_fds[CPUMAP_KTHREAD_CNT];
541	558	map_collect_percpu(fd, 0, &rec->kthread);
542	559
543		− fd = exception_cnt_map_fd;
	560	+ fd = map_fds[EXCEPTION_CNT];
544	561	map_collect_percpu(fd, 0, &rec->exception);
545	562	}
546	563
@@ -565,7 +582,7 @@ static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value, · 18 unmodified lines
565	582	/* Add a CPU entry to cpumap, as this allocate a cpu entry in
566	583	* the kernel for the cpu.
567	584	*/
568		− ret = bpf_map_update_elem(cpu_map_fd, &cpu, value, 0);
	585	+ ret = bpf_map_update_elem(map_fds[CPU_MAP], &cpu, value, 0);
569	586	if (ret) {
570	587	fprintf(stderr, "Create CPU entry failed (err:%d)\n", ret);
571	588	exit(EXIT_FAIL_BPF);
@@ -574,21 +591,21 @@ static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value, · 2 unmodified lines
574	591	/* Inform bpf_prog's that a new CPU is available to select
575	592	* from via some control maps.
576	593	*/
577		− ret = bpf_map_update_elem(cpus_available_map_fd, &avail_idx, &cpu, 0);
	594	+ ret = bpf_map_update_elem(map_fds[CPUS_AVAILABLE], &avail_idx, &cpu, 0);
578	595	if (ret) {
579	596	fprintf(stderr, "Add to avail CPUs failed\n");
580	597	exit(EXIT_FAIL_BPF);
581	598	}
582	599
583	600	/* When not replacing/updating existing entry, bump the count */
584		− ret = bpf_map_lookup_elem(cpus_count_map_fd, &key, &curr_cpus_count);
	601	+ ret = bpf_map_lookup_elem(map_fds[CPUS_COUNT], &key, &curr_cpus_count);
585	602	if (ret) {
586	603	fprintf(stderr, "Failed reading curr cpus_count\n");
587	604	exit(EXIT_FAIL_BPF);
588	605	}
589	606	if (new) {
590	607	curr_cpus_count++;
591		− ret = bpf_map_update_elem(cpus_count_map_fd, &key,
	608	+ ret = bpf_map_update_elem(map_fds[CPUS_COUNT], &key,
592	609	&curr_cpus_count, 0);
593	610	if (ret) {
594	611	fprintf(stderr, "Failed write curr cpus_count\n");
@@ -612,7 +629,7 @@ static void mark_cpus_unavailable(void) · 17 unmodified lines
612	629	int ret, i;
613	630
614	631	for (i = 0; i < n_cpus; i++) {
615		− ret = bpf_map_update_elem(cpus_available_map_fd, &i,
	632	+ ret = bpf_map_update_elem(map_fds[CPUS_AVAILABLE], &i,
616	633	&invalid_cpu, 0);
617	634	if (ret) {
618	635	fprintf(stderr, "Failed marking CPU unavailable\n");
@@ -665,68 +682,37 @@ static void stats_poll(int interval, bool use_separators, char *prog_name, · 46 unmodified lines
665	682	free_stats_record(prev);
666	683	}
667	684
668		−static struct bpf_link * attach_tp(struct bpf_object *obj,
669		− const char *tp_category,
670		− const char* tp_name)
	685	+static int init_tracepoints(struct bpf_object *obj)
671	686	{
672	687	struct bpf_program *prog;
673		− struct bpf_link *link;
674		− char sec_name[PATH_MAX];
675		− int len;
676	688
677		− len = snprintf(sec_name, PATH_MAX, "tracepoint/%s/%s",
678		− tp_category, tp_name);
679		− if (len < 0)
680		− exit(EXIT_FAIL);
	689	+ bpf_object__for_each_program(prog, obj) {
	690	+ if (bpf_program__is_tracepoint(prog) != true)
	691	+ continue;
681	692
682		− prog = bpf_object__find_program_by_title(obj, sec_name);
683		− if (!prog) {
684		− fprintf(stderr, "ERR: finding progsec: %s\n", sec_name);
685		− exit(EXIT_FAIL_BPF);
	693	+ tp_links[tp_cnt] = bpf_program__attach(prog);
	694	+ if (libbpf_get_error(tp_links[tp_cnt])) {
	695	+ tp_links[tp_cnt] = NULL;
	696	+ return -EINVAL;
	697	+ }
	698	+ tp_cnt++;
686	699	}
687	700
688		− link = bpf_program__attach_tracepoint(prog, tp_category, tp_name);
689		− if (libbpf_get_error(link))
690		− exit(EXIT_FAIL_BPF);
691		−
692		− return link;
693		−}
694		−
695		−static void init_tracepoints(struct bpf_object *obj) {
696		− tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_redirect_err");
697		− tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_redirect_map_err");
698		− tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_exception");
699		− tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_cpumap_enqueue");
700		− tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_cpumap_kthread");
	701	+ return 0;
701	702	}
702	703
703	704	static int init_map_fds(struct bpf_object *obj)
704	705	{
705		− /* Maps updated by tracepoints */
706		− redirect_err_cnt_map_fd =
707		− bpf_object__find_map_fd_by_name(obj, "redirect_err_cnt");
708		− exception_cnt_map_fd =
709		− bpf_object__find_map_fd_by_name(obj, "exception_cnt");
710		− cpumap_enqueue_cnt_map_fd =
711		− bpf_object__find_map_fd_by_name(obj, "cpumap_enqueue_cnt");
712		− cpumap_kthread_cnt_map_fd =
713		− bpf_object__find_map_fd_by_name(obj, "cpumap_kthread_cnt");
714		−
715		− /* Maps used by XDP */
716		− rx_cnt_map_fd = bpf_object__find_map_fd_by_name(obj, "rx_cnt");
717		− cpu_map_fd = bpf_object__find_map_fd_by_name(obj, "cpu_map");
718		− cpus_available_map_fd =
719		− bpf_object__find_map_fd_by_name(obj, "cpus_available");
720		− cpus_count_map_fd = bpf_object__find_map_fd_by_name(obj, "cpus_count");
721		− cpus_iterator_map_fd =
722		− bpf_object__find_map_fd_by_name(obj, "cpus_iterator");
723		−
724		− if (cpu_map_fd < 0 \|\| rx_cnt_map_fd < 0 \|\|
725		− redirect_err_cnt_map_fd < 0 \|\| cpumap_enqueue_cnt_map_fd < 0 \|\|
726		− cpumap_kthread_cnt_map_fd < 0 \|\| cpus_available_map_fd < 0 \|\|
727		− cpus_count_map_fd < 0 \|\| cpus_iterator_map_fd < 0 \|\|
728		− exception_cnt_map_fd < 0)
729		− return -ENOENT;
	706	+ enum map_type type;
	707	+
	708	+ for (type = 0; type < NUM_MAP; type++) {
	709	+ map_fds[type] =
	710	+ bpf_object__find_map_fd_by_name(obj,
	711	+ map_type_strings[type]);
	712	+
	713	+ if (map_fds[type] < 0)
	714	+ return -ENOENT;
	715	+ }
730	716
731	717	return 0;
732	718	}
@@ -795,13 +781,13 @@ int main(int argc, char **argv) · 62 unmodified lines
795	781	bool stress_mode = false;
796	782	struct bpf_program *prog;
797	783	struct bpf_object *obj;
	784	+ int err = EXIT_FAIL;
798	785	char filename[256];
799	786	int added_cpus = 0;
800	787	int longindex = 0;
801	788	int interval = 2;
802	789	int add_cpu = -1;
803		− int opt, err;
804		− int prog_fd;
	790	+ int opt, prog_fd;
805	791	int *cpu, i;
806	792	__u32 qsize;
807	793
@@ -824,24 +810,29 @@ int main(int argc, char **argv) · 16 unmodified lines
824	810	}
825	811
826	812	if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))
827		− return EXIT_FAIL;
	813	+ return err;
828	814
829	815	if (prog_fd < 0) {
830	816	fprintf(stderr, "ERR: bpf_prog_load_xattr: %s\n",
831	817	strerror(errno));
832		− return EXIT_FAIL;
	818	+ return err;
833	819	}
834		− init_tracepoints(obj);
	820	+
	821	+ if (init_tracepoints(obj) < 0) {
	822	+ fprintf(stderr, "ERR: bpf_program__attach failed\n");
	823	+ return err;
	824	+ }
	825	+
835	826	if (init_map_fds(obj) < 0) {
836	827	fprintf(stderr, "bpf_object__find_map_fd_by_name failed\n");
837		− return EXIT_FAIL;
	828	+ return err;
838	829	}
839	830	mark_cpus_unavailable();
840	831
841	832	cpu = malloc(n_cpus * sizeof(int));
842	833	if (!cpu) {
843	834	fprintf(stderr, "failed to allocate cpu array\n");
844		− return EXIT_FAIL;
	835	+ return err;
845	836	}
846	837	memset(cpu, 0, n_cpus * sizeof(int));
847	838
@@ -960,14 +951,12 @@ int main(int argc, char **argv) · 112 unmodified lines
960	951	prog = bpf_object__find_program_by_title(obj, prog_name);
961	952	if (!prog) {
962	953	fprintf(stderr, "bpf_object__find_program_by_title failed\n");
963		− err = EXIT_FAIL;
964	954	goto out;
965	955	}
966	956
967	957	prog_fd = bpf_program__fd(prog);
968	958	if (prog_fd < 0) {
969	959	fprintf(stderr, "bpf_program__fd failed\n");
970		− err = EXIT_FAIL;
971	960	goto out;
972	961	}
973	962
@@ -986,6 +975,8 @@ int main(int argc, char **argv) · 12 unmodified lines
986	975
987	976	stats_poll(interval, use_separators, prog_name, mprog_name,
988	977	&value, stress_mode);
	978	+
	979	+ err = EXIT_OK;
989	980	out:
990	981	free(cpu);
991	982	return err;

samples/bpf/xdp_sample_pkts_kern.c+6−8

@@ -5,14 +5,12 @@ · 4 unmodified lines
5	5	#include <bpf/bpf_helpers.h>
6	6
7	7	#define SAMPLE_SIZE 64ul
8		−#define MAX_CPUS 128
9		−
10		−struct bpf_map_def SEC("maps") my_map = {
11		− .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
12		− .key_size = sizeof(int),
13		− .value_size = sizeof(u32),
14		− .max_entries = MAX_CPUS,
15		−};
	8	+
	9	+struct {
	10	+ __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
	11	+ __uint(key_size, sizeof(int));
	12	+ __uint(value_size, sizeof(u32));
	13	+} my_map SEC(".maps");
16	14
17	15	SEC("xdp_sample")
18	16	int xdp_sample_prog(struct xdp_md *ctx)

samples/bpf/xdp_sample_pkts_user.c+0−1

@@ -18,7 +18,6 @@ · 17 unmodified lines
18	18
19	19	#include "perf-sys.h"
20	20
21		−#define MAX_CPUS 128
22	21	static int if_idx;
23	22	static char *if_name;
24	23	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;

samples/bpf/xdpsock_user.c+290−64

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <linux/if_xdp.h>
12	12	#include <linux/if_ether.h>
13	13	#include <linux/ip.h>
	14	+#include <linux/limits.h>
14	15	#include <linux/udp.h>
15	16	#include <arpa/inet.h>
16	17	#include <locale.h>
@@ -79,6 +80,10 @@ static u16 opt_pkt_size = MIN_PKT_SIZE; · 62 unmodified lines
79	80	static u32 opt_pkt_fill_pattern = 0x12345678;
80	81	static bool opt_extra_stats;
81	82	static bool opt_quiet;
	83	+static bool opt_app_stats;
	84	+static const char *opt_irq_str = "";
	85	+static u32 irq_no;
	86	+static int irqs_at_init = -1;
82	87	static int opt_poll;
83	88	static int opt_interval = 1;
84	89	static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP;
@@ -91,18 +96,7 @@ static bool opt_need_wakeup = true; · 6 unmodified lines
91	96	static u32 opt_num_xsks = 1;
92	97	static u32 prog_id;
93	98
94		−struct xsk_umem_info {
95		− struct xsk_ring_prod fq;
96		− struct xsk_ring_cons cq;
97		− struct xsk_umem *umem;
98		− void *buffer;
99		−};
100		−
101		−struct xsk_socket_info {
102		− struct xsk_ring_cons rx;
103		− struct xsk_ring_prod tx;
104		− struct xsk_umem_info *umem;
105		− struct xsk_socket *xsk;
	99	+struct xsk_ring_stats {
106	100	unsigned long rx_npkts;
107	101	unsigned long tx_npkts;
108	102	unsigned long rx_dropped_npkts;
@@ -119,6 +113,41 @@ struct xsk_socket_info { · 10 unmodified lines
119	113	unsigned long prev_rx_full_npkts;
120	114	unsigned long prev_rx_fill_empty_npkts;
121	115	unsigned long prev_tx_empty_npkts;
	116	+};
	117	+
	118	+struct xsk_driver_stats {
	119	+ unsigned long intrs;
	120	+ unsigned long prev_intrs;
	121	+};
	122	+
	123	+struct xsk_app_stats {
	124	+ unsigned long rx_empty_polls;
	125	+ unsigned long fill_fail_polls;
	126	+ unsigned long copy_tx_sendtos;
	127	+ unsigned long tx_wakeup_sendtos;
	128	+ unsigned long opt_polls;
	129	+ unsigned long prev_rx_empty_polls;
	130	+ unsigned long prev_fill_fail_polls;
	131	+ unsigned long prev_copy_tx_sendtos;
	132	+ unsigned long prev_tx_wakeup_sendtos;
	133	+ unsigned long prev_opt_polls;
	134	+};
	135	+
	136	+struct xsk_umem_info {
	137	+ struct xsk_ring_prod fq;
	138	+ struct xsk_ring_cons cq;
	139	+ struct xsk_umem *umem;
	140	+ void *buffer;
	141	+};
	142	+
	143	+struct xsk_socket_info {
	144	+ struct xsk_ring_cons rx;
	145	+ struct xsk_ring_prod tx;
	146	+ struct xsk_umem_info *umem;
	147	+ struct xsk_socket *xsk;
	148	+ struct xsk_ring_stats ring_stats;
	149	+ struct xsk_app_stats app_stats;
	150	+ struct xsk_driver_stats drv_stats;
122	151	u32 outstanding_tx;
123	152	};
124	153
@@ -173,18 +202,151 @@ static int xsk_get_xdp_stats(int fd, struct xsk_socket_info *xsk) · 48 unmodified lines
173	202	return err;
174	203
175	204	if (optlen == sizeof(struct xdp_statistics)) {
176		− xsk->rx_dropped_npkts = stats.rx_dropped;
177		− xsk->rx_invalid_npkts = stats.rx_invalid_descs;
178		− xsk->tx_invalid_npkts = stats.tx_invalid_descs;
179		− xsk->rx_full_npkts = stats.rx_ring_full;
180		− xsk->rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs;
181		− xsk->tx_empty_npkts = stats.tx_ring_empty_descs;
	205	+ xsk->ring_stats.rx_dropped_npkts = stats.rx_dropped;
	206	+ xsk->ring_stats.rx_invalid_npkts = stats.rx_invalid_descs;
	207	+ xsk->ring_stats.tx_invalid_npkts = stats.tx_invalid_descs;
	208	+ xsk->ring_stats.rx_full_npkts = stats.rx_ring_full;
	209	+ xsk->ring_stats.rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs;
	210	+ xsk->ring_stats.tx_empty_npkts = stats.tx_ring_empty_descs;
182	211	return 0;
183	212	}
184	213
185	214	return -EINVAL;
186	215	}
187	216
	217	+static void dump_app_stats(long dt)
	218	+{
	219	+ int i;
	220	+
	221	+ for (i = 0; i < num_socks && xsks[i]; i++) {
	222	+ char *fmt = "%-18s %'-14.0f %'-14lu\n";
	223	+ double rx_empty_polls_ps, fill_fail_polls_ps, copy_tx_sendtos_ps,
	224	+ tx_wakeup_sendtos_ps, opt_polls_ps;
	225	+
	226	+ rx_empty_polls_ps = (xsks[i]->app_stats.rx_empty_polls -
	227	+ xsks[i]->app_stats.prev_rx_empty_polls) * 1000000000. / dt;
	228	+ fill_fail_polls_ps = (xsks[i]->app_stats.fill_fail_polls -
	229	+ xsks[i]->app_stats.prev_fill_fail_polls) * 1000000000. / dt;
	230	+ copy_tx_sendtos_ps = (xsks[i]->app_stats.copy_tx_sendtos -
	231	+ xsks[i]->app_stats.prev_copy_tx_sendtos) * 1000000000. / dt;
	232	+ tx_wakeup_sendtos_ps = (xsks[i]->app_stats.tx_wakeup_sendtos -
	233	+ xsks[i]->app_stats.prev_tx_wakeup_sendtos)
	234	+ * 1000000000. / dt;
	235	+ opt_polls_ps = (xsks[i]->app_stats.opt_polls -
	236	+ xsks[i]->app_stats.prev_opt_polls) * 1000000000. / dt;
	237	+
	238	+ printf("\n%-18s %-14s %-14s\n", "", "calls/s", "count");
	239	+ printf(fmt, "rx empty polls", rx_empty_polls_ps, xsks[i]->app_stats.rx_empty_polls);
	240	+ printf(fmt, "fill fail polls", fill_fail_polls_ps,
	241	+ xsks[i]->app_stats.fill_fail_polls);
	242	+ printf(fmt, "copy tx sendtos", copy_tx_sendtos_ps,
	243	+ xsks[i]->app_stats.copy_tx_sendtos);
	244	+ printf(fmt, "tx wakeup sendtos", tx_wakeup_sendtos_ps,
	245	+ xsks[i]->app_stats.tx_wakeup_sendtos);
	246	+ printf(fmt, "opt polls", opt_polls_ps, xsks[i]->app_stats.opt_polls);
	247	+
	248	+ xsks[i]->app_stats.prev_rx_empty_polls = xsks[i]->app_stats.rx_empty_polls;
	249	+ xsks[i]->app_stats.prev_fill_fail_polls = xsks[i]->app_stats.fill_fail_polls;
	250	+ xsks[i]->app_stats.prev_copy_tx_sendtos = xsks[i]->app_stats.copy_tx_sendtos;
	251	+ xsks[i]->app_stats.prev_tx_wakeup_sendtos = xsks[i]->app_stats.tx_wakeup_sendtos;
	252	+ xsks[i]->app_stats.prev_opt_polls = xsks[i]->app_stats.opt_polls;
	253	+ }
	254	+}
	255	+
	256	+static bool get_interrupt_number(void)
	257	+{
	258	+ FILE *f_int_proc;
	259	+ char line[4096];
	260	+ bool found = false;
	261	+
	262	+ f_int_proc = fopen("/proc/interrupts", "r");
	263	+ if (f_int_proc == NULL) {
	264	+ printf("Failed to open /proc/interrupts.\n");
	265	+ return found;
	266	+ }
	267	+
	268	+ while (!feof(f_int_proc) && !found) {
	269	+ /* Make sure to read a full line at a time */
	270	+ if (fgets(line, sizeof(line), f_int_proc) == NULL \|\|
	271	+ line[strlen(line) - 1] != '\n') {
	272	+ printf("Error reading from interrupts file\n");
	273	+ break;
	274	+ }
	275	+
	276	+ /* Extract interrupt number from line */
	277	+ if (strstr(line, opt_irq_str) != NULL) {
	278	+ irq_no = atoi(line);
	279	+ found = true;
	280	+ break;
	281	+ }
	282	+ }
	283	+
	284	+ fclose(f_int_proc);
	285	+
	286	+ return found;
	287	+}
	288	+
	289	+static int get_irqs(void)
	290	+{
	291	+ char count_path[PATH_MAX];
	292	+ int total_intrs = -1;
	293	+ FILE *f_count_proc;
	294	+ char line[4096];
	295	+
	296	+ snprintf(count_path, sizeof(count_path),
	297	+ "/sys/kernel/irq/%i/per_cpu_count", irq_no);
	298	+ f_count_proc = fopen(count_path, "r");
	299	+ if (f_count_proc == NULL) {
	300	+ printf("Failed to open %s\n", count_path);
	301	+ return total_intrs;
	302	+ }
	303	+
	304	+ if (fgets(line, sizeof(line), f_count_proc) == NULL \|\|
	305	+ line[strlen(line) - 1] != '\n') {
	306	+ printf("Error reading from %s\n", count_path);
	307	+ } else {
	308	+ static const char com[2] = ",";
	309	+ char *token;
	310	+
	311	+ total_intrs = 0;
	312	+ token = strtok(line, com);
	313	+ while (token != NULL) {
	314	+ /* sum up interrupts across all cores */
	315	+ total_intrs += atoi(token);
	316	+ token = strtok(NULL, com);
	317	+ }
	318	+ }
	319	+
	320	+ fclose(f_count_proc);
	321	+
	322	+ return total_intrs;
	323	+}
	324	+
	325	+static void dump_driver_stats(long dt)
	326	+{
	327	+ int i;
	328	+
	329	+ for (i = 0; i < num_socks && xsks[i]; i++) {
	330	+ char *fmt = "%-18s %'-14.0f %'-14lu\n";
	331	+ double intrs_ps;
	332	+ int n_ints = get_irqs();
	333	+
	334	+ if (n_ints < 0) {
	335	+ printf("error getting intr info for intr %i\n", irq_no);
	336	+ return;
	337	+ }
	338	+ xsks[i]->drv_stats.intrs = n_ints - irqs_at_init;
	339	+
	340	+ intrs_ps = (xsks[i]->drv_stats.intrs - xsks[i]->drv_stats.prev_intrs) *
	341	+ 1000000000. / dt;
	342	+
	343	+ printf("\n%-18s %-14s %-14s\n", "", "intrs/s", "count");
	344	+ printf(fmt, "irqs", intrs_ps, xsks[i]->drv_stats.intrs);
	345	+
	346	+ xsks[i]->drv_stats.prev_intrs = xsks[i]->drv_stats.intrs;
	347	+ }
	348	+}
	349	+
188	350	static void dump_stats(void)
189	351	{
190	352	unsigned long now = get_nsecs();
@@ -194,67 +356,83 @@ static void dump_stats(void) · 3 unmodified lines
194	356	prev_time = now;
195	357
196	358	for (i = 0; i < num_socks && xsks[i]; i++) {
197		− char *fmt = "%-15s %'-11.0f %'-11lu\n";
	359	+ char *fmt = "%-18s %'-14.0f %'-14lu\n";
198	360	double rx_pps, tx_pps, dropped_pps, rx_invalid_pps, full_pps, fill_empty_pps,
199	361	tx_invalid_pps, tx_empty_pps;
200	362
201		− rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) *
	363	+ rx_pps = (xsks[i]->ring_stats.rx_npkts - xsks[i]->ring_stats.prev_rx_npkts) *
202	364	1000000000. / dt;
203		− tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) *
	365	+ tx_pps = (xsks[i]->ring_stats.tx_npkts - xsks[i]->ring_stats.prev_tx_npkts) *
204	366	1000000000. / dt;
205	367
206	368	printf("\n sock%d@", i);
207	369	print_benchmark(false);
208	370	printf("\n");
209	371
210		− printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
	372	+ printf("%-18s %-14s %-14s %-14.2f\n", "", "pps", "pkts",
211	373	dt / 1000000000.);
212		− printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts);
213		− printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts);
	374	+ printf(fmt, "rx", rx_pps, xsks[i]->ring_stats.rx_npkts);
	375	+ printf(fmt, "tx", tx_pps, xsks[i]->ring_stats.tx_npkts);
214	376
215		− xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts;
216		− xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts;
	377	+ xsks[i]->ring_stats.prev_rx_npkts = xsks[i]->ring_stats.rx_npkts;
	378	+ xsks[i]->ring_stats.prev_tx_npkts = xsks[i]->ring_stats.tx_npkts;
217	379
218	380	if (opt_extra_stats) {
219	381	if (!xsk_get_xdp_stats(xsk_socket__fd(xsks[i]->xsk), xsks[i])) {
220		− dropped_pps = (xsks[i]->rx_dropped_npkts -
221		− xsks[i]->prev_rx_dropped_npkts) * 1000000000. / dt;
222		− rx_invalid_pps = (xsks[i]->rx_invalid_npkts -
223		− xsks[i]->prev_rx_invalid_npkts) * 1000000000. / dt;
224		− tx_invalid_pps = (xsks[i]->tx_invalid_npkts -
225		− xsks[i]->prev_tx_invalid_npkts) * 1000000000. / dt;
226		− full_pps = (xsks[i]->rx_full_npkts -
227		− xsks[i]->prev_rx_full_npkts) * 1000000000. / dt;
228		− fill_empty_pps = (xsks[i]->rx_fill_empty_npkts -
229		− xsks[i]->prev_rx_fill_empty_npkts)
230		− * 1000000000. / dt;
231		− tx_empty_pps = (xsks[i]->tx_empty_npkts -
232		− xsks[i]->prev_tx_empty_npkts) * 1000000000. / dt;
	382	+ dropped_pps = (xsks[i]->ring_stats.rx_dropped_npkts -
	383	+ xsks[i]->ring_stats.prev_rx_dropped_npkts) *
	384	+ 1000000000. / dt;
	385	+ rx_invalid_pps = (xsks[i]->ring_stats.rx_invalid_npkts -
	386	+ xsks[i]->ring_stats.prev_rx_invalid_npkts) *
	387	+ 1000000000. / dt;
	388	+ tx_invalid_pps = (xsks[i]->ring_stats.tx_invalid_npkts -
	389	+ xsks[i]->ring_stats.prev_tx_invalid_npkts) *
	390	+ 1000000000. / dt;
	391	+ full_pps = (xsks[i]->ring_stats.rx_full_npkts -
	392	+ xsks[i]->ring_stats.prev_rx_full_npkts) *
	393	+ 1000000000. / dt;
	394	+ fill_empty_pps = (xsks[i]->ring_stats.rx_fill_empty_npkts -
	395	+ xsks[i]->ring_stats.prev_rx_fill_empty_npkts) *
	396	+ 1000000000. / dt;
	397	+ tx_empty_pps = (xsks[i]->ring_stats.tx_empty_npkts -
	398	+ xsks[i]->ring_stats.prev_tx_empty_npkts) *
	399	+ 1000000000. / dt;
233	400
234	401	printf(fmt, "rx dropped", dropped_pps,
235		− xsks[i]->rx_dropped_npkts);
	402	+ xsks[i]->ring_stats.rx_dropped_npkts);
236	403	printf(fmt, "rx invalid", rx_invalid_pps,
237		− xsks[i]->rx_invalid_npkts);
	404	+ xsks[i]->ring_stats.rx_invalid_npkts);
238	405	printf(fmt, "tx invalid", tx_invalid_pps,
239		− xsks[i]->tx_invalid_npkts);
	406	+ xsks[i]->ring_stats.tx_invalid_npkts);
240	407	printf(fmt, "rx queue full", full_pps,
241		− xsks[i]->rx_full_npkts);
	408	+ xsks[i]->ring_stats.rx_full_npkts);
242	409	printf(fmt, "fill ring empty", fill_empty_pps,
243		− xsks[i]->rx_fill_empty_npkts);
	410	+ xsks[i]->ring_stats.rx_fill_empty_npkts);
244	411	printf(fmt, "tx ring empty", tx_empty_pps,
245		− xsks[i]->tx_empty_npkts);
246		−
247		− xsks[i]->prev_rx_dropped_npkts = xsks[i]->rx_dropped_npkts;
248		− xsks[i]->prev_rx_invalid_npkts = xsks[i]->rx_invalid_npkts;
249		− xsks[i]->prev_tx_invalid_npkts = xsks[i]->tx_invalid_npkts;
250		− xsks[i]->prev_rx_full_npkts = xsks[i]->rx_full_npkts;
251		− xsks[i]->prev_rx_fill_empty_npkts = xsks[i]->rx_fill_empty_npkts;
252		− xsks[i]->prev_tx_empty_npkts = xsks[i]->tx_empty_npkts;
	412	+ xsks[i]->ring_stats.tx_empty_npkts);
	413	+
	414	+ xsks[i]->ring_stats.prev_rx_dropped_npkts =
	415	+ xsks[i]->ring_stats.rx_dropped_npkts;
	416	+ xsks[i]->ring_stats.prev_rx_invalid_npkts =
	417	+ xsks[i]->ring_stats.rx_invalid_npkts;
	418	+ xsks[i]->ring_stats.prev_tx_invalid_npkts =
	419	+ xsks[i]->ring_stats.tx_invalid_npkts;
	420	+ xsks[i]->ring_stats.prev_rx_full_npkts =
	421	+ xsks[i]->ring_stats.rx_full_npkts;
	422	+ xsks[i]->ring_stats.prev_rx_fill_empty_npkts =
	423	+ xsks[i]->ring_stats.rx_fill_empty_npkts;
	424	+ xsks[i]->ring_stats.prev_tx_empty_npkts =
	425	+ xsks[i]->ring_stats.tx_empty_npkts;
253	426	} else {
254	427	printf("%-15s\n", "Error retrieving extra stats");
255	428	}
256	429	}
257	430	}
	431	+
	432	+ if (opt_app_stats)
	433	+ dump_app_stats(dt);
	434	+ if (irq_no)
	435	+ dump_driver_stats(dt);
258	436	}
259	437
260	438	static bool is_benchmark_done(void)
@@ -693,6 +871,17 @@ static struct xsk_socket_info xsk_configure_socket(struct xsk_umem_info umem, · 432 unmodified lines
693	871	if (ret)
694	872	exit_with_error(-ret);
695	873
	874	+ xsk->app_stats.rx_empty_polls = 0;
	875	+ xsk->app_stats.fill_fail_polls = 0;
	876	+ xsk->app_stats.copy_tx_sendtos = 0;
	877	+ xsk->app_stats.tx_wakeup_sendtos = 0;
	878	+ xsk->app_stats.opt_polls = 0;
	879	+ xsk->app_stats.prev_rx_empty_polls = 0;
	880	+ xsk->app_stats.prev_fill_fail_polls = 0;
	881	+ xsk->app_stats.prev_copy_tx_sendtos = 0;
	882	+ xsk->app_stats.prev_tx_wakeup_sendtos = 0;
	883	+ xsk->app_stats.prev_opt_polls = 0;
	884	+
696	885	return xsk;
697	886	}
698	887
@@ -720,6 +909,8 @@ static struct option long_options[] = { · 21 unmodified lines
720	909	{"tx-pkt-pattern", required_argument, 0, 'P'},
721	910	{"extra-stats", no_argument, 0, 'x'},
722	911	{"quiet", no_argument, 0, 'Q'},
	912	+ {"app-stats", no_argument, 0, 'a'},
	913	+ {"irq-string", no_argument, 0, 'I'},
723	914	{0, 0, 0, 0}
724	915	};
725	916
@@ -756,6 +947,8 @@ static void usage(const char *prog) · 30 unmodified lines
756	947	" -P, --tx-pkt-pattern=nPacket fill pattern. Default: 0x%x\n"
757	948	" -x, --extra-stats Display extra statistics.\n"
758	949	" -Q, --quiet Do not display any stats.\n"
	950	+ " -a, --app-stats Display application (syscall) statistics.\n"
	951	+ " -I, --irq-string Display driver interrupt statistics for interface associated with irq-string.\n"
759	952	"\n";
760	953	fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE,
761	954	opt_batch_size, MIN_PKT_SIZE, MIN_PKT_SIZE,
@@ -771,7 +964,7 @@ static void parse_command_line(int argc, char **argv) · 9 unmodified lines
771	964	opterr = 0;
772	965
773	966	for (;;) {
774		− c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:xQ",
	967	+ c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:xQaI:",
775	968	long_options, &option_index);
776	969	if (c == -1)
777	970	break;
@@ -857,6 +1050,19 @@ static void parse_command_line(int argc, char **argv) · 79 unmodified lines
857	1050	break;
858	1051	case 'Q':
859	1052	opt_quiet = 1;
	1053	+ break;
	1054	+ case 'a':
	1055	+ opt_app_stats = 1;
	1056	+ break;
	1057	+ case 'I':
	1058	+ opt_irq_str = optarg;
	1059	+ if (get_interrupt_number())
	1060	+ irqs_at_init = get_irqs();
	1061	+ if (irqs_at_init < 0) {
	1062	+ fprintf(stderr, "ERROR: Failed to get irqs for %s\n", opt_irq_str);
	1063	+ usage(basename(argv[0]));
	1064	+ }
	1065	+
860	1066	break;
861	1067	default:
862	1068	usage(basename(argv[0]));
@@ -908,8 +1114,10 @@ static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, · 45 unmodified lines
908	1114	* is driven by the NAPI loop. So as an optimization, we do not have to call
909	1115	* sendto() all the time in zero-copy mode for l2fwd.
910	1116	*/
911		− if (opt_xdp_bind_flags & XDP_COPY)
	1117	+ if (opt_xdp_bind_flags & XDP_COPY) {
	1118	+ xsk->app_stats.copy_tx_sendtos++;
912	1119	kick_tx(xsk);
	1120	+ }
913	1121
914	1122	ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size :
915	1123	xsk->outstanding_tx;
@@ -924,8 +1132,10 @@ static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, · 8 unmodified lines
924	1132	while (ret != rcvd) {
925	1133	if (ret < 0)
926	1134	exit_with_error(-ret);
927		− if (xsk_ring_prod__needs_wakeup(&umem->fq))
	1135	+ if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
	1136	+ xsk->app_stats.fill_fail_polls++;
928	1137	ret = poll(fds, num_socks, opt_timeout);
	1138	+ }
929	1139	ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
930	1140	}
931	1141
@@ -936,7 +1146,7 @@ static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk, · 4 unmodified lines
936	1146	xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
937	1147	xsk_ring_cons__release(&xsk->umem->cq, rcvd);
938	1148	xsk->outstanding_tx -= rcvd;
939		− xsk->tx_npkts += rcvd;
	1149	+ xsk->ring_stats.tx_npkts += rcvd;
940	1150	}
941	1151	}
942	1152
@@ -949,14 +1159,16 @@ static inline void complete_tx_only(struct xsk_socket_info *xsk, · 6 unmodified lines
949	1159	if (!xsk->outstanding_tx)
950	1160	return;
951	1161
952		− if (!opt_need_wakeup \|\| xsk_ring_prod__needs_wakeup(&xsk->tx))
	1162	+ if (!opt_need_wakeup \|\| xsk_ring_prod__needs_wakeup(&xsk->tx)) {
	1163	+ xsk->app_stats.tx_wakeup_sendtos++;
953	1164	kick_tx(xsk);
	1165	+ }
954	1166
955	1167	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
956	1168	if (rcvd > 0) {
957	1169	xsk_ring_cons__release(&xsk->umem->cq, rcvd);
958	1170	xsk->outstanding_tx -= rcvd;
959		− xsk->tx_npkts += rcvd;
	1171	+ xsk->ring_stats.tx_npkts += rcvd;
960	1172	}
961	1173	}
962	1174
@@ -968,8 +1180,10 @@ static void rx_drop(struct xsk_socket_info xsk, struct pollfd fds) · 5 unmodified lines
968	1180
969	1181	rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);
970	1182	if (!rcvd) {
971		− if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
	1183	+ if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
	1184	+ xsk->app_stats.rx_empty_polls++;
972	1185	ret = poll(fds, num_socks, opt_timeout);
	1186	+ }
973	1187	return;
974	1188	}
975	1189
@@ -977,8 +1191,10 @@ static void rx_drop(struct xsk_socket_info xsk, struct pollfd fds) · 1 unmodified line
977	1191	while (ret != rcvd) {
978	1192	if (ret < 0)
979	1193	exit_with_error(-ret);
980		− if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
	1194	+ if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
	1195	+ xsk->app_stats.fill_fail_polls++;
981	1196	ret = poll(fds, num_socks, opt_timeout);
	1197	+ }
982	1198	ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
983	1199	}
984	1200
@@ -996,7 +1212,7 @@ static void rx_drop(struct xsk_socket_info xsk, struct pollfd fds) · 11 unmodified lines
996	1212
997	1213	xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
998	1214	xsk_ring_cons__release(&xsk->rx, rcvd);
999		− xsk->rx_npkts += rcvd;
	1215	+ xsk->ring_stats.rx_npkts += rcvd;
1000	1216	}
1001	1217
1002	1218	static void rx_drop_all(void)
@@ -1011,6 +1227,8 @@ static void rx_drop_all(void) · 8 unmodified lines
1011	1227
1012	1228	for (;;) {
1013	1229	if (opt_poll) {
	1230	+ for (i = 0; i < num_socks; i++)
	1231	+ xsks[i]->app_stats.opt_polls++;
1014	1232	ret = poll(fds, num_socks, opt_timeout);
1015	1233	if (ret <= 0)
1016	1234	continue;
@@ -1091,6 +1309,8 @@ static void tx_only_all(void) · 74 unmodified lines
1091	1309	int batch_size = get_batch_size(pkt_cnt);
1092	1310
1093	1311	if (opt_poll) {
	1312	+ for (i = 0; i < num_socks; i++)
	1313	+ xsks[i]->app_stats.opt_polls++;
1094	1314	ret = poll(fds, num_socks, opt_timeout);
1095	1315	if (ret <= 0)
1096	1316	continue;
@@ -1122,8 +1342,10 @@ static void l2fwd(struct xsk_socket_info xsk, struct pollfd fds) · 25 unmodified lines
1122	1342
1123	1343	rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);
1124	1344	if (!rcvd) {
1125		− if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
	1345	+ if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
	1346	+ xsk->app_stats.rx_empty_polls++;
1126	1347	ret = poll(fds, num_socks, opt_timeout);
	1348	+ }
1127	1349	return;
1128	1350	}
1129	1351
@@ -1132,8 +1354,10 @@ static void l2fwd(struct xsk_socket_info xsk, struct pollfd fds) · 2 unmodified lines
1132	1354	if (ret < 0)
1133	1355	exit_with_error(-ret);
1134	1356	complete_tx_l2fwd(xsk, fds);
1135		− if (xsk_ring_prod__needs_wakeup(&xsk->tx))
	1357	+ if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
	1358	+ xsk->app_stats.tx_wakeup_sendtos++;
1136	1359	kick_tx(xsk);
	1360	+ }
1137	1361	ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
1138	1362	}
1139	1363
@@ -1155,7 +1379,7 @@ static void l2fwd(struct xsk_socket_info xsk, struct pollfd fds) · 15 unmodified lines
1155	1379	xsk_ring_prod__submit(&xsk->tx, rcvd);
1156	1380	xsk_ring_cons__release(&xsk->rx, rcvd);
1157	1381
1158		− xsk->rx_npkts += rcvd;
	1382	+ xsk->ring_stats.rx_npkts += rcvd;
1159	1383	xsk->outstanding_tx += rcvd;
1160	1384	}
1161	1385
@@ -1171,6 +1395,8 @@ static void l2fwd_all(void) · 9 unmodified lines
1171	1395
1172	1396	for (;;) {
1173	1397	if (opt_poll) {
	1398	+ for (i = 0; i < num_socks; i++)
	1399	+ xsks[i]->app_stats.opt_polls++;
1174	1400	ret = poll(fds, num_socks, opt_timeout);
1175	1401	if (ret <= 0)
1176	1402	continue;

tools/include/uapi/linux/bpf.h+87−14

@@ -356,18 +356,36 @@ enum bpf_link_type { · 355 unmodified lines
356	356	#define BPF_F_SLEEPABLE (1U << 4)
357	357
358	358	/* When BPF ldimm64's insn[0].src_reg != 0 then this can have
359		− * two extensions:
360		− *
361		− * insn[0].src_reg: BPF_PSEUDO_MAP_FD BPF_PSEUDO_MAP_VALUE
362		− * insn[0].imm: map fd map fd
363		− * insn[1].imm: 0 offset into value
364		− * insn[0].off: 0 0
365		− * insn[1].off: 0 0
366		− * ldimm64 rewrite: address of map address of map[0]+offset
367		− * verifier type: CONST_PTR_TO_MAP PTR_TO_MAP_VALUE
	359	+ * the following extensions:
	360	+ *
	361	+ * insn[0].src_reg: BPF_PSEUDO_MAP_FD
	362	+ * insn[0].imm: map fd
	363	+ * insn[1].imm: 0
	364	+ * insn[0].off: 0
	365	+ * insn[1].off: 0
	366	+ * ldimm64 rewrite: address of map
	367	+ * verifier type: CONST_PTR_TO_MAP
368	368	*/
369	369	#define BPF_PSEUDO_MAP_FD 1
	370	+/* insn[0].src_reg: BPF_PSEUDO_MAP_VALUE
	371	+ * insn[0].imm: map fd
	372	+ * insn[1].imm: offset into value
	373	+ * insn[0].off: 0
	374	+ * insn[1].off: 0
	375	+ * ldimm64 rewrite: address of map[0]+offset
	376	+ * verifier type: PTR_TO_MAP_VALUE
	377	+ */
370	378	#define BPF_PSEUDO_MAP_VALUE 2
	379	+/* insn[0].src_reg: BPF_PSEUDO_BTF_ID
	380	+ * insn[0].imm: kernel btd id of VAR
	381	+ * insn[1].imm: 0
	382	+ * insn[0].off: 0
	383	+ * insn[1].off: 0
	384	+ * ldimm64 rewrite: address of the kernel variable
	385	+ * verifier type: PTR_TO_BTF_ID or PTR_TO_MEM, depending on whether the var
	386	+ * is struct/union.
	387	+ */
	388	+#define BPF_PSEUDO_BTF_ID 3
371	389
372	390	/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
373	391	* offset to another bpf function
@@ -417,6 +435,9 @@ enum { · 43 unmodified lines
417	435
418	436	/* Share perf_event among processes */
419	437	BPF_F_PRESERVE_ELEMS = (1U << 11),
	438	+
	439	+/* Create a map that is suitable to be an inner map with dynamic max entries */
	440	+ BPF_F_INNER_MAP = (1U << 12),
420	441	};
421	442
422	443	/* Flags for BPF_PROG_QUERY. */
@@ -1680,7 +1701,7 @@ union bpf_attr { · 1257 unmodified lines
1680	1701	* TCP_CONGESTION, TCP_BPF_IW,
1681	1702	* TCP_BPF_SNDCWND_CLAMP, TCP_SAVE_SYN,
1682	1703	* TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT,
1683		− * TCP_SYNCNT, TCP_USER_TIMEOUT.
	1704	+ * TCP_SYNCNT, TCP_USER_TIMEOUT, TCP_NOTSENT_LOWAT.
1684	1705	* * IPPROTO_IP, which supports optname IP_TOS.
1685	1706	* * IPPROTO_IPV6, which supports optname IPV6_TCLASS.
1686	1707	* Return
@@ -2235,7 +2256,7 @@ union bpf_attr { · 548 unmodified lines
2235	2256	* Description
2236	2257	* This helper is used in programs implementing policies at the
2237	2258	* skb socket level. If the sk_buff skb is allowed to pass (i.e.
2238		− * if the verdeict eBPF program returns SK_PASS), redirect it
	2259	+ * if the verdict eBPF program returns SK_PASS), redirect it
2239	2260	* to the socket referenced by map (of type
2240	2261	* BPF_MAP_TYPE_SOCKHASH) using hash key. Both ingress and
2241	2262	* egress interfaces can be used for redirection. The
@@ -3661,10 +3682,59 @@ union bpf_attr { · 1419 unmodified lines
3661	3682	* Redirect the packet to another net device of index ifindex
3662	3683	* and fill in L2 addresses from neighboring subsystem. This helper
3663	3684	* is somewhat similar to bpf_redirect\ (), except that it
3664		− * fills in e.g. MAC addresses based on the L3 information from
3665		− * the packet. This helper is supported for IPv4 and IPv6 protocols.
	3685	+ * populates L2 addresses as well, meaning, internally, the helper
	3686	+ * performs a FIB lookup based on the skb's networking header to
	3687	+ * get the address of the next hop and then relies on the neighbor
	3688	+ * lookup for the L2 address of the nexthop.
	3689	+ *
	3690	+ * The flags argument is reserved and must be 0. The helper is
	3691	+ * currently only supported for tc BPF program types, and enabled
	3692	+ * for IPv4 and IPv6 protocols.
	3693	+ * Return
	3694	+ * The helper returns TC_ACT_REDIRECT on success or
	3695	+ * TC_ACT_SHOT on error.
	3696	+ *
	3697	+ * void bpf_per_cpu_ptr(const void percpu_ptr, u32 cpu)
	3698	+ * Description
	3699	+ * Take a pointer to a percpu ksym, percpu_ptr, and return a
	3700	+ * pointer to the percpu kernel variable on cpu. A ksym is an
	3701	+ * extern variable decorated with '__ksym'. For ksym, there is a
	3702	+ * global var (either static or global) defined of the same name
	3703	+ * in the kernel. The ksym is percpu if the global var is percpu.
	3704	+ * The returned pointer points to the global percpu var on cpu.
	3705	+ *
	3706	+ * bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the
	3707	+ * kernel, except that bpf_per_cpu_ptr() may return NULL. This
	3708	+ * happens if cpu is larger than nr_cpu_ids. The caller of
	3709	+ * bpf_per_cpu_ptr() must check the returned value.
	3710	+ * Return
	3711	+ * A pointer pointing to the kernel percpu variable on cpu, or
	3712	+ * NULL, if cpu is invalid.
	3713	+ *
	3714	+ * void bpf_this_cpu_ptr(const void percpu_ptr)
	3715	+ * Description
	3716	+ * Take a pointer to a percpu ksym, percpu_ptr, and return a
	3717	+ * pointer to the percpu kernel variable on this cpu. See the
	3718	+ * description of 'ksym' in bpf_per_cpu_ptr\ ().
	3719	+ *
	3720	+ * bpf_this_cpu_ptr() has the same semantic as this_cpu_ptr() in
	3721	+ * the kernel. Different from bpf_per_cpu_ptr\ (), it would
	3722	+ * never return NULL.
	3723	+ * Return
	3724	+ * A pointer pointing to the kernel percpu variable on this cpu.
	3725	+ *
	3726	+ * long bpf_redirect_peer(u32 ifindex, u64 flags)
	3727	+ * Description
	3728	+ * Redirect the packet to another net device of index ifindex.
	3729	+ * This helper is somewhat similar to bpf_redirect\ (), except
	3730	+ * that the redirection happens to the ifindex' peer device and
	3731	+ * the netns switch takes place from ingress to ingress without
	3732	+ * going through the CPU's backlog queue.
	3733	+ *
3666	3734	* The flags argument is reserved and must be 0. The helper is
3667		− * currently only supported for tc BPF program types.
	3735	+ * currently only supported for tc BPF program types at the ingress
	3736	+ * hook and for veth device types. The peer device must reside in a
	3737	+ * different network namespace.
3668	3738	* Return
3669	3739	* The helper returns TC_ACT_REDIRECT on success or
3670	3740	* TC_ACT_SHOT on error.
@@ -3823,6 +3893,9 @@ union bpf_attr { · 152 unmodified lines
3823	3893	FN(seq_printf_btf), \
3824	3894	FN(skb_cgroup_classid), \
3825	3895	FN(redirect_neigh), \
	3896	+ FN(bpf_per_cpu_ptr), \
	3897	+ FN(bpf_this_cpu_ptr), \
	3898	+ FN(redirect_peer), \
3826	3899	/* */
3827	3900
3828	3901	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

tools/lib/bpf/libbpf.c+289−59

@@ -390,6 +390,12 @@ struct extern_desc { · 389 unmodified lines
390	390	} kcfg;
391	391	struct {
392	392	unsigned long long addr;
	393	+
	394	+ /* target btf_id of the corresponding kernel var. */
	395	+ int vmlinux_btf_id;
	396	+
	397	+ /* local btf_id of the ksym extern's type. */
	398	+ __u32 type_id;
393	399	} ksym;
394	400	};
395	401	};
@@ -2522,12 +2528,23 @@ static int bpf_object__load_vmlinux_btf(struct bpf_object *obj) · 2126 unmodified lines
2522	2528	{
2523	2529	bool need_vmlinux_btf = false;
2524	2530	struct bpf_program *prog;
2525		− int err;
	2531	+ int i, err;
2526	2532
2527	2533	/* CO-RE relocations need kernel BTF */
2528	2534	if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
2529	2535	need_vmlinux_btf = true;
2530	2536
	2537	+ /* Support for typed ksyms needs kernel BTF */
	2538	+ for (i = 0; i < obj->nr_extern; i++) {
	2539	+ const struct extern_desc *ext;
	2540	+
	2541	+ ext = &obj->externs[i];
	2542	+ if (ext->type == EXT_KSYM && ext->ksym.type_id) {
	2543	+ need_vmlinux_btf = true;
	2544	+ break;
	2545	+ }
	2546	+ }
	2547	+
2531	2548	bpf_object__for_each_program(prog, obj) {
2532	2549	if (!prog->load)
2533	2550	continue;
@@ -3156,16 +3173,10 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 622 unmodified lines
3156	3173	return -ENOTSUP;
3157	3174	}
3158	3175	} else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3159		− const struct btf_type *vt;
3160		−
3161	3176	ksym_sec = sec;
3162	3177	ext->type = EXT_KSYM;
3163		−
3164		− vt = skip_mods_and_typedefs(obj->btf, t->type, NULL);
3165		− if (!btf_is_void(vt)) {
3166		− pr_warn("extern (ksym) '%s' is not typeless (void)\n", ext_name);
3167		− return -ENOTSUP;
3168		− }
	3178	+ skip_mods_and_typedefs(obj->btf, t->type,
	3179	+ &ext->ksym.type_id);
3169	3180	} else {
3170	3181	pr_warn("unrecognized extern section '%s'\n", sec_name);
3171	3182	return -ENOTSUP;
@@ -4192,6 +4203,36 @@ static int bpf_object__create_map(struct bpf_object obj, struct bpf_map map) · 1020 unmodified lines
4192	4203	return 0;
4193	4204	}
4194	4205
	4206	+static int init_map_slots(struct bpf_map *map)
	4207	+{
	4208	+ const struct bpf_map *targ_map;
	4209	+ unsigned int i;
	4210	+ int fd, err;
	4211	+
	4212	+ for (i = 0; i < map->init_slots_sz; i++) {
	4213	+ if (!map->init_slots[i])
	4214	+ continue;
	4215	+
	4216	+ targ_map = map->init_slots[i];
	4217	+ fd = bpf_map__fd(targ_map);
	4218	+ err = bpf_map_update_elem(map->fd, &i, &fd, 0);
	4219	+ if (err) {
	4220	+ err = -errno;
	4221	+ pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
	4222	+ map->name, i, targ_map->name,
	4223	+ fd, err);
	4224	+ return err;
	4225	+ }
	4226	+ pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
	4227	+ map->name, i, targ_map->name, fd);
	4228	+ }
	4229	+
	4230	+ zfree(&map->init_slots);
	4231	+ map->init_slots_sz = 0;
	4232	+
	4233	+ return 0;
	4234	+}
	4235	+
4195	4236	static int
4196	4237	bpf_object__create_maps(struct bpf_object *obj)
4197	4238	{
@@ -4215,47 +4256,29 @@ bpf_object__create_maps(struct bpf_object *obj) · 17 unmodified lines
4215	4256	if (map->fd >= 0) {
4216	4257	pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4217	4258	map->name, map->fd);
4218		− continue;
4219		− }
4220		−
4221		− err = bpf_object__create_map(obj, map);
4222		− if (err)
4223		− goto err_out;
4224		−
4225		− pr_debug("map '%s': created successfully, fd=%d\n", map->name,
4226		− map->fd);
4227		−
4228		− if (bpf_map__is_internal(map)) {
4229		− err = bpf_object__populate_internal_map(obj, map);
4230		− if (err < 0) {
4231		− zclose(map->fd);
	4259	+ } else {
	4260	+ err = bpf_object__create_map(obj, map);
	4261	+ if (err)
4232	4262	goto err_out;
4233		− }
4234		− }
4235	4263
4236		− if (map->init_slots_sz) {
4237		− for (j = 0; j < map->init_slots_sz; j++) {
4238		− const struct bpf_map *targ_map;
4239		− int fd;
	4264	+ pr_debug("map '%s': created successfully, fd=%d\n",
	4265	+ map->name, map->fd);
4240	4266
4241		− if (!map->init_slots[j])
4242		− continue;
	4267	+ if (bpf_map__is_internal(map)) {
	4268	+ err = bpf_object__populate_internal_map(obj, map);
	4269	+ if (err < 0) {
	4270	+ zclose(map->fd);
	4271	+ goto err_out;
	4272	+ }
	4273	+ }
4243	4274
4244		− targ_map = map->init_slots[j];
4245		− fd = bpf_map__fd(targ_map);
4246		− err = bpf_map_update_elem(map->fd, &j, &fd, 0);
4247		− if (err) {
4248		− err = -errno;
4249		− pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4250		− map->name, j, targ_map->name,
4251		− fd, err);
	4275	+ if (map->init_slots_sz) {
	4276	+ err = init_map_slots(map);
	4277	+ if (err < 0) {
	4278	+ zclose(map->fd);
4252	4279	goto err_out;
4253	4280	}
4254		− pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4255		− map->name, j, targ_map->name, fd);
4256	4281	}
4257		− zfree(&map->init_slots);
4258		− map->init_slots_sz = 0;
4259	4282	}
4260	4283
4261	4284	if (map->pin_path && !map->pinned) {
@@ -5017,16 +5040,19 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, · 755 unmodified lines
5017	5040	static int bpf_core_calc_field_relo(const struct bpf_program *prog,
5018	5041	const struct bpf_core_relo *relo,
5019	5042	const struct bpf_core_spec *spec,
5020		− __u32 val, bool validate)
	5043	+ __u32 val, __u32 field_sz, __u32 *type_id,
	5044	+ bool *validate)
5021	5045	{
5022	5046	const struct bpf_core_accessor *acc;
5023	5047	const struct btf_type *t;
5024		− __u32 byte_off, byte_sz, bit_off, bit_sz;
	5048	+ __u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
5025	5049	const struct btf_member *m;
5026	5050	const struct btf_type *mt;
5027	5051	bool bitfield;
5028	5052	__s64 sz;
5029	5053
	5054	+ *field_sz = 0;
	5055	+
5030	5056	if (relo->kind == BPF_FIELD_EXISTS) {
5031	5057	*val = spec ? 1 : 0;
5032	5058	return 0;
@@ -5042,6 +5068,12 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 9 unmodified lines
5042	5068	if (!acc->name) {
5043	5069	if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
5044	5070	*val = spec->bit_offset / 8;
	5071	+ /* remember field size for load/store mem size */
	5072	+ sz = btf__resolve_size(spec->btf, acc->type_id);
	5073	+ if (sz < 0)
	5074	+ return -EINVAL;
	5075	+ *field_sz = sz;
	5076	+ *type_id = acc->type_id;
5045	5077	} else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
5046	5078	sz = btf__resolve_size(spec->btf, acc->type_id);
5047	5079	if (sz < 0)
@@ -5058,7 +5090,7 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 10 unmodified lines
5058	5090	}
5059	5091
5060	5092	m = btf_members(t) + acc->idx;
5061		− mt = skip_mods_and_typedefs(spec->btf, m->type, NULL);
	5093	+ mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
5062	5094	bit_off = spec->bit_offset;
5063	5095	bit_sz = btf_member_bitfield_size(t, acc->idx);
5064	5096
@@ -5078,7 +5110,7 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 13 unmodified lines
5078	5110	byte_off = bit_off / 8 / byte_sz * byte_sz;
5079	5111	}
5080	5112	} else {
5081		− sz = btf__resolve_size(spec->btf, m->type);
	5113	+ sz = btf__resolve_size(spec->btf, field_type_id);
5082	5114	if (sz < 0)
5083	5115	return -EINVAL;
5084	5116	byte_sz = sz;
@@ -5096,6 +5128,10 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 11 unmodified lines
5096	5128	switch (relo->kind) {
5097	5129	case BPF_FIELD_BYTE_OFFSET:
5098	5130	*val = byte_off;
	5131	+ if (!bitfield) {
	5132	+ *field_sz = byte_sz;
	5133	+ *type_id = field_type_id;
	5134	+ }
5099	5135	break;
5100	5136	case BPF_FIELD_BYTE_SIZE:
5101	5137	*val = byte_sz;
@@ -5196,6 +5232,19 @@ struct bpf_core_relo_res · 94 unmodified lines
5196	5232	bool poison;
5197	5233	/* some relocations can't be validated against orig_val */
5198	5234	bool validate;
	5235	+ /* for field byte offset relocations or the forms:
	5236	+ * (T )(rX + <off>) = rY
	5237	+ * rX = (T )(rY + <off>),
	5238	+ * we remember original and resolved field size to adjust direct
	5239	+ * memory loads of pointers and integers; this is necessary for 32-bit
	5240	+ * host kernel architectures, but also allows to automatically
	5241	+ * relocate fields that were resized from, e.g., u32 to u64, etc.
	5242	+ */
	5243	+ bool fail_memsz_adjust;
	5244	+ __u32 orig_sz;
	5245	+ __u32 orig_type_id;
	5246	+ __u32 new_sz;
	5247	+ __u32 new_type_id;
5199	5248	};
5200	5249
5201	5250	/* Calculate original and target relocation values, given local and target
@@ -5217,10 +5266,56 @@ static int bpf_core_calc_relo(const struct bpf_program *prog, · 15 unmodified lines
5217	5266	res->new_val = 0;
5218	5267	res->poison = false;
5219	5268	res->validate = true;
	5269	+ res->fail_memsz_adjust = false;
	5270	+ res->orig_sz = res->new_sz = 0;
	5271	+ res->orig_type_id = res->new_type_id = 0;
5220	5272
5221	5273	if (core_relo_is_field_based(relo->kind)) {
5222		− err = bpf_core_calc_field_relo(prog, relo, local_spec, &res->orig_val, &res->validate);
5223		− err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec, &res->new_val, NULL);
	5274	+ err = bpf_core_calc_field_relo(prog, relo, local_spec,
	5275	+ &res->orig_val, &res->orig_sz,
	5276	+ &res->orig_type_id, &res->validate);
	5277	+ err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec,
	5278	+ &res->new_val, &res->new_sz,
	5279	+ &res->new_type_id, NULL);
	5280	+ if (err)
	5281	+ goto done;
	5282	+ /* Validate if it's safe to adjust load/store memory size.
	5283	+ * Adjustments are performed only if original and new memory
	5284	+ * sizes differ.
	5285	+ */
	5286	+ res->fail_memsz_adjust = false;
	5287	+ if (res->orig_sz != res->new_sz) {
	5288	+ const struct btf_type orig_t, new_t;
	5289	+
	5290	+ orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id);
	5291	+ new_t = btf__type_by_id(targ_spec->btf, res->new_type_id);
	5292	+
	5293	+ /* There are two use cases in which it's safe to
	5294	+ * adjust load/store's mem size:
	5295	+ * - reading a 32-bit kernel pointer, while on BPF
	5296	+ * size pointers are always 64-bit; in this case
	5297	+ * it's safe to "downsize" instruction size due to
	5298	+ * pointer being treated as unsigned integer with
	5299	+ * zero-extended upper 32-bits;
	5300	+ * - reading unsigned integers, again due to
	5301	+ * zero-extension is preserving the value correctly.
	5302	+ *
	5303	+ * In all other cases it's incorrect to attempt to
	5304	+ * load/store field because read value will be
	5305	+ * incorrect, so we poison relocated instruction.
	5306	+ */
	5307	+ if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
	5308	+ goto done;
	5309	+ if (btf_is_int(orig_t) && btf_is_int(new_t) &&
	5310	+ btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
	5311	+ btf_int_encoding(new_t) != BTF_INT_SIGNED)
	5312	+ goto done;
	5313	+
	5314	+ /* mark as invalid mem size adjustment, but this will
	5315	+ * only be checked for LDX/STX/ST insns
	5316	+ */
	5317	+ res->fail_memsz_adjust = true;
	5318	+ }
5224	5319	} else if (core_relo_is_type_based(relo->kind)) {
5225	5320	err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
5226	5321	err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
@@ -5229,6 +5324,7 @@ static int bpf_core_calc_relo(const struct bpf_program *prog, · 2 unmodified lines
5229	5324	err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
5230	5325	}
5231	5326
	5327	+done:
5232	5328	if (err == -EUCLEAN) {
5233	5329	/* EUCLEAN is used to signal instruction poisoning request */
5234	5330	res->poison = true;
@@ -5268,6 +5364,28 @@ static bool is_ldimm64(struct bpf_insn *insn) · 33 unmodified lines
5268	5364	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW);
5269	5365	}
5270	5366
	5367	+static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
	5368	+{
	5369	+ switch (BPF_SIZE(insn->code)) {
	5370	+ case BPF_DW: return 8;
	5371	+ case BPF_W: return 4;
	5372	+ case BPF_H: return 2;
	5373	+ case BPF_B: return 1;
	5374	+ default: return -1;
	5375	+ }
	5376	+}
	5377	+
	5378	+static int insn_bytes_to_bpf_size(__u32 sz)
	5379	+{
	5380	+ switch (sz) {
	5381	+ case 8: return BPF_DW;
	5382	+ case 4: return BPF_W;
	5383	+ case 2: return BPF_H;
	5384	+ case 1: return BPF_B;
	5385	+ default: return -1;
	5386	+ }
	5387	+}
	5388	+
5271	5389	/*
5272	5390	* Patch relocatable BPF instruction.
5273	5391	*
@@ -5277,10 +5395,13 @@ static bool is_ldimm64(struct bpf_insn *insn) · 3 unmodified lines
5277	5395	* spec, and is checked before patching instruction. If actual insn->imm value
5278	5396	* is wrong, bail out with error.
5279	5397	*
5280		− * Currently three kinds of BPF instructions are supported:
	5398	+ * Currently supported classes of BPF instruction are:
5281	5399	* 1. rX = <imm> (assignment with immediate operand);
5282	5400	* 2. rX += <imm> (arithmetic operations with immediate operand);
5283		− * 3. rX = <imm64> (load with 64-bit immediate value).
	5401	+ * 3. rX = <imm64> (load with 64-bit immediate value);
	5402	+ * 4. rX = (T )(rY + <off>), where T is one of {u8, u16, u32, u64};
	5403	+ * 5. (T )(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
	5404	+ * 6. (T )(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
5284	5405	*/
5285	5406	static int bpf_core_patch_insn(struct bpf_program *prog,
5286	5407	const struct bpf_core_relo *relo,
@@ -5304,6 +5425,7 @@ static int bpf_core_patch_insn(struct bpf_program *prog, · 17 unmodified lines
5304	5425	class = BPF_CLASS(insn->code);
5305	5426
5306	5427	if (res->poison) {
	5428	+poison:
5307	5429	/* poison second part of ldimm64 to avoid confusing error from
5308	5430	* verifier about "unknown opcode 00"
5309	5431	*/
@@ -5346,10 +5468,39 @@ static int bpf_core_patch_insn(struct bpf_program *prog, · 36 unmodified lines
5346	5468	prog->name, relo_idx, insn_idx, new_val);
5347	5469	return -ERANGE;
5348	5470	}
	5471	+ if (res->fail_memsz_adjust) {
	5472	+ pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
	5473	+ "Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
	5474	+ prog->name, relo_idx, insn_idx);
	5475	+ goto poison;
	5476	+ }
	5477	+
5349	5478	orig_val = insn->off;
5350	5479	insn->off = new_val;
5351	5480	pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
5352	5481	prog->name, relo_idx, insn_idx, orig_val, new_val);
	5482	+
	5483	+ if (res->new_sz != res->orig_sz) {
	5484	+ int insn_bytes_sz, insn_bpf_sz;
	5485	+
	5486	+ insn_bytes_sz = insn_bpf_size_to_bytes(insn);
	5487	+ if (insn_bytes_sz != res->orig_sz) {
	5488	+ pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
	5489	+ prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
	5490	+ return -EINVAL;
	5491	+ }
	5492	+
	5493	+ insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
	5494	+ if (insn_bpf_sz < 0) {
	5495	+ pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
	5496	+ prog->name, relo_idx, insn_idx, res->new_sz);
	5497	+ return -EINVAL;
	5498	+ }
	5499	+
	5500	+ insn->code = BPF_MODE(insn->code) \| insn_bpf_sz \| BPF_CLASS(insn->code);
	5501	+ pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
	5502	+ prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
	5503	+ }
5353	5504	break;
5354	5505	case BPF_LD: {
5355	5506	__u64 imm;
@@ -5691,7 +5842,7 @@ bpf_object__relocate_core(struct bpf_object obj, const char targ_btf_path) · 335 unmodified lines
5691	5842	return 0;
5692	5843
5693	5844	if (targ_btf_path)
5694		− targ_btf = btf__parse_elf(targ_btf_path, NULL);
	5845	+ targ_btf = btf__parse(targ_btf_path, NULL);
5695	5846	else
5696	5847	targ_btf = obj->btf_vmlinux;
5697	5848	if (IS_ERR_OR_NULL(targ_btf)) {
@@ -5742,6 +5893,11 @@ bpf_object__relocate_core(struct bpf_object obj, const char targ_btf_path) · 44 unmodified lines
5742	5893	err = -EINVAL;
5743	5894	goto out;
5744	5895	}
	5896	+ /* no need to apply CO-RE relocation if the program is
	5897	+ * not going to be loaded
	5898	+ */
	5899	+ if (!prog->load)
	5900	+ continue;
5745	5901
5746	5902	err = bpf_core_apply_relo(prog, rec, i, obj->btf,
5747	5903	targ_btf, cand_cache);
@@ -5800,8 +5956,13 @@ bpf_object__relocate_data(struct bpf_object obj, struct bpf_program prog) · 52 unmodified lines
5800	5956	insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5801	5957	insn[1].imm = ext->kcfg.data_off;
5802	5958	} else /* EXT_KSYM */ {
5803		− insn[0].imm = (__u32)ext->ksym.addr;
5804		− insn[1].imm = ext->ksym.addr >> 32;
	5959	+ if (ext->ksym.type_id) { /* typed ksyms */
	5960	+ insn[0].src_reg = BPF_PSEUDO_BTF_ID;
	5961	+ insn[0].imm = ext->ksym.vmlinux_btf_id;
	5962	+ } else { /* typeless ksyms */
	5963	+ insn[0].imm = (__u32)ext->ksym.addr;
	5964	+ insn[1].imm = ext->ksym.addr >> 32;
	5965	+ }
5805	5966	}
5806	5967	relo->processed = true;
5807	5968	break;
@@ -6933,10 +7094,72 @@ out: · 1125 unmodified lines
6933	7094	return err;
6934	7095	}
6935	7096
	7097	+static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
	7098	+{
	7099	+ struct extern_desc *ext;
	7100	+ int i, id;
	7101	+
	7102	+ for (i = 0; i < obj->nr_extern; i++) {
	7103	+ const struct btf_type targ_var, targ_type;
	7104	+ __u32 targ_type_id, local_type_id;
	7105	+ const char *targ_var_name;
	7106	+ int ret;
	7107	+
	7108	+ ext = &obj->externs[i];
	7109	+ if (ext->type != EXT_KSYM \|\| !ext->ksym.type_id)
	7110	+ continue;
	7111	+
	7112	+ id = btf__find_by_name_kind(obj->btf_vmlinux, ext->name,
	7113	+ BTF_KIND_VAR);
	7114	+ if (id <= 0) {
	7115	+ pr_warn("extern (ksym) '%s': failed to find BTF ID in vmlinux BTF.\n",
	7116	+ ext->name);
	7117	+ return -ESRCH;
	7118	+ }
	7119	+
	7120	+ /* find local type_id */
	7121	+ local_type_id = ext->ksym.type_id;
	7122	+
	7123	+ /* find target type_id */
	7124	+ targ_var = btf__type_by_id(obj->btf_vmlinux, id);
	7125	+ targ_var_name = btf__name_by_offset(obj->btf_vmlinux,
	7126	+ targ_var->name_off);
	7127	+ targ_type = skip_mods_and_typedefs(obj->btf_vmlinux,
	7128	+ targ_var->type,
	7129	+ &targ_type_id);
	7130	+
	7131	+ ret = bpf_core_types_are_compat(obj->btf, local_type_id,
	7132	+ obj->btf_vmlinux, targ_type_id);
	7133	+ if (ret <= 0) {
	7134	+ const struct btf_type *local_type;
	7135	+ const char targ_name, local_name;
	7136	+
	7137	+ local_type = btf__type_by_id(obj->btf, local_type_id);
	7138	+ local_name = btf__name_by_offset(obj->btf,
	7139	+ local_type->name_off);
	7140	+ targ_name = btf__name_by_offset(obj->btf_vmlinux,
	7141	+ targ_type->name_off);
	7142	+
	7143	+ pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
	7144	+ ext->name, local_type_id,
	7145	+ btf_kind_str(local_type), local_name, targ_type_id,
	7146	+ btf_kind_str(targ_type), targ_name);
	7147	+ return -EINVAL;
	7148	+ }
	7149	+
	7150	+ ext->is_set = true;
	7151	+ ext->ksym.vmlinux_btf_id = id;
	7152	+ pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",
	7153	+ ext->name, id, btf_kind_str(targ_var), targ_var_name);
	7154	+ }
	7155	+ return 0;
	7156	+}
	7157	+
6936	7158	static int bpf_object__resolve_externs(struct bpf_object *obj,
6937	7159	const char *extra_kconfig)
6938	7160	{
6939	7161	bool need_config = false, need_kallsyms = false;
	7162	+ bool need_vmlinux_btf = false;
6940	7163	struct extern_desc *ext;
6941	7164	void *kcfg_data = NULL;
6942	7165	int err, i;
@@ -6967,7 +7190,10 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, · 24 unmodified lines
6967	7190	strncmp(ext->name, "CONFIG_", 7) == 0) {
6968	7191	need_config = true;
6969	7192	} else if (ext->type == EXT_KSYM) {
6970		− need_kallsyms = true;
	7193	+ if (ext->ksym.type_id)
	7194	+ need_vmlinux_btf = true;
	7195	+ else
	7196	+ need_kallsyms = true;
6971	7197	} else {
6972	7198	pr_warn("unrecognized extern '%s'\n", ext->name);
6973	7199	return -EINVAL;
@@ -6996,6 +7222,11 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, · 22 unmodified lines
6996	7222	if (err)
6997	7223	return -EINVAL;
6998	7224	}
	7225	+ if (need_vmlinux_btf) {
	7226	+ err = bpf_object__resolve_ksyms_btf_id(obj);
	7227	+ if (err)
	7228	+ return -EINVAL;
	7229	+ }
6999	7230	for (i = 0; i < obj->nr_extern; i++) {
7000	7231	ext = &obj->externs[i];
7001	7232
@@ -7028,10 +7259,10 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) · 26 unmodified lines
7028	7259	}
7029	7260
7030	7261	err = bpf_object__probe_loading(obj);
	7262	+ err = err ? : bpf_object__load_vmlinux_btf(obj);
7031	7263	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7032	7264	err = err ? : bpf_object__sanitize_and_load_btf(obj);
7033	7265	err = err ? : bpf_object__sanitize_maps(obj);
7034		− err = err ? : bpf_object__load_vmlinux_btf(obj);
7035	7266	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7036	7267	err = err ? : bpf_object__create_maps(obj);
7037	7268	err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
@@ -10353,9 +10584,8 @@ int bpf_program__set_attach_target(struct bpf_program *prog, · 3315 unmodified lines
10353	10584	btf_id = libbpf_find_prog_btf_id(attach_func_name,
10354	10585	attach_prog_fd);
10355	10586	else
10356		− btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,
10357		− attach_func_name,
10358		− prog->expected_attach_type);
	10587	+ btf_id = libbpf_find_vmlinux_btf_id(attach_func_name,
	10588	+ prog->expected_attach_type);
10359	10589
10360	10590	if (btf_id < 0)
10361	10591	return btf_id;

tools/lib/bpf/xsk.c+6−1

@@ -705,7 +705,7 @@ int xsk_socket__create_shared(struct xsk_socket **xsk_ptr, · 704 unmodified lines
705	705	struct xsk_ctx *ctx;
706	706	int err, ifindex;
707	707
708		− if (!umem \|\| !xsk_ptr \|\| !(rx \|\| tx) \|\| !fill \|\| !comp)
	708	+ if (!umem \|\| !xsk_ptr \|\| !(rx \|\| tx))
709	709	return -EFAULT;
710	710
711	711	xsk = calloc(1, sizeof(*xsk));
@@ -735,6 +735,11 @@ int xsk_socket__create_shared(struct xsk_socket **xsk_ptr, · 23 unmodified lines
735	735
736	736	ctx = xsk_get_ctx(umem, ifindex, queue_id);
737	737	if (!ctx) {
	738	+ if (!fill \|\| !comp) {
	739	+ err = -EFAULT;
	740	+ goto out_socket;
	741	+ }
	742	+
738	743	ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
739	744	fill, comp);
740	745	if (!ctx) {

tools/testing/selftests/bpf/README.rst+38−0

@@ -7,6 +7,44 @@ General instructions on running selftests can be found in · 6 unmodified lines
7	7	Additional information about selftest failures are
8	8	documented here.
9	9
	10	+profiler[23] test failures with clang/llvm <12.0.0
	11	+==================================================
	12	+
	13	+With clang/llvm <12.0.0, the profiler[23] test may fail.
	14	+The symptom looks like
	15	+
	16	+.. code-block:: c
	17	+
	18	+ // r9 is a pointer to map_value
	19	+ // r7 is a scalar
	20	+ 17: bf 96 00 00 00 00 00 00 r6 = r9
	21	+ 18: 0f 76 00 00 00 00 00 00 r6 += r7
	22	+ math between map_value pointer and register with unbounded min value is not allowed
	23	+
	24	+ // the instructions below will not be seen in the verifier log
	25	+ 19: a5 07 01 00 01 01 00 00 if r7 < 257 goto +1
	26	+ 20: bf 96 00 00 00 00 00 00 r6 = r9
	27	+ // r6 is used here
	28	+
	29	+The verifier will reject such code with above error.
	30	+At insn 18 the r7 is indeed unbounded. The later insn 19 checks the bounds and
	31	+the insn 20 undoes map_value addition. It is currently impossible for the
	32	+verifier to understand such speculative pointer arithmetic.
	33	+Hence
	34	+ https://reviews.llvm.org/D85570
	35	+addresses it on the compiler side. It was committed on llvm 12.
	36	+
	37	+The corresponding C code
	38	+.. code-block:: c
	39	+
	40	+ for (int i = 0; i < MAX_CGROUPS_PATH_DEPTH; i++) {
	41	+ filepart_length = bpf_probe_read_str(payload, ...);
	42	+ if (filepart_length <= MAX_PATH) {
	43	+ barrier_var(filepart_length); // workaround
	44	+ payload += filepart_length;
	45	+ }
	46	+ }
	47	+
10	48	bpf_iter test failures with clang/llvm 10.0.0
11	49	=============================================
12	50

tools/testing/selftests/bpf/prog_tests/align.c+8−8

@@ -195,13 +195,13 @@ static struct bpf_align_test tests[] = { · 194 unmodified lines
195	195	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
196	196	.matches = {
197	197	{7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
198		− {8, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	198	+ {8, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
199	199	{9, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
200		− {10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	200	+ {10, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
201	201	{11, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
202		− {12, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	202	+ {12, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
203	203	{13, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
204		− {14, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
	204	+ {14, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
205	205	{15, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
206	206	{16, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
207	207	},
@@ -518,7 +518,7 @@ static struct bpf_align_test tests[] = { · 310 unmodified lines
518	518	* the total offset is 4-byte aligned and meets the
519	519	* load's requirements.
520	520	*/
521		− {20, "R5=pkt(id=1,off=0,r=4,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc)"},
	521	+ {20, "R5=pkt(id=2,off=0,r=4,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc)"},
522	522
523	523	},
524	524	},
@@ -561,18 +561,18 @@ static struct bpf_align_test tests[] = { · 36 unmodified lines
561	561	/* Adding 14 makes R6 be (4n+2) */
562	562	{11, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"},
563	563	/* Subtracting from packet pointer overflows ubounds */
564		− {13, "R5_w=pkt(id=1,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"},
	564	+ {13, "R5_w=pkt(id=2,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"},
565	565	/* New unknown value in R7 is (4n), >= 76 */
566	566	{15, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"},
567	567	/* Adding it to packet pointer gives nice bounds again */
568		− {16, "R5_w=pkt(id=2,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},
	568	+ {16, "R5_w=pkt(id=3,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},
569	569	/* At the time the word size load is performed from R5,
570	570	* its total fixed offset is NET_IP_ALIGN + reg->off (0)
571	571	* which is 2. Then the variable offset is (4n+2), so
572	572	* the total offset is 4-byte aligned and meets the
573	573	* load's requirements.
574	574	*/
575		− {20, "R5=pkt(id=2,off=0,r=4,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},
	575	+ {20, "R5=pkt(id=3,off=0,r=4,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},
576	576	},
577	577	},
578	578	};

tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c+29−10

@@ -55,10 +55,10 @@ static int kern_sync_rcu(void) · 54 unmodified lines
55	55
56	56	static void test_lookup_update(void)
57	57	{
58		− int err, key = 0, val, i;
	58	+ int map1_fd, map2_fd, map3_fd, map4_fd, map5_fd, map1_id, map2_id;
	59	+ int outer_arr_fd, outer_hash_fd, outer_arr_dyn_fd;
59	60	struct test_btf_map_in_map *skel;
60		− int outer_arr_fd, outer_hash_fd;
61		− int fd, map1_fd, map2_fd, map1_id, map2_id;
	61	+ int err, key = 0, val, i, fd;
62	62
63	63	skel = test_btf_map_in_map__open_and_load();
64	64	if (CHECK(!skel, "skel_open", "failed to open&load skeleton\n"))
@@ -70,32 +70,45 @@ static void test_lookup_update(void) · 5 unmodified lines
70	70
71	71	map1_fd = bpf_map__fd(skel->maps.inner_map1);
72	72	map2_fd = bpf_map__fd(skel->maps.inner_map2);
	73	+ map3_fd = bpf_map__fd(skel->maps.inner_map3);
	74	+ map4_fd = bpf_map__fd(skel->maps.inner_map4);
	75	+ map5_fd = bpf_map__fd(skel->maps.inner_map5);
	76	+ outer_arr_dyn_fd = bpf_map__fd(skel->maps.outer_arr_dyn);
73	77	outer_arr_fd = bpf_map__fd(skel->maps.outer_arr);
74	78	outer_hash_fd = bpf_map__fd(skel->maps.outer_hash);
75	79
76		− /* inner1 = input, inner2 = input + 1 */
77		− map1_fd = bpf_map__fd(skel->maps.inner_map1);
	80	+ /* inner1 = input, inner2 = input + 1, inner3 = input + 2 */
78	81	bpf_map_update_elem(outer_arr_fd, &key, &map1_fd, 0);
79		− map2_fd = bpf_map__fd(skel->maps.inner_map2);
80	82	bpf_map_update_elem(outer_hash_fd, &key, &map2_fd, 0);
	83	+ bpf_map_update_elem(outer_arr_dyn_fd, &key, &map3_fd, 0);
81	84	skel->bss->input = 1;
82	85	usleep(1);
83		−
84	86	bpf_map_lookup_elem(map1_fd, &key, &val);
85	87	CHECK(val != 1, "inner1", "got %d != exp %d\n", val, 1);
86	88	bpf_map_lookup_elem(map2_fd, &key, &val);
87	89	CHECK(val != 2, "inner2", "got %d != exp %d\n", val, 2);
	90	+ bpf_map_lookup_elem(map3_fd, &key, &val);
	91	+ CHECK(val != 3, "inner3", "got %d != exp %d\n", val, 3);
88	92
89		− /* inner1 = input + 1, inner2 = input */
	93	+ /* inner2 = input, inner1 = input + 1, inner4 = input + 2 */
90	94	bpf_map_update_elem(outer_arr_fd, &key, &map2_fd, 0);
91	95	bpf_map_update_elem(outer_hash_fd, &key, &map1_fd, 0);
	96	+ bpf_map_update_elem(outer_arr_dyn_fd, &key, &map4_fd, 0);
92	97	skel->bss->input = 3;
93	98	usleep(1);
94		−
95	99	bpf_map_lookup_elem(map1_fd, &key, &val);
96	100	CHECK(val != 4, "inner1", "got %d != exp %d\n", val, 4);
97	101	bpf_map_lookup_elem(map2_fd, &key, &val);
98	102	CHECK(val != 3, "inner2", "got %d != exp %d\n", val, 3);
	103	+ bpf_map_lookup_elem(map4_fd, &key, &val);
	104	+ CHECK(val != 5, "inner4", "got %d != exp %d\n", val, 5);
	105	+
	106	+ /* inner5 = input + 2 */
	107	+ bpf_map_update_elem(outer_arr_dyn_fd, &key, &map5_fd, 0);
	108	+ skel->bss->input = 5;
	109	+ usleep(1);
	110	+ bpf_map_lookup_elem(map5_fd, &key, &val);
	111	+ CHECK(val != 7, "inner5", "got %d != exp %d\n", val, 7);
99	112
100	113	for (i = 0; i < 5; i++) {
101	114	val = i % 2 ? map1_fd : map2_fd;
@@ -106,7 +119,13 @@ static void test_lookup_update(void) · 4 unmodified lines
106	119	}
107	120	err = bpf_map_update_elem(outer_arr_fd, &key, &val, 0);
108	121	if (CHECK_FAIL(err)) {
109		− printf("failed to update hash_of_maps on iter #%d\n", i);
	122	+ printf("failed to update array_of_maps on iter #%d\n", i);
	123	+ goto cleanup;
	124	+ }
	125	+ val = i % 2 ? map4_fd : map5_fd;
	126	+ err = bpf_map_update_elem(outer_arr_dyn_fd, &key, &val, 0);
	127	+ if (CHECK_FAIL(err)) {
	128	+ printf("failed to update array_of_maps (dyn) on iter #%d\n", i);
110	129	goto cleanup;
111	130	}
112	131	}

tools/testing/selftests/bpf/prog_tests/core_autosize.c+225−0

@@ -0,0 +1,225 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+
	4	+#include <test_progs.h>
	5	+#include <bpf/btf.h>
	6	+
	7	+/* real layout and sizes according to test's (32-bit) BTF
	8	+ * needs to be defined before skeleton is included */
	9	+struct test_struct___real {
	10	+ unsigned int ptr; /* can't use `void `, it is always 8 byte in BPF target /
	11	+ unsigned int val2;
	12	+ unsigned long long val1;
	13	+ unsigned short val3;
	14	+ unsigned char val4;
	15	+ unsigned char _pad;
	16	+};
	17	+
	18	+#include "test_core_autosize.skel.h"
	19	+
	20	+static int duration = 0;
	21	+
	22	+static struct {
	23	+ unsigned long long ptr_samesized;
	24	+ unsigned long long val1_samesized;
	25	+ unsigned long long val2_samesized;
	26	+ unsigned long long val3_samesized;
	27	+ unsigned long long val4_samesized;
	28	+ struct test_struct___real output_samesized;
	29	+
	30	+ unsigned long long ptr_downsized;
	31	+ unsigned long long val1_downsized;
	32	+ unsigned long long val2_downsized;
	33	+ unsigned long long val3_downsized;
	34	+ unsigned long long val4_downsized;
	35	+ struct test_struct___real output_downsized;
	36	+
	37	+ unsigned long long ptr_probed;
	38	+ unsigned long long val1_probed;
	39	+ unsigned long long val2_probed;
	40	+ unsigned long long val3_probed;
	41	+ unsigned long long val4_probed;
	42	+
	43	+ unsigned long long ptr_signed;
	44	+ unsigned long long val1_signed;
	45	+ unsigned long long val2_signed;
	46	+ unsigned long long val3_signed;
	47	+ unsigned long long val4_signed;
	48	+ struct test_struct___real output_signed;
	49	+} out;
	50	+
	51	+void test_core_autosize(void)
	52	+{
	53	+ char btf_file[] = "/tmp/core_autosize.btf.XXXXXX";
	54	+ int err, fd = -1, zero = 0;
	55	+ int char_id, short_id, int_id, long_long_id, void_ptr_id, id;
	56	+ struct test_core_autosize* skel = NULL;
	57	+ struct bpf_object_load_attr load_attr = {};
	58	+ struct bpf_program *prog;
	59	+ struct bpf_map *bss_map;
	60	+ struct btf *btf = NULL;
	61	+ size_t written;
	62	+ const void *raw_data;
	63	+ __u32 raw_sz;
	64	+ FILE *f = NULL;
	65	+
	66	+ btf = btf__new_empty();
	67	+ if (!ASSERT_OK_PTR(btf, "empty_btf"))
	68	+ return;
	69	+ /* Emit the following struct with 32-bit pointer size:
	70	+ *
	71	+ * struct test_struct {
	72	+ * void *ptr;
	73	+ * unsigned long val2;
	74	+ * unsigned long long val1;
	75	+ * unsigned short val3;
	76	+ * unsigned char val4;
	77	+ * char: 8;
	78	+ * };
	79	+ *
	80	+ * This struct is going to be used as the "kernel BTF" for this test.
	81	+ * It's equivalent memory-layout-wise to test_struct__real above.
	82	+ */
	83	+
	84	+ /* force 32-bit pointer size */
	85	+ btf__set_pointer_size(btf, 4);
	86	+
	87	+ char_id = btf__add_int(btf, "unsigned char", 1, 0);
	88	+ ASSERT_EQ(char_id, 1, "char_id");
	89	+ short_id = btf__add_int(btf, "unsigned short", 2, 0);
	90	+ ASSERT_EQ(short_id, 2, "short_id");
	91	+ /* "long unsigned int" of 4 byte size tells BTF that sizeof(void ) == 4 /
	92	+ int_id = btf__add_int(btf, "long unsigned int", 4, 0);
	93	+ ASSERT_EQ(int_id, 3, "int_id");
	94	+ long_long_id = btf__add_int(btf, "unsigned long long", 8, 0);
	95	+ ASSERT_EQ(long_long_id, 4, "long_long_id");
	96	+ void_ptr_id = btf__add_ptr(btf, 0);
	97	+ ASSERT_EQ(void_ptr_id, 5, "void_ptr_id");
	98	+
	99	+ id = btf__add_struct(btf, "test_struct", 20 /* bytes */);
	100	+ ASSERT_EQ(id, 6, "struct_id");
	101	+ err = btf__add_field(btf, "ptr", void_ptr_id, 0, 0);
	102	+ err = err ?: btf__add_field(btf, "val2", int_id, 32, 0);
	103	+ err = err ?: btf__add_field(btf, "val1", long_long_id, 64, 0);
	104	+ err = err ?: btf__add_field(btf, "val3", short_id, 128, 0);
	105	+ err = err ?: btf__add_field(btf, "val4", char_id, 144, 0);
	106	+ ASSERT_OK(err, "struct_fields");
	107	+
	108	+ fd = mkstemp(btf_file);
	109	+ if (CHECK(fd < 0, "btf_tmp", "failed to create file: %d\n", fd))
	110	+ goto cleanup;
	111	+ f = fdopen(fd, "w");
	112	+ if (!ASSERT_OK_PTR(f, "btf_fdopen"))
	113	+ goto cleanup;
	114	+
	115	+ raw_data = btf__get_raw_data(btf, &raw_sz);
	116	+ if (!ASSERT_OK_PTR(raw_data, "raw_data"))
	117	+ goto cleanup;
	118	+ written = fwrite(raw_data, 1, raw_sz, f);
	119	+ if (CHECK(written != raw_sz, "btf_write", "written: %zu, errno: %d\n", written, errno))
	120	+ goto cleanup;
	121	+ fflush(f);
	122	+ fclose(f);
	123	+ f = NULL;
	124	+ close(fd);
	125	+ fd = -1;
	126	+
	127	+ /* open and load BPF program with custom BTF as the kernel BTF */
	128	+ skel = test_core_autosize__open();
	129	+ if (!ASSERT_OK_PTR(skel, "skel_open"))
	130	+ return;
	131	+
	132	+ /* disable handle_signed() for now */
	133	+ prog = bpf_object__find_program_by_name(skel->obj, "handle_signed");
	134	+ if (!ASSERT_OK_PTR(prog, "prog_find"))
	135	+ goto cleanup;
	136	+ bpf_program__set_autoload(prog, false);
	137	+
	138	+ load_attr.obj = skel->obj;
	139	+ load_attr.target_btf_path = btf_file;
	140	+ err = bpf_object__load_xattr(&load_attr);
	141	+ if (!ASSERT_OK(err, "prog_load"))
	142	+ goto cleanup;
	143	+
	144	+ prog = bpf_object__find_program_by_name(skel->obj, "handle_samesize");
	145	+ if (!ASSERT_OK_PTR(prog, "prog_find"))
	146	+ goto cleanup;
	147	+ skel->links.handle_samesize = bpf_program__attach(prog);
	148	+ if (!ASSERT_OK_PTR(skel->links.handle_samesize, "prog_attach"))
	149	+ goto cleanup;
	150	+
	151	+ prog = bpf_object__find_program_by_name(skel->obj, "handle_downsize");
	152	+ if (!ASSERT_OK_PTR(prog, "prog_find"))
	153	+ goto cleanup;
	154	+ skel->links.handle_downsize = bpf_program__attach(prog);
	155	+ if (!ASSERT_OK_PTR(skel->links.handle_downsize, "prog_attach"))
	156	+ goto cleanup;
	157	+
	158	+ prog = bpf_object__find_program_by_name(skel->obj, "handle_probed");
	159	+ if (!ASSERT_OK_PTR(prog, "prog_find"))
	160	+ goto cleanup;
	161	+ skel->links.handle_probed = bpf_program__attach(prog);
	162	+ if (!ASSERT_OK_PTR(skel->links.handle_probed, "prog_attach"))
	163	+ goto cleanup;
	164	+
	165	+ usleep(1);
	166	+
	167	+ bss_map = bpf_object__find_map_by_name(skel->obj, "test_cor.bss");
	168	+ if (!ASSERT_OK_PTR(bss_map, "bss_map_find"))
	169	+ goto cleanup;
	170	+
	171	+ err = bpf_map_lookup_elem(bpf_map__fd(bss_map), &zero, (void *)&out);
	172	+ if (!ASSERT_OK(err, "bss_lookup"))
	173	+ goto cleanup;
	174	+
	175	+ ASSERT_EQ(out.ptr_samesized, 0x01020304, "ptr_samesized");
	176	+ ASSERT_EQ(out.val1_samesized, 0x1020304050607080, "val1_samesized");
	177	+ ASSERT_EQ(out.val2_samesized, 0x0a0b0c0d, "val2_samesized");
	178	+ ASSERT_EQ(out.val3_samesized, 0xfeed, "val3_samesized");
	179	+ ASSERT_EQ(out.val4_samesized, 0xb9, "val4_samesized");
	180	+ ASSERT_EQ(out.output_samesized.ptr, 0x01020304, "ptr_samesized");
	181	+ ASSERT_EQ(out.output_samesized.val1, 0x1020304050607080, "val1_samesized");
	182	+ ASSERT_EQ(out.output_samesized.val2, 0x0a0b0c0d, "val2_samesized");
	183	+ ASSERT_EQ(out.output_samesized.val3, 0xfeed, "val3_samesized");
	184	+ ASSERT_EQ(out.output_samesized.val4, 0xb9, "val4_samesized");
	185	+
	186	+ ASSERT_EQ(out.ptr_downsized, 0x01020304, "ptr_downsized");
	187	+ ASSERT_EQ(out.val1_downsized, 0x1020304050607080, "val1_downsized");
	188	+ ASSERT_EQ(out.val2_downsized, 0x0a0b0c0d, "val2_downsized");
	189	+ ASSERT_EQ(out.val3_downsized, 0xfeed, "val3_downsized");
	190	+ ASSERT_EQ(out.val4_downsized, 0xb9, "val4_downsized");
	191	+ ASSERT_EQ(out.output_downsized.ptr, 0x01020304, "ptr_downsized");
	192	+ ASSERT_EQ(out.output_downsized.val1, 0x1020304050607080, "val1_downsized");
	193	+ ASSERT_EQ(out.output_downsized.val2, 0x0a0b0c0d, "val2_downsized");
	194	+ ASSERT_EQ(out.output_downsized.val3, 0xfeed, "val3_downsized");
	195	+ ASSERT_EQ(out.output_downsized.val4, 0xb9, "val4_downsized");
	196	+
	197	+ ASSERT_EQ(out.ptr_probed, 0x01020304, "ptr_probed");
	198	+ ASSERT_EQ(out.val1_probed, 0x1020304050607080, "val1_probed");
	199	+ ASSERT_EQ(out.val2_probed, 0x0a0b0c0d, "val2_probed");
	200	+ ASSERT_EQ(out.val3_probed, 0xfeed, "val3_probed");
	201	+ ASSERT_EQ(out.val4_probed, 0xb9, "val4_probed");
	202	+
	203	+ test_core_autosize__destroy(skel);
	204	+ skel = NULL;
	205	+
	206	+ /* now re-load with handle_signed() enabled, it should fail loading */
	207	+ skel = test_core_autosize__open();
	208	+ if (!ASSERT_OK_PTR(skel, "skel_open"))
	209	+ return;
	210	+
	211	+ load_attr.obj = skel->obj;
	212	+ load_attr.target_btf_path = btf_file;
	213	+ err = bpf_object__load_xattr(&load_attr);
	214	+ if (!ASSERT_ERR(err, "bad_prog_load"))
	215	+ goto cleanup;
	216	+
	217	+cleanup:
	218	+ if (f)
	219	+ fclose(f);
	220	+ if (fd >= 0)
	221	+ close(fd);
	222	+ remove(btf_file);
	223	+ btf__free(btf);
	224	+ test_core_autosize__destroy(skel);
	225	+}

tools/testing/selftests/bpf/prog_tests/ksyms.c+13−25

@@ -7,40 +7,28 @@ · 6 unmodified lines
7	7
8	8	static int duration;
9	9
10		−static __u64 kallsyms_find(const char *sym)
11		−{
12		− char type, name[500];
13		− __u64 addr, res = 0;
14		− FILE *f;
15		−
16		− f = fopen("/proc/kallsyms", "r");
17		− if (CHECK(!f, "kallsyms_fopen", "failed to open: %d\n", errno))
18		− return 0;
19		−
20		− while (fscanf(f, "%llx %c %499s%*[^\n]\n", &addr, &type, name) > 0) {
21		− if (strcmp(name, sym) == 0) {
22		− res = addr;
23		− goto out;
24		− }
25		− }
26		−
27		− CHECK(false, "not_found", "symbol %s not found\n", sym);
28		−out:
29		− fclose(f);
30		− return res;
31		−}
32		−
33	10	void test_ksyms(void)
34	11	{
35		− __u64 per_cpu_start_addr = kallsyms_find("__per_cpu_start");
36		− __u64 link_fops_addr = kallsyms_find("bpf_link_fops");
37	12	const char *btf_path = "/sys/kernel/btf/vmlinux";
38	13	struct test_ksyms *skel;
39	14	struct test_ksyms__data *data;
	15	+ __u64 link_fops_addr, per_cpu_start_addr;
40	16	struct stat st;
41	17	__u64 btf_size;
42	18	int err;
43	19
	20	+ err = kallsyms_find("bpf_link_fops", &link_fops_addr);
	21	+ if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
	22	+ return;
	23	+ if (CHECK(err == -ENOENT, "ksym_find", "symbol 'bpf_link_fops' not found\n"))
	24	+ return;
	25	+
	26	+ err = kallsyms_find("__per_cpu_start", &per_cpu_start_addr);
	27	+ if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
	28	+ return;
	29	+ if (CHECK(err == -ENOENT, "ksym_find", "symbol 'per_cpu_start' not found\n"))
	30	+ return;
	31	+
44	32	if (CHECK(stat(btf_path, &st), "stat_btf", "err %d\n", errno))
45	33	return;
46	34	btf_size = st.st_size;

tools/testing/selftests/bpf/prog_tests/ksyms_btf.c+88−0

@@ -0,0 +1,88 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Google */
	3	+
	4	+#include <test_progs.h>
	5	+#include <bpf/libbpf.h>
	6	+#include <bpf/btf.h>
	7	+#include "test_ksyms_btf.skel.h"
	8	+
	9	+static int duration;
	10	+
	11	+void test_ksyms_btf(void)
	12	+{
	13	+ __u64 runqueues_addr, bpf_prog_active_addr;
	14	+ __u32 this_rq_cpu;
	15	+ int this_bpf_prog_active;
	16	+ struct test_ksyms_btf *skel = NULL;
	17	+ struct test_ksyms_btf__data *data;
	18	+ struct btf *btf;
	19	+ int percpu_datasec;
	20	+ int err;
	21	+
	22	+ err = kallsyms_find("runqueues", &runqueues_addr);
	23	+ if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
	24	+ return;
	25	+ if (CHECK(err == -ENOENT, "ksym_find", "symbol 'runqueues' not found\n"))
	26	+ return;
	27	+
	28	+ err = kallsyms_find("bpf_prog_active", &bpf_prog_active_addr);
	29	+ if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
	30	+ return;
	31	+ if (CHECK(err == -ENOENT, "ksym_find", "symbol 'bpf_prog_active' not found\n"))
	32	+ return;
	33	+
	34	+ btf = libbpf_find_kernel_btf();
	35	+ if (CHECK(IS_ERR(btf), "btf_exists", "failed to load kernel BTF: %ld\n",
	36	+ PTR_ERR(btf)))
	37	+ return;
	38	+
	39	+ percpu_datasec = btf__find_by_name_kind(btf, ".data..percpu",
	40	+ BTF_KIND_DATASEC);
	41	+ if (percpu_datasec < 0) {
	42	+ printf("%s:SKIP:no PERCPU DATASEC in kernel btf\n",
	43	+ __func__);
	44	+ test__skip();
	45	+ goto cleanup;
	46	+ }
	47	+
	48	+ skel = test_ksyms_btf__open_and_load();
	49	+ if (CHECK(!skel, "skel_open", "failed to open and load skeleton\n"))
	50	+ goto cleanup;
	51	+
	52	+ err = test_ksyms_btf__attach(skel);
	53	+ if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
	54	+ goto cleanup;
	55	+
	56	+ /* trigger tracepoint */
	57	+ usleep(1);
	58	+
	59	+ data = skel->data;
	60	+ CHECK(data->out__runqueues_addr != runqueues_addr, "runqueues_addr",
	61	+ "got %llu, exp %llu\n",
	62	+ (unsigned long long)data->out__runqueues_addr,
	63	+ (unsigned long long)runqueues_addr);
	64	+ CHECK(data->out__bpf_prog_active_addr != bpf_prog_active_addr, "bpf_prog_active_addr",
	65	+ "got %llu, exp %llu\n",
	66	+ (unsigned long long)data->out__bpf_prog_active_addr,
	67	+ (unsigned long long)bpf_prog_active_addr);
	68	+
	69	+ CHECK(data->out__rq_cpu == -1, "rq_cpu",
	70	+ "got %u, exp != -1\n", data->out__rq_cpu);
	71	+ CHECK(data->out__bpf_prog_active < 0, "bpf_prog_active",
	72	+ "got %d, exp >= 0\n", data->out__bpf_prog_active);
	73	+ CHECK(data->out__cpu_0_rq_cpu != 0, "cpu_rq(0)->cpu",
	74	+ "got %u, exp 0\n", data->out__cpu_0_rq_cpu);
	75	+
	76	+ this_rq_cpu = data->out__this_rq_cpu;
	77	+ CHECK(this_rq_cpu != data->out__rq_cpu, "this_rq_cpu",
	78	+ "got %u, exp %u\n", this_rq_cpu, data->out__rq_cpu);
	79	+
	80	+ this_bpf_prog_active = data->out__this_bpf_prog_active;
	81	+ CHECK(this_bpf_prog_active != data->out__bpf_prog_active, "this_bpf_prog_active",
	82	+ "got %d, exp %d\n", this_bpf_prog_active,
	83	+ data->out__bpf_prog_active);
	84	+
	85	+cleanup:
	86	+ btf__free(btf);
	87	+ test_ksyms_btf__destroy(skel);
	88	+}

tools/testing/selftests/bpf/prog_tests/pinning.c+48−1

@@ -37,7 +37,7 @@ void test_pinning(void) · 36 unmodified lines
37	37	struct stat statbuf = {};
38	38	struct bpf_object *obj;
39	39	struct bpf_map *map;
40		− int err;
	40	+ int err, map_fd;
41	41	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
42	42	.pin_root_path = custpath,
43	43	);
@@ -213,6 +213,53 @@ void test_pinning(void) · 169 unmodified lines
213	213	if (CHECK(err, "stat custpinpath", "err %d errno %d\n", err, errno))
214	214	goto out;
215	215
	216	+ /* remove the custom pin path to re-test it with reuse fd below */
	217	+ err = unlink(custpinpath);
	218	+ if (CHECK(err, "unlink custpinpath", "err %d errno %d\n", err, errno))
	219	+ goto out;
	220	+
	221	+ err = rmdir(custpath);
	222	+ if (CHECK(err, "rmdir custpindir", "err %d errno %d\n", err, errno))
	223	+ goto out;
	224	+
	225	+ bpf_object__close(obj);
	226	+
	227	+ /* test pinning at custom path with reuse fd */
	228	+ obj = bpf_object__open_file(file, NULL);
	229	+ err = libbpf_get_error(obj);
	230	+ if (CHECK(err, "default open", "err %d errno %d\n", err, errno)) {
	231	+ obj = NULL;
	232	+ goto out;
	233	+ }
	234	+
	235	+ map_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(__u32),
	236	+ sizeof(__u64), 1, 0);
	237	+ if (CHECK(map_fd < 0, "create pinmap manually", "fd %d\n", map_fd))
	238	+ goto out;
	239	+
	240	+ map = bpf_object__find_map_by_name(obj, "pinmap");
	241	+ if (CHECK(!map, "find map", "NULL map"))
	242	+ goto close_map_fd;
	243	+
	244	+ err = bpf_map__reuse_fd(map, map_fd);
	245	+ if (CHECK(err, "reuse pinmap fd", "err %d errno %d\n", err, errno))
	246	+ goto close_map_fd;
	247	+
	248	+ err = bpf_map__set_pin_path(map, custpinpath);
	249	+ if (CHECK(err, "set pin path", "err %d errno %d\n", err, errno))
	250	+ goto close_map_fd;
	251	+
	252	+ err = bpf_object__load(obj);
	253	+ if (CHECK(err, "custom load", "err %d errno %d\n", err, errno))
	254	+ goto close_map_fd;
	255	+
	256	+ /* check that pinmap was pinned at the custom path */
	257	+ err = stat(custpinpath, &statbuf);
	258	+ if (CHECK(err, "stat custpinpath", "err %d errno %d\n", err, errno))
	259	+ goto close_map_fd;
	260	+
	261	+close_map_fd:
	262	+ close(map_fd);
216	263	out:
217	264	unlink(pinpath);
218	265	unlink(nopinpath);

tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+1−1

@@ -198,7 +198,7 @@ static void test_sockmap_copy(enum bpf_map_type map_type) · 197 unmodified lines
198	198	{
199	199	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
200	200	int err, len, src_fd, iter_fd, duration = 0;
201		− union bpf_iter_link_info linfo = {0};
	201	+ union bpf_iter_link_info linfo = {};
202	202	__u32 i, num_sockets, num_elems;
203	203	struct bpf_iter_sockmap *skel;
204	204	__s64 *sock_fd = NULL;

tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c+12−0

@@ -264,9 +264,19 @@ static int check_error_linum(const struct sk_fds *sk_fds) · 263 unmodified lines
264	264
265	265	static void check_hdr_and_close_fds(struct sk_fds *sk_fds)
266	266	{
	267	+ const __u32 expected_inherit_cb_flags =
	268	+ BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG \|
	269	+ BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG \|
	270	+ BPF_SOCK_OPS_STATE_CB_FLAG;
	271	+
267	272	if (sk_fds_shutdown(sk_fds))
268	273	goto check_linum;
269	274
	275	+ if (CHECK(expected_inherit_cb_flags != skel->bss->inherit_cb_flags,
	276	+ "Unexpected inherit_cb_flags", "0x%x != 0x%x\n",
	277	+ skel->bss->inherit_cb_flags, expected_inherit_cb_flags))
	278	+ goto check_linum;
	279	+
270	280	if (check_hdr_stg(&exp_passive_hdr_stg, sk_fds->passive_fd,
271	281	"passive_hdr_stg"))
272	282	goto check_linum;
@@ -321,6 +331,8 @@ static void reset_test(void) · 48 unmodified lines
321	331	memset(&skel->bss->active_estab_in, 0, optsize);
322	332	memset(&skel->bss->active_fin_in, 0, optsize);
323	333
	334	+ skel->bss->inherit_cb_flags = 0;
	335	+
324	336	skel->data->test_kind = TCPOPT_EXP;
325	337	skel->data->test_magic = 0xeB9F;
326	338

tools/testing/selftests/bpf/prog_tests/test_profiler.c+72−0

@@ -0,0 +1,72 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#include <test_progs.h>
	4	+#include "progs/profiler.h"
	5	+#include "profiler1.skel.h"
	6	+#include "profiler2.skel.h"
	7	+#include "profiler3.skel.h"
	8	+
	9	+static int sanity_run(struct bpf_program *prog)
	10	+{
	11	+ struct bpf_prog_test_run_attr test_attr = {};
	12	+ __u64 args[] = {1, 2, 3};
	13	+ __u32 duration = 0;
	14	+ int err, prog_fd;
	15	+
	16	+ prog_fd = bpf_program__fd(prog);
	17	+ test_attr.prog_fd = prog_fd;
	18	+ test_attr.ctx_in = args;
	19	+ test_attr.ctx_size_in = sizeof(args);
	20	+ err = bpf_prog_test_run_xattr(&test_attr);
	21	+ if (CHECK(err \|\| test_attr.retval, "test_run",
	22	+ "err %d errno %d retval %d duration %d\n",
	23	+ err, errno, test_attr.retval, duration))
	24	+ return -1;
	25	+ return 0;
	26	+}
	27	+
	28	+void test_test_profiler(void)
	29	+{
	30	+ struct profiler1 *profiler1_skel = NULL;
	31	+ struct profiler2 *profiler2_skel = NULL;
	32	+ struct profiler3 *profiler3_skel = NULL;
	33	+ __u32 duration = 0;
	34	+ int err;
	35	+
	36	+ profiler1_skel = profiler1__open_and_load();
	37	+ if (CHECK(!profiler1_skel, "profiler1_skel_load", "profiler1 skeleton failed\n"))
	38	+ goto cleanup;
	39	+
	40	+ err = profiler1__attach(profiler1_skel);
	41	+ if (CHECK(err, "profiler1_attach", "profiler1 attach failed: %d\n", err))
	42	+ goto cleanup;
	43	+
	44	+ if (sanity_run(profiler1_skel->progs.raw_tracepoint__sched_process_exec))
	45	+ goto cleanup;
	46	+
	47	+ profiler2_skel = profiler2__open_and_load();
	48	+ if (CHECK(!profiler2_skel, "profiler2_skel_load", "profiler2 skeleton failed\n"))
	49	+ goto cleanup;
	50	+
	51	+ err = profiler2__attach(profiler2_skel);
	52	+ if (CHECK(err, "profiler2_attach", "profiler2 attach failed: %d\n", err))
	53	+ goto cleanup;
	54	+
	55	+ if (sanity_run(profiler2_skel->progs.raw_tracepoint__sched_process_exec))
	56	+ goto cleanup;
	57	+
	58	+ profiler3_skel = profiler3__open_and_load();
	59	+ if (CHECK(!profiler3_skel, "profiler3_skel_load", "profiler3 skeleton failed\n"))
	60	+ goto cleanup;
	61	+
	62	+ err = profiler3__attach(profiler3_skel);
	63	+ if (CHECK(err, "profiler3_attach", "profiler3 attach failed: %d\n", err))
	64	+ goto cleanup;
	65	+
	66	+ if (sanity_run(profiler3_skel->progs.raw_tracepoint__sched_process_exec))
	67	+ goto cleanup;
	68	+cleanup:
	69	+ profiler1__destroy(profiler1_skel);
	70	+ profiler2__destroy(profiler2_skel);
	71	+ profiler3__destroy(profiler3_skel);
	72	+}

tools/testing/selftests/bpf/prog_tests/xdp_noinline.c+1−1

@@ -25,7 +25,7 @@ void test_xdp_noinline(void) · 24 unmodified lines
25	25	__u8 flags;
26	26	} real_def = {.dst = MAGIC_VAL};
27	27	__u32 ch_key = 11, real_num = 3;
28		− __u32 duration, retval, size;
	28	+ __u32 duration = 0, retval, size;
29	29	int err, i;
30	30	__u64 bytes = 0, pkts = 0;
31	31	char buf[128];

tools/testing/selftests/bpf/progs/connect4_prog.c+19−0

@@ -23,6 +23,10 @@ · 22 unmodified lines
23	23	#define TCP_CA_NAME_MAX 16
24	24	#endif
25	25
	26	+#ifndef TCP_NOTSENT_LOWAT
	27	+#define TCP_NOTSENT_LOWAT 25
	28	+#endif
	29	+
26	30	#ifndef IFNAMSIZ
27	31	#define IFNAMSIZ 16
28	32	#endif
@@ -128,6 +132,18 @@ static __inline int set_keepalive(struct bpf_sock_addr *ctx) · 99 unmodified lines
128	132	return 0;
129	133	}
130	134
	135	+static __inline int set_notsent_lowat(struct bpf_sock_addr *ctx)
	136	+{
	137	+ int lowat = 65535;
	138	+
	139	+ if (ctx->type == SOCK_STREAM) {
	140	+ if (bpf_setsockopt(ctx, SOL_TCP, TCP_NOTSENT_LOWAT, &lowat, sizeof(lowat)))
	141	+ return 1;
	142	+ }
	143	+
	144	+ return 0;
	145	+}
	146	+
131	147	SEC("cgroup/connect4")
132	148	int connect_v4_prog(struct bpf_sock_addr *ctx)
133	149	{
@@ -148,6 +164,9 @@ int connect_v4_prog(struct bpf_sock_addr *ctx) · 14 unmodified lines
148	164	if (set_keepalive(ctx))
149	165	return 0;
150	166
	167	+ if (set_notsent_lowat(ctx))
	168	+ return 0;
	169	+
151	170	if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
152	171	return 0;
153	172	else if (ctx->type == SOCK_STREAM)

tools/testing/selftests/bpf/progs/profiler.h+177−0

@@ -0,0 +1,177 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#pragma once
	4	+
	5	+#define TASK_COMM_LEN 16
	6	+#define MAX_ANCESTORS 4
	7	+#define MAX_PATH 256
	8	+#define KILL_TARGET_LEN 64
	9	+#define CTL_MAXNAME 10
	10	+#define MAX_ARGS_LEN 4096
	11	+#define MAX_FILENAME_LEN 512
	12	+#define MAX_ENVIRON_LEN 8192
	13	+#define MAX_PATH_DEPTH 32
	14	+#define MAX_FILEPATH_LENGTH (MAX_PATH_DEPTH * MAX_PATH)
	15	+#define MAX_CGROUPS_PATH_DEPTH 8
	16	+
	17	+#define MAX_METADATA_PAYLOAD_LEN TASK_COMM_LEN
	18	+
	19	+#define MAX_CGROUP_PAYLOAD_LEN \
	20	+ (MAX_PATH * 2 + (MAX_PATH * MAX_CGROUPS_PATH_DEPTH))
	21	+
	22	+#define MAX_CAP_PAYLOAD_LEN (MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN)
	23	+
	24	+#define MAX_SYSCTL_PAYLOAD_LEN \
	25	+ (MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + CTL_MAXNAME + MAX_PATH)
	26	+
	27	+#define MAX_KILL_PAYLOAD_LEN \
	28	+ (MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + TASK_COMM_LEN + \
	29	+ KILL_TARGET_LEN)
	30	+
	31	+#define MAX_EXEC_PAYLOAD_LEN \
	32	+ (MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + MAX_FILENAME_LEN + \
	33	+ MAX_ARGS_LEN + MAX_ENVIRON_LEN)
	34	+
	35	+#define MAX_FILEMOD_PAYLOAD_LEN \
	36	+ (MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + MAX_FILEPATH_LENGTH + \
	37	+ MAX_FILEPATH_LENGTH)
	38	+
	39	+enum data_type {
	40	+ INVALID_EVENT,
	41	+ EXEC_EVENT,
	42	+ FORK_EVENT,
	43	+ KILL_EVENT,
	44	+ SYSCTL_EVENT,
	45	+ FILEMOD_EVENT,
	46	+ MAX_DATA_TYPE_EVENT
	47	+};
	48	+
	49	+enum filemod_type {
	50	+ FMOD_OPEN,
	51	+ FMOD_LINK,
	52	+ FMOD_SYMLINK,
	53	+};
	54	+
	55	+struct ancestors_data_t {
	56	+ pid_t ancestor_pids[MAX_ANCESTORS];
	57	+ uint32_t ancestor_exec_ids[MAX_ANCESTORS];
	58	+ uint64_t ancestor_start_times[MAX_ANCESTORS];
	59	+ uint32_t num_ancestors;
	60	+};
	61	+
	62	+struct var_metadata_t {
	63	+ enum data_type type;
	64	+ pid_t pid;
	65	+ uint32_t exec_id;
	66	+ uid_t uid;
	67	+ gid_t gid;
	68	+ uint64_t start_time;
	69	+ uint32_t cpu_id;
	70	+ uint64_t bpf_stats_num_perf_events;
	71	+ uint64_t bpf_stats_start_ktime_ns;
	72	+ uint8_t comm_length;
	73	+};
	74	+
	75	+struct cgroup_data_t {
	76	+ ino_t cgroup_root_inode;
	77	+ ino_t cgroup_proc_inode;
	78	+ uint64_t cgroup_root_mtime;
	79	+ uint64_t cgroup_proc_mtime;
	80	+ uint16_t cgroup_root_length;
	81	+ uint16_t cgroup_proc_length;
	82	+ uint16_t cgroup_full_length;
	83	+ int cgroup_full_path_root_pos;
	84	+};
	85	+
	86	+struct var_sysctl_data_t {
	87	+ struct var_metadata_t meta;
	88	+ struct cgroup_data_t cgroup_data;
	89	+ struct ancestors_data_t ancestors_info;
	90	+ uint8_t sysctl_val_length;
	91	+ uint16_t sysctl_path_length;
	92	+ char payload[MAX_SYSCTL_PAYLOAD_LEN];
	93	+};
	94	+
	95	+struct var_kill_data_t {
	96	+ struct var_metadata_t meta;
	97	+ struct cgroup_data_t cgroup_data;
	98	+ struct ancestors_data_t ancestors_info;
	99	+ pid_t kill_target_pid;
	100	+ int kill_sig;
	101	+ uint32_t kill_count;
	102	+ uint64_t last_kill_time;
	103	+ uint8_t kill_target_name_length;
	104	+ uint8_t kill_target_cgroup_proc_length;
	105	+ char payload[MAX_KILL_PAYLOAD_LEN];
	106	+ size_t payload_length;
	107	+};
	108	+
	109	+struct var_exec_data_t {
	110	+ struct var_metadata_t meta;
	111	+ struct cgroup_data_t cgroup_data;
	112	+ pid_t parent_pid;
	113	+ uint32_t parent_exec_id;
	114	+ uid_t parent_uid;
	115	+ uint64_t parent_start_time;
	116	+ uint16_t bin_path_length;
	117	+ uint16_t cmdline_length;
	118	+ uint16_t environment_length;
	119	+ char payload[MAX_EXEC_PAYLOAD_LEN];
	120	+};
	121	+
	122	+struct var_fork_data_t {
	123	+ struct var_metadata_t meta;
	124	+ pid_t parent_pid;
	125	+ uint32_t parent_exec_id;
	126	+ uint64_t parent_start_time;
	127	+ char payload[MAX_METADATA_PAYLOAD_LEN];
	128	+};
	129	+
	130	+struct var_filemod_data_t {
	131	+ struct var_metadata_t meta;
	132	+ struct cgroup_data_t cgroup_data;
	133	+ enum filemod_type fmod_type;
	134	+ unsigned int dst_flags;
	135	+ uint32_t src_device_id;
	136	+ uint32_t dst_device_id;
	137	+ ino_t src_inode;
	138	+ ino_t dst_inode;
	139	+ uint16_t src_filepath_length;
	140	+ uint16_t dst_filepath_length;
	141	+ char payload[MAX_FILEMOD_PAYLOAD_LEN];
	142	+};
	143	+
	144	+struct profiler_config_struct {
	145	+ bool fetch_cgroups_from_bpf;
	146	+ ino_t cgroup_fs_inode;
	147	+ ino_t cgroup_login_session_inode;
	148	+ uint64_t kill_signals_mask;
	149	+ ino_t inode_filter;
	150	+ uint32_t stale_info_secs;
	151	+ bool use_variable_buffers;
	152	+ bool read_environ_from_exec;
	153	+ bool enable_cgroup_v1_resolver;
	154	+};
	155	+
	156	+struct bpf_func_stats_data {
	157	+ uint64_t time_elapsed_ns;
	158	+ uint64_t num_executions;
	159	+ uint64_t num_perf_events;
	160	+};
	161	+
	162	+struct bpf_func_stats_ctx {
	163	+ uint64_t start_time_ns;
	164	+ struct bpf_func_stats_data* bpf_func_stats_data_val;
	165	+};
	166	+
	167	+enum bpf_function_id {
	168	+ profiler_bpf_proc_sys_write,
	169	+ profiler_bpf_sched_process_exec,
	170	+ profiler_bpf_sched_process_exit,
	171	+ profiler_bpf_sys_enter_kill,
	172	+ profiler_bpf_do_filp_open_ret,
	173	+ profiler_bpf_sched_process_fork,
	174	+ profiler_bpf_vfs_link,
	175	+ profiler_bpf_vfs_symlink,
	176	+ profiler_bpf_max_function_id
	177	+};

tools/testing/selftests/bpf/progs/profiler.inc.h+969−0

@@ -0,0 +1,969 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#include <vmlinux.h>
	4	+#include <bpf/bpf_core_read.h>
	5	+#include <bpf/bpf_helpers.h>
	6	+#include <bpf/bpf_tracing.h>
	7	+
	8	+#include "profiler.h"
	9	+
	10	+#ifndef NULL
	11	+#define NULL 0
	12	+#endif
	13	+
	14	+#define O_WRONLY 00000001
	15	+#define O_RDWR 00000002
	16	+#define O_DIRECTORY 00200000
	17	+#define __O_TMPFILE 020000000
	18	+#define O_TMPFILE (__O_TMPFILE \| O_DIRECTORY)
	19	+#define MAX_ERRNO 4095
	20	+#define S_IFMT 00170000
	21	+#define S_IFSOCK 0140000
	22	+#define S_IFLNK 0120000
	23	+#define S_IFREG 0100000
	24	+#define S_IFBLK 0060000
	25	+#define S_IFDIR 0040000
	26	+#define S_IFCHR 0020000
	27	+#define S_IFIFO 0010000
	28	+#define S_ISUID 0004000
	29	+#define S_ISGID 0002000
	30	+#define S_ISVTX 0001000
	31	+#define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK)
	32	+#define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR)
	33	+#define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR)
	34	+#define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK)
	35	+#define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO)
	36	+#define S_ISSOCK(m) (((m)&S_IFMT) == S_IFSOCK)
	37	+#define IS_ERR_VALUE(x) (unsigned long)(void*)(x) >= (unsigned long)-MAX_ERRNO
	38	+
	39	+#define KILL_DATA_ARRAY_SIZE 8
	40	+
	41	+struct var_kill_data_arr_t {
	42	+ struct var_kill_data_t array[KILL_DATA_ARRAY_SIZE];
	43	+};
	44	+
	45	+union any_profiler_data_t {
	46	+ struct var_exec_data_t var_exec;
	47	+ struct var_kill_data_t var_kill;
	48	+ struct var_sysctl_data_t var_sysctl;
	49	+ struct var_filemod_data_t var_filemod;
	50	+ struct var_fork_data_t var_fork;
	51	+ struct var_kill_data_arr_t var_kill_data_arr;
	52	+};
	53	+
	54	+volatile struct profiler_config_struct bpf_config = {};
	55	+
	56	+#define FETCH_CGROUPS_FROM_BPF (bpf_config.fetch_cgroups_from_bpf)
	57	+#define CGROUP_FS_INODE (bpf_config.cgroup_fs_inode)
	58	+#define CGROUP_LOGIN_SESSION_INODE \
	59	+ (bpf_config.cgroup_login_session_inode)
	60	+#define KILL_SIGNALS (bpf_config.kill_signals_mask)
	61	+#define STALE_INFO (bpf_config.stale_info_secs)
	62	+#define INODE_FILTER (bpf_config.inode_filter)
	63	+#define READ_ENVIRON_FROM_EXEC (bpf_config.read_environ_from_exec)
	64	+#define ENABLE_CGROUP_V1_RESOLVER (bpf_config.enable_cgroup_v1_resolver)
	65	+
	66	+struct kernfs_iattrs___52 {
	67	+ struct iattr ia_iattr;
	68	+};
	69	+
	70	+struct kernfs_node___52 {
	71	+ union /* kernfs_node_id */ {
	72	+ struct {
	73	+ u32 ino;
	74	+ u32 generation;
	75	+ };
	76	+ u64 id;
	77	+ } id;
	78	+};
	79	+
	80	+struct {
	81	+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	82	+ __uint(max_entries, 1);
	83	+ __type(key, u32);
	84	+ __type(value, union any_profiler_data_t);
	85	+} data_heap SEC(".maps");
	86	+
	87	+struct {
	88	+ __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
	89	+ __uint(key_size, sizeof(int));
	90	+ __uint(value_size, sizeof(int));
	91	+} events SEC(".maps");
	92	+
	93	+struct {
	94	+ __uint(type, BPF_MAP_TYPE_HASH);
	95	+ __uint(max_entries, KILL_DATA_ARRAY_SIZE);
	96	+ __type(key, u32);
	97	+ __type(value, struct var_kill_data_arr_t);
	98	+} var_tpid_to_data SEC(".maps");
	99	+
	100	+struct {
	101	+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	102	+ __uint(max_entries, profiler_bpf_max_function_id);
	103	+ __type(key, u32);
	104	+ __type(value, struct bpf_func_stats_data);
	105	+} bpf_func_stats SEC(".maps");
	106	+
	107	+struct {
	108	+ __uint(type, BPF_MAP_TYPE_HASH);
	109	+ __type(key, u32);
	110	+ __type(value, bool);
	111	+ __uint(max_entries, 16);
	112	+} allowed_devices SEC(".maps");
	113	+
	114	+struct {
	115	+ __uint(type, BPF_MAP_TYPE_HASH);
	116	+ __type(key, u64);
	117	+ __type(value, bool);
	118	+ __uint(max_entries, 1024);
	119	+} allowed_file_inodes SEC(".maps");
	120	+
	121	+struct {
	122	+ __uint(type, BPF_MAP_TYPE_HASH);
	123	+ __type(key, u64);
	124	+ __type(value, bool);
	125	+ __uint(max_entries, 1024);
	126	+} allowed_directory_inodes SEC(".maps");
	127	+
	128	+struct {
	129	+ __uint(type, BPF_MAP_TYPE_HASH);
	130	+ __type(key, u32);
	131	+ __type(value, bool);
	132	+ __uint(max_entries, 16);
	133	+} disallowed_exec_inodes SEC(".maps");
	134	+
	135	+#ifndef ARRAY_SIZE
	136	+#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
	137	+#endif
	138	+
	139	+static INLINE bool IS_ERR(const void* ptr)
	140	+{
	141	+ return IS_ERR_VALUE((unsigned long)ptr);
	142	+}
	143	+
	144	+static INLINE u32 get_userspace_pid()
	145	+{
	146	+ return bpf_get_current_pid_tgid() >> 32;
	147	+}
	148	+
	149	+static INLINE bool is_init_process(u32 tgid)
	150	+{
	151	+ return tgid == 1 \|\| tgid == 0;
	152	+}
	153	+
	154	+static INLINE unsigned long
	155	+probe_read_lim(void* dst, void* src, unsigned long len, unsigned long max)
	156	+{
	157	+ len = len < max ? len : max;
	158	+ if (len > 1) {
	159	+ if (bpf_probe_read(dst, len, src))
	160	+ return 0;
	161	+ } else if (len == 1) {
	162	+ if (bpf_probe_read(dst, 1, src))
	163	+ return 0;
	164	+ }
	165	+ return len;
	166	+}
	167	+
	168	+static INLINE int get_var_spid_index(struct var_kill_data_arr_t* arr_struct,
	169	+ int spid)
	170	+{
	171	+#ifdef UNROLL
	172	+#pragma unroll
	173	+#endif
	174	+ for (int i = 0; i < ARRAY_SIZE(arr_struct->array); i++)
	175	+ if (arr_struct->array[i].meta.pid == spid)
	176	+ return i;
	177	+ return -1;
	178	+}
	179	+
	180	+static INLINE void populate_ancestors(struct task_struct* task,
	181	+ struct ancestors_data_t* ancestors_data)
	182	+{
	183	+ struct task_struct* parent = task;
	184	+ u32 num_ancestors, ppid;
	185	+
	186	+ ancestors_data->num_ancestors = 0;
	187	+#ifdef UNROLL
	188	+#pragma unroll
	189	+#endif
	190	+ for (num_ancestors = 0; num_ancestors < MAX_ANCESTORS; num_ancestors++) {
	191	+ parent = BPF_CORE_READ(parent, real_parent);
	192	+ if (parent == NULL)
	193	+ break;
	194	+ ppid = BPF_CORE_READ(parent, tgid);
	195	+ if (is_init_process(ppid))
	196	+ break;
	197	+ ancestors_data->ancestor_pids[num_ancestors] = ppid;
	198	+ ancestors_data->ancestor_exec_ids[num_ancestors] =
	199	+ BPF_CORE_READ(parent, self_exec_id);
	200	+ ancestors_data->ancestor_start_times[num_ancestors] =
	201	+ BPF_CORE_READ(parent, start_time);
	202	+ ancestors_data->num_ancestors = num_ancestors;
	203	+ }
	204	+}
	205	+
	206	+static INLINE void* read_full_cgroup_path(struct kernfs_node* cgroup_node,
	207	+ struct kernfs_node* cgroup_root_node,
	208	+ void* payload,
	209	+ int* root_pos)
	210	+{
	211	+ void* payload_start = payload;
	212	+ size_t filepart_length;
	213	+
	214	+#ifdef UNROLL
	215	+#pragma unroll
	216	+#endif
	217	+ for (int i = 0; i < MAX_CGROUPS_PATH_DEPTH; i++) {
	218	+ filepart_length =
	219	+ bpf_probe_read_str(payload, MAX_PATH, BPF_CORE_READ(cgroup_node, name));
	220	+ if (!cgroup_node)
	221	+ return payload;
	222	+ if (cgroup_node == cgroup_root_node)
	223	+ *root_pos = payload - payload_start;
	224	+ if (filepart_length <= MAX_PATH) {
	225	+ barrier_var(filepart_length);
	226	+ payload += filepart_length;
	227	+ }
	228	+ cgroup_node = BPF_CORE_READ(cgroup_node, parent);
	229	+ }
	230	+ return payload;
	231	+}
	232	+
	233	+static ino_t get_inode_from_kernfs(struct kernfs_node* node)
	234	+{
	235	+ struct kernfs_node___52* node52 = (void*)node;
	236	+
	237	+ if (bpf_core_field_exists(node52->id.ino)) {
	238	+ barrier_var(node52);
	239	+ return BPF_CORE_READ(node52, id.ino);
	240	+ } else {
	241	+ barrier_var(node);
	242	+ return (u64)BPF_CORE_READ(node, id);
	243	+ }
	244	+}
	245	+
	246	+int pids_cgrp_id = 1;
	247	+
	248	+static INLINE void* populate_cgroup_info(struct cgroup_data_t* cgroup_data,
	249	+ struct task_struct* task,
	250	+ void* payload)
	251	+{
	252	+ struct kernfs_node* root_kernfs =
	253	+ BPF_CORE_READ(task, nsproxy, cgroup_ns, root_cset, dfl_cgrp, kn);
	254	+ struct kernfs_node* proc_kernfs = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn);
	255	+
	256	+ if (ENABLE_CGROUP_V1_RESOLVER) {
	257	+#ifdef UNROLL
	258	+#pragma unroll
	259	+#endif
	260	+ for (int i = 0; i < CGROUP_SUBSYS_COUNT; i++) {
	261	+ struct cgroup_subsys_state* subsys =
	262	+ BPF_CORE_READ(task, cgroups, subsys[i]);
	263	+ if (subsys != NULL) {
	264	+ int subsys_id = BPF_CORE_READ(subsys, ss, id);
	265	+ if (subsys_id == pids_cgrp_id) {
	266	+ proc_kernfs = BPF_CORE_READ(subsys, cgroup, kn);
	267	+ root_kernfs = BPF_CORE_READ(subsys, ss, root, kf_root, kn);
	268	+ break;
	269	+ }
	270	+ }
	271	+ }
	272	+ }
	273	+
	274	+ cgroup_data->cgroup_root_inode = get_inode_from_kernfs(root_kernfs);
	275	+ cgroup_data->cgroup_proc_inode = get_inode_from_kernfs(proc_kernfs);
	276	+
	277	+ if (bpf_core_field_exists(root_kernfs->iattr->ia_mtime)) {
	278	+ cgroup_data->cgroup_root_mtime =
	279	+ BPF_CORE_READ(root_kernfs, iattr, ia_mtime.tv_nsec);
	280	+ cgroup_data->cgroup_proc_mtime =
	281	+ BPF_CORE_READ(proc_kernfs, iattr, ia_mtime.tv_nsec);
	282	+ } else {
	283	+ struct kernfs_iattrs___52* root_iattr =
	284	+ (struct kernfs_iattrs___52*)BPF_CORE_READ(root_kernfs, iattr);
	285	+ cgroup_data->cgroup_root_mtime =
	286	+ BPF_CORE_READ(root_iattr, ia_iattr.ia_mtime.tv_nsec);
	287	+
	288	+ struct kernfs_iattrs___52* proc_iattr =
	289	+ (struct kernfs_iattrs___52*)BPF_CORE_READ(proc_kernfs, iattr);
	290	+ cgroup_data->cgroup_proc_mtime =
	291	+ BPF_CORE_READ(proc_iattr, ia_iattr.ia_mtime.tv_nsec);
	292	+ }
	293	+
	294	+ cgroup_data->cgroup_root_length = 0;
	295	+ cgroup_data->cgroup_proc_length = 0;
	296	+ cgroup_data->cgroup_full_length = 0;
	297	+
	298	+ size_t cgroup_root_length =
	299	+ bpf_probe_read_str(payload, MAX_PATH, BPF_CORE_READ(root_kernfs, name));
	300	+ barrier_var(cgroup_root_length);
	301	+ if (cgroup_root_length <= MAX_PATH) {
	302	+ barrier_var(cgroup_root_length);
	303	+ cgroup_data->cgroup_root_length = cgroup_root_length;
	304	+ payload += cgroup_root_length;
	305	+ }
	306	+
	307	+ size_t cgroup_proc_length =
	308	+ bpf_probe_read_str(payload, MAX_PATH, BPF_CORE_READ(proc_kernfs, name));
	309	+ barrier_var(cgroup_proc_length);
	310	+ if (cgroup_proc_length <= MAX_PATH) {
	311	+ barrier_var(cgroup_proc_length);
	312	+ cgroup_data->cgroup_proc_length = cgroup_proc_length;
	313	+ payload += cgroup_proc_length;
	314	+ }
	315	+
	316	+ if (FETCH_CGROUPS_FROM_BPF) {
	317	+ cgroup_data->cgroup_full_path_root_pos = -1;
	318	+ void* payload_end_pos = read_full_cgroup_path(proc_kernfs, root_kernfs, payload,
	319	+ &cgroup_data->cgroup_full_path_root_pos);
	320	+ cgroup_data->cgroup_full_length = payload_end_pos - payload;
	321	+ payload = payload_end_pos;
	322	+ }
	323	+
	324	+ return (void*)payload;
	325	+}
	326	+
	327	+static INLINE void* populate_var_metadata(struct var_metadata_t* metadata,
	328	+ struct task_struct* task,
	329	+ u32 pid, void* payload)
	330	+{
	331	+ u64 uid_gid = bpf_get_current_uid_gid();
	332	+
	333	+ metadata->uid = (u32)uid_gid;
	334	+ metadata->gid = uid_gid >> 32;
	335	+ metadata->pid = pid;
	336	+ metadata->exec_id = BPF_CORE_READ(task, self_exec_id);
	337	+ metadata->start_time = BPF_CORE_READ(task, start_time);
	338	+ metadata->comm_length = 0;
	339	+
	340	+ size_t comm_length = bpf_core_read_str(payload, TASK_COMM_LEN, &task->comm);
	341	+ barrier_var(comm_length);
	342	+ if (comm_length <= TASK_COMM_LEN) {
	343	+ barrier_var(comm_length);
	344	+ metadata->comm_length = comm_length;
	345	+ payload += comm_length;
	346	+ }
	347	+
	348	+ return (void*)payload;
	349	+}
	350	+
	351	+static INLINE struct var_kill_data_t*
	352	+get_var_kill_data(struct pt_regs* ctx, int spid, int tpid, int sig)
	353	+{
	354	+ int zero = 0;
	355	+ struct var_kill_data_t* kill_data = bpf_map_lookup_elem(&data_heap, &zero);
	356	+
	357	+ if (kill_data == NULL)
	358	+ return NULL;
	359	+ struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	360	+
	361	+ void* payload = populate_var_metadata(&kill_data->meta, task, spid, kill_data->payload);
	362	+ payload = populate_cgroup_info(&kill_data->cgroup_data, task, payload);
	363	+ size_t payload_length = payload - (void*)kill_data->payload;
	364	+ kill_data->payload_length = payload_length;
	365	+ populate_ancestors(task, &kill_data->ancestors_info);
	366	+ kill_data->meta.type = KILL_EVENT;
	367	+ kill_data->kill_target_pid = tpid;
	368	+ kill_data->kill_sig = sig;
	369	+ kill_data->kill_count = 1;
	370	+ kill_data->last_kill_time = bpf_ktime_get_ns();
	371	+ return kill_data;
	372	+}
	373	+
	374	+static INLINE int trace_var_sys_kill(void* ctx, int tpid, int sig)
	375	+{
	376	+ if ((KILL_SIGNALS & (1ULL << sig)) == 0)
	377	+ return 0;
	378	+
	379	+ u32 spid = get_userspace_pid();
	380	+ struct var_kill_data_arr_t* arr_struct = bpf_map_lookup_elem(&var_tpid_to_data, &tpid);
	381	+
	382	+ if (arr_struct == NULL) {
	383	+ struct var_kill_data_t* kill_data = get_var_kill_data(ctx, spid, tpid, sig);
	384	+ int zero = 0;
	385	+
	386	+ if (kill_data == NULL)
	387	+ return 0;
	388	+ arr_struct = bpf_map_lookup_elem(&data_heap, &zero);
	389	+ if (arr_struct == NULL)
	390	+ return 0;
	391	+ bpf_probe_read(&arr_struct->array[0], sizeof(arr_struct->array[0]), kill_data);
	392	+ } else {
	393	+ int index = get_var_spid_index(arr_struct, spid);
	394	+
	395	+ if (index == -1) {
	396	+ struct var_kill_data_t* kill_data =
	397	+ get_var_kill_data(ctx, spid, tpid, sig);
	398	+ if (kill_data == NULL)
	399	+ return 0;
	400	+#ifdef UNROLL
	401	+#pragma unroll
	402	+#endif
	403	+ for (int i = 0; i < ARRAY_SIZE(arr_struct->array); i++)
	404	+ if (arr_struct->array[i].meta.pid == 0) {
	405	+ bpf_probe_read(&arr_struct->array[i],
	406	+ sizeof(arr_struct->array[i]), kill_data);
	407	+ bpf_map_update_elem(&var_tpid_to_data, &tpid,
	408	+ arr_struct, 0);
	409	+
	410	+ return 0;
	411	+ }
	412	+ return 0;
	413	+ }
	414	+
	415	+ struct var_kill_data_t* kill_data = &arr_struct->array[index];
	416	+
	417	+ u64 delta_sec =
	418	+ (bpf_ktime_get_ns() - kill_data->last_kill_time) / 1000000000;
	419	+
	420	+ if (delta_sec < STALE_INFO) {
	421	+ kill_data->kill_count++;
	422	+ kill_data->last_kill_time = bpf_ktime_get_ns();
	423	+ bpf_probe_read(&arr_struct->array[index],
	424	+ sizeof(arr_struct->array[index]),
	425	+ kill_data);
	426	+ } else {
	427	+ struct var_kill_data_t* kill_data =
	428	+ get_var_kill_data(ctx, spid, tpid, sig);
	429	+ if (kill_data == NULL)
	430	+ return 0;
	431	+ bpf_probe_read(&arr_struct->array[index],
	432	+ sizeof(arr_struct->array[index]),
	433	+ kill_data);
	434	+ }
	435	+ }
	436	+ bpf_map_update_elem(&var_tpid_to_data, &tpid, arr_struct, 0);
	437	+ return 0;
	438	+}
	439	+
	440	+static INLINE void bpf_stats_enter(struct bpf_func_stats_ctx* bpf_stat_ctx,
	441	+ enum bpf_function_id func_id)
	442	+{
	443	+ int func_id_key = func_id;
	444	+
	445	+ bpf_stat_ctx->start_time_ns = bpf_ktime_get_ns();
	446	+ bpf_stat_ctx->bpf_func_stats_data_val =
	447	+ bpf_map_lookup_elem(&bpf_func_stats, &func_id_key);
	448	+ if (bpf_stat_ctx->bpf_func_stats_data_val)
	449	+ bpf_stat_ctx->bpf_func_stats_data_val->num_executions++;
	450	+}
	451	+
	452	+static INLINE void bpf_stats_exit(struct bpf_func_stats_ctx* bpf_stat_ctx)
	453	+{
	454	+ if (bpf_stat_ctx->bpf_func_stats_data_val)
	455	+ bpf_stat_ctx->bpf_func_stats_data_val->time_elapsed_ns +=
	456	+ bpf_ktime_get_ns() - bpf_stat_ctx->start_time_ns;
	457	+}
	458	+
	459	+static INLINE void
	460	+bpf_stats_pre_submit_var_perf_event(struct bpf_func_stats_ctx* bpf_stat_ctx,
	461	+ struct var_metadata_t* meta)
	462	+{
	463	+ if (bpf_stat_ctx->bpf_func_stats_data_val) {
	464	+ bpf_stat_ctx->bpf_func_stats_data_val->num_perf_events++;
	465	+ meta->bpf_stats_num_perf_events =
	466	+ bpf_stat_ctx->bpf_func_stats_data_val->num_perf_events;
	467	+ }
	468	+ meta->bpf_stats_start_ktime_ns = bpf_stat_ctx->start_time_ns;
	469	+ meta->cpu_id = bpf_get_smp_processor_id();
	470	+}
	471	+
	472	+static INLINE size_t
	473	+read_absolute_file_path_from_dentry(struct dentry* filp_dentry, void* payload)
	474	+{
	475	+ size_t length = 0;
	476	+ size_t filepart_length;
	477	+ struct dentry* parent_dentry;
	478	+
	479	+#ifdef UNROLL
	480	+#pragma unroll
	481	+#endif
	482	+ for (int i = 0; i < MAX_PATH_DEPTH; i++) {
	483	+ filepart_length = bpf_probe_read_str(payload, MAX_PATH,
	484	+ BPF_CORE_READ(filp_dentry, d_name.name));
	485	+ barrier_var(filepart_length);
	486	+ if (filepart_length > MAX_PATH)
	487	+ break;
	488	+ barrier_var(filepart_length);
	489	+ payload += filepart_length;
	490	+ length += filepart_length;
	491	+
	492	+ parent_dentry = BPF_CORE_READ(filp_dentry, d_parent);
	493	+ if (filp_dentry == parent_dentry)
	494	+ break;
	495	+ filp_dentry = parent_dentry;
	496	+ }
	497	+
	498	+ return length;
	499	+}
	500	+
	501	+static INLINE bool
	502	+is_ancestor_in_allowed_inodes(struct dentry* filp_dentry)
	503	+{
	504	+ struct dentry* parent_dentry;
	505	+#ifdef UNROLL
	506	+#pragma unroll
	507	+#endif
	508	+ for (int i = 0; i < MAX_PATH_DEPTH; i++) {
	509	+ u64 dir_ino = BPF_CORE_READ(filp_dentry, d_inode, i_ino);
	510	+ bool* allowed_dir = bpf_map_lookup_elem(&allowed_directory_inodes, &dir_ino);
	511	+
	512	+ if (allowed_dir != NULL)
	513	+ return true;
	514	+ parent_dentry = BPF_CORE_READ(filp_dentry, d_parent);
	515	+ if (filp_dentry == parent_dentry)
	516	+ break;
	517	+ filp_dentry = parent_dentry;
	518	+ }
	519	+ return false;
	520	+}
	521	+
	522	+static INLINE bool is_dentry_allowed_for_filemod(struct dentry* file_dentry,
	523	+ u32* device_id,
	524	+ u64* file_ino)
	525	+{
	526	+ u32 dev_id = BPF_CORE_READ(file_dentry, d_sb, s_dev);
	527	+ *device_id = dev_id;
	528	+ bool* allowed_device = bpf_map_lookup_elem(&allowed_devices, &dev_id);
	529	+
	530	+ if (allowed_device == NULL)
	531	+ return false;
	532	+
	533	+ u64 ino = BPF_CORE_READ(file_dentry, d_inode, i_ino);
	534	+ *file_ino = ino;
	535	+ bool* allowed_file = bpf_map_lookup_elem(&allowed_file_inodes, &ino);
	536	+
	537	+ if (allowed_file == NULL)
	538	+ if (!is_ancestor_in_allowed_inodes(BPF_CORE_READ(file_dentry, d_parent)))
	539	+ return false;
	540	+ return true;
	541	+}
	542	+
	543	+SEC("kprobe/proc_sys_write")
	544	+ssize_t BPF_KPROBE(kprobe__proc_sys_write,
	545	+ struct file* filp, const char* buf,
	546	+ size_t count, loff_t* ppos)
	547	+{
	548	+ struct bpf_func_stats_ctx stats_ctx;
	549	+ bpf_stats_enter(&stats_ctx, profiler_bpf_proc_sys_write);
	550	+
	551	+ u32 pid = get_userspace_pid();
	552	+ int zero = 0;
	553	+ struct var_sysctl_data_t* sysctl_data =
	554	+ bpf_map_lookup_elem(&data_heap, &zero);
	555	+ if (!sysctl_data)
	556	+ goto out;
	557	+
	558	+ struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	559	+ sysctl_data->meta.type = SYSCTL_EVENT;
	560	+ void* payload = populate_var_metadata(&sysctl_data->meta, task, pid, sysctl_data->payload);
	561	+ payload = populate_cgroup_info(&sysctl_data->cgroup_data, task, payload);
	562	+
	563	+ populate_ancestors(task, &sysctl_data->ancestors_info);
	564	+
	565	+ sysctl_data->sysctl_val_length = 0;
	566	+ sysctl_data->sysctl_path_length = 0;
	567	+
	568	+ size_t sysctl_val_length = bpf_probe_read_str(payload, CTL_MAXNAME, buf);
	569	+ barrier_var(sysctl_val_length);
	570	+ if (sysctl_val_length <= CTL_MAXNAME) {
	571	+ barrier_var(sysctl_val_length);
	572	+ sysctl_data->sysctl_val_length = sysctl_val_length;
	573	+ payload += sysctl_val_length;
	574	+ }
	575	+
	576	+ size_t sysctl_path_length = bpf_probe_read_str(payload, MAX_PATH,
	577	+ BPF_CORE_READ(filp, f_path.dentry, d_name.name));
	578	+ barrier_var(sysctl_path_length);
	579	+ if (sysctl_path_length <= MAX_PATH) {
	580	+ barrier_var(sysctl_path_length);
	581	+ sysctl_data->sysctl_path_length = sysctl_path_length;
	582	+ payload += sysctl_path_length;
	583	+ }
	584	+
	585	+ bpf_stats_pre_submit_var_perf_event(&stats_ctx, &sysctl_data->meta);
	586	+ unsigned long data_len = payload - (void*)sysctl_data;
	587	+ data_len = data_len > sizeof(struct var_sysctl_data_t)
	588	+ ? sizeof(struct var_sysctl_data_t)
	589	+ : data_len;
	590	+ bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, sysctl_data, data_len);
	591	+out:
	592	+ bpf_stats_exit(&stats_ctx);
	593	+ return 0;
	594	+}
	595	+
	596	+SEC("tracepoint/syscalls/sys_enter_kill")
	597	+int tracepoint__syscalls__sys_enter_kill(struct trace_event_raw_sys_enter* ctx)
	598	+{
	599	+ struct bpf_func_stats_ctx stats_ctx;
	600	+
	601	+ bpf_stats_enter(&stats_ctx, profiler_bpf_sys_enter_kill);
	602	+ int pid = ctx->args[0];
	603	+ int sig = ctx->args[1];
	604	+ int ret = trace_var_sys_kill(ctx, pid, sig);
	605	+ bpf_stats_exit(&stats_ctx);
	606	+ return ret;
	607	+};
	608	+
	609	+SEC("raw_tracepoint/sched_process_exit")
	610	+int raw_tracepoint__sched_process_exit(void* ctx)
	611	+{
	612	+ int zero = 0;
	613	+ struct bpf_func_stats_ctx stats_ctx;
	614	+ bpf_stats_enter(&stats_ctx, profiler_bpf_sched_process_exit);
	615	+
	616	+ u32 tpid = get_userspace_pid();
	617	+
	618	+ struct var_kill_data_arr_t* arr_struct = bpf_map_lookup_elem(&var_tpid_to_data, &tpid);
	619	+ struct var_kill_data_t* kill_data = bpf_map_lookup_elem(&data_heap, &zero);
	620	+
	621	+ if (arr_struct == NULL \|\| kill_data == NULL)
	622	+ goto out;
	623	+
	624	+ struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	625	+ struct kernfs_node* proc_kernfs = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn);
	626	+
	627	+#ifdef UNROLL
	628	+#pragma unroll
	629	+#endif
	630	+ for (int i = 0; i < ARRAY_SIZE(arr_struct->array); i++) {
	631	+ struct var_kill_data_t* past_kill_data = &arr_struct->array[i];
	632	+
	633	+ if (past_kill_data != NULL && past_kill_data->kill_target_pid == tpid) {
	634	+ bpf_probe_read(kill_data, sizeof(*past_kill_data), past_kill_data);
	635	+ void* payload = kill_data->payload;
	636	+ size_t offset = kill_data->payload_length;
	637	+ if (offset >= MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN)
	638	+ return 0;
	639	+ payload += offset;
	640	+
	641	+ kill_data->kill_target_name_length = 0;
	642	+ kill_data->kill_target_cgroup_proc_length = 0;
	643	+
	644	+ size_t comm_length = bpf_core_read_str(payload, TASK_COMM_LEN, &task->comm);
	645	+ barrier_var(comm_length);
	646	+ if (comm_length <= TASK_COMM_LEN) {
	647	+ barrier_var(comm_length);
	648	+ kill_data->kill_target_name_length = comm_length;
	649	+ payload += comm_length;
	650	+ }
	651	+
	652	+ size_t cgroup_proc_length = bpf_probe_read_str(payload, KILL_TARGET_LEN,
	653	+ BPF_CORE_READ(proc_kernfs, name));
	654	+ barrier_var(cgroup_proc_length);
	655	+ if (cgroup_proc_length <= KILL_TARGET_LEN) {
	656	+ barrier_var(cgroup_proc_length);
	657	+ kill_data->kill_target_cgroup_proc_length = cgroup_proc_length;
	658	+ payload += cgroup_proc_length;
	659	+ }
	660	+
	661	+ bpf_stats_pre_submit_var_perf_event(&stats_ctx, &kill_data->meta);
	662	+ unsigned long data_len = (void)payload - (void)kill_data;
	663	+ data_len = data_len > sizeof(struct var_kill_data_t)
	664	+ ? sizeof(struct var_kill_data_t)
	665	+ : data_len;
	666	+ bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, kill_data, data_len);
	667	+ }
	668	+ }
	669	+ bpf_map_delete_elem(&var_tpid_to_data, &tpid);
	670	+out:
	671	+ bpf_stats_exit(&stats_ctx);
	672	+ return 0;
	673	+}
	674	+
	675	+SEC("raw_tracepoint/sched_process_exec")
	676	+int raw_tracepoint__sched_process_exec(struct bpf_raw_tracepoint_args* ctx)
	677	+{
	678	+ struct bpf_func_stats_ctx stats_ctx;
	679	+ bpf_stats_enter(&stats_ctx, profiler_bpf_sched_process_exec);
	680	+
	681	+ struct linux_binprm* bprm = (struct linux_binprm*)ctx->args[2];
	682	+ u64 inode = BPF_CORE_READ(bprm, file, f_inode, i_ino);
	683	+
	684	+ bool* should_filter_binprm = bpf_map_lookup_elem(&disallowed_exec_inodes, &inode);
	685	+ if (should_filter_binprm != NULL)
	686	+ goto out;
	687	+
	688	+ int zero = 0;
	689	+ struct var_exec_data_t* proc_exec_data = bpf_map_lookup_elem(&data_heap, &zero);
	690	+ if (!proc_exec_data)
	691	+ goto out;
	692	+
	693	+ if (INODE_FILTER && inode != INODE_FILTER)
	694	+ return 0;
	695	+
	696	+ u32 pid = get_userspace_pid();
	697	+ struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	698	+
	699	+ proc_exec_data->meta.type = EXEC_EVENT;
	700	+ proc_exec_data->bin_path_length = 0;
	701	+ proc_exec_data->cmdline_length = 0;
	702	+ proc_exec_data->environment_length = 0;
	703	+ void* payload = populate_var_metadata(&proc_exec_data->meta, task, pid,
	704	+ proc_exec_data->payload);
	705	+ payload = populate_cgroup_info(&proc_exec_data->cgroup_data, task, payload);
	706	+
	707	+ struct task_struct* parent_task = BPF_CORE_READ(task, real_parent);
	708	+ proc_exec_data->parent_pid = BPF_CORE_READ(parent_task, tgid);
	709	+ proc_exec_data->parent_uid = BPF_CORE_READ(parent_task, real_cred, uid.val);
	710	+ proc_exec_data->parent_exec_id = BPF_CORE_READ(parent_task, self_exec_id);
	711	+ proc_exec_data->parent_start_time = BPF_CORE_READ(parent_task, start_time);
	712	+
	713	+ const char* filename = BPF_CORE_READ(bprm, filename);
	714	+ size_t bin_path_length = bpf_probe_read_str(payload, MAX_FILENAME_LEN, filename);
	715	+ barrier_var(bin_path_length);
	716	+ if (bin_path_length <= MAX_FILENAME_LEN) {
	717	+ barrier_var(bin_path_length);
	718	+ proc_exec_data->bin_path_length = bin_path_length;
	719	+ payload += bin_path_length;
	720	+ }
	721	+
	722	+ void* arg_start = (void*)BPF_CORE_READ(task, mm, arg_start);
	723	+ void* arg_end = (void*)BPF_CORE_READ(task, mm, arg_end);
	724	+ unsigned int cmdline_length = probe_read_lim(payload, arg_start,
	725	+ arg_end - arg_start, MAX_ARGS_LEN);
	726	+
	727	+ if (cmdline_length <= MAX_ARGS_LEN) {
	728	+ barrier_var(cmdline_length);
	729	+ proc_exec_data->cmdline_length = cmdline_length;
	730	+ payload += cmdline_length;
	731	+ }
	732	+
	733	+ if (READ_ENVIRON_FROM_EXEC) {
	734	+ void* env_start = (void*)BPF_CORE_READ(task, mm, env_start);
	735	+ void* env_end = (void*)BPF_CORE_READ(task, mm, env_end);
	736	+ unsigned long env_len = probe_read_lim(payload, env_start,
	737	+ env_end - env_start, MAX_ENVIRON_LEN);
	738	+ if (cmdline_length <= MAX_ENVIRON_LEN) {
	739	+ proc_exec_data->environment_length = env_len;
	740	+ payload += env_len;
	741	+ }
	742	+ }
	743	+
	744	+ bpf_stats_pre_submit_var_perf_event(&stats_ctx, &proc_exec_data->meta);
	745	+ unsigned long data_len = payload - (void*)proc_exec_data;
	746	+ data_len = data_len > sizeof(struct var_exec_data_t)
	747	+ ? sizeof(struct var_exec_data_t)
	748	+ : data_len;
	749	+ bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, proc_exec_data, data_len);
	750	+out:
	751	+ bpf_stats_exit(&stats_ctx);
	752	+ return 0;
	753	+}
	754	+
	755	+SEC("kretprobe/do_filp_open")
	756	+int kprobe_ret__do_filp_open(struct pt_regs* ctx)
	757	+{
	758	+ struct bpf_func_stats_ctx stats_ctx;
	759	+ bpf_stats_enter(&stats_ctx, profiler_bpf_do_filp_open_ret);
	760	+
	761	+ struct file* filp = (struct file*)PT_REGS_RC_CORE(ctx);
	762	+
	763	+ if (filp == NULL \|\| IS_ERR(filp))
	764	+ goto out;
	765	+ unsigned int flags = BPF_CORE_READ(filp, f_flags);
	766	+ if ((flags & (O_RDWR \| O_WRONLY)) == 0)
	767	+ goto out;
	768	+ if ((flags & O_TMPFILE) > 0)
	769	+ goto out;
	770	+ struct inode* file_inode = BPF_CORE_READ(filp, f_inode);
	771	+ umode_t mode = BPF_CORE_READ(file_inode, i_mode);
	772	+ if (S_ISDIR(mode) \|\| S_ISCHR(mode) \|\| S_ISBLK(mode) \|\| S_ISFIFO(mode) \|\|
	773	+ S_ISSOCK(mode))
	774	+ goto out;
	775	+
	776	+ struct dentry* filp_dentry = BPF_CORE_READ(filp, f_path.dentry);
	777	+ u32 device_id = 0;
	778	+ u64 file_ino = 0;
	779	+ if (!is_dentry_allowed_for_filemod(filp_dentry, &device_id, &file_ino))
	780	+ goto out;
	781	+
	782	+ int zero = 0;
	783	+ struct var_filemod_data_t* filemod_data = bpf_map_lookup_elem(&data_heap, &zero);
	784	+ if (!filemod_data)
	785	+ goto out;
	786	+
	787	+ u32 pid = get_userspace_pid();
	788	+ struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	789	+
	790	+ filemod_data->meta.type = FILEMOD_EVENT;
	791	+ filemod_data->fmod_type = FMOD_OPEN;
	792	+ filemod_data->dst_flags = flags;
	793	+ filemod_data->src_inode = 0;
	794	+ filemod_data->dst_inode = file_ino;
	795	+ filemod_data->src_device_id = 0;
	796	+ filemod_data->dst_device_id = device_id;
	797	+ filemod_data->src_filepath_length = 0;
	798	+ filemod_data->dst_filepath_length = 0;
	799	+
	800	+ void* payload = populate_var_metadata(&filemod_data->meta, task, pid,
	801	+ filemod_data->payload);
	802	+ payload = populate_cgroup_info(&filemod_data->cgroup_data, task, payload);
	803	+
	804	+ size_t len = read_absolute_file_path_from_dentry(filp_dentry, payload);
	805	+ barrier_var(len);
	806	+ if (len <= MAX_FILEPATH_LENGTH) {
	807	+ barrier_var(len);
	808	+ payload += len;
	809	+ filemod_data->dst_filepath_length = len;
	810	+ }
	811	+ bpf_stats_pre_submit_var_perf_event(&stats_ctx, &filemod_data->meta);
	812	+ unsigned long data_len = payload - (void*)filemod_data;
	813	+ data_len = data_len > sizeof(filemod_data) ? sizeof(filemod_data) : data_len;
	814	+ bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, filemod_data, data_len);
	815	+out:
	816	+ bpf_stats_exit(&stats_ctx);
	817	+ return 0;
	818	+}
	819	+
	820	+SEC("kprobe/vfs_link")
	821	+int BPF_KPROBE(kprobe__vfs_link,
	822	+ struct dentry* old_dentry, struct inode* dir,
	823	+ struct dentry* new_dentry, struct inode** delegated_inode)
	824	+{
	825	+ struct bpf_func_stats_ctx stats_ctx;
	826	+ bpf_stats_enter(&stats_ctx, profiler_bpf_vfs_link);
	827	+
	828	+ u32 src_device_id = 0;
	829	+ u64 src_file_ino = 0;
	830	+ u32 dst_device_id = 0;
	831	+ u64 dst_file_ino = 0;
	832	+ if (!is_dentry_allowed_for_filemod(old_dentry, &src_device_id, &src_file_ino) &&
	833	+ !is_dentry_allowed_for_filemod(new_dentry, &dst_device_id, &dst_file_ino))
	834	+ goto out;
	835	+
	836	+ int zero = 0;
	837	+ struct var_filemod_data_t* filemod_data = bpf_map_lookup_elem(&data_heap, &zero);
	838	+ if (!filemod_data)
	839	+ goto out;
	840	+
	841	+ u32 pid = get_userspace_pid();
	842	+ struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	843	+
	844	+ filemod_data->meta.type = FILEMOD_EVENT;
	845	+ filemod_data->fmod_type = FMOD_LINK;
	846	+ filemod_data->dst_flags = 0;
	847	+ filemod_data->src_inode = src_file_ino;
	848	+ filemod_data->dst_inode = dst_file_ino;
	849	+ filemod_data->src_device_id = src_device_id;
	850	+ filemod_data->dst_device_id = dst_device_id;
	851	+ filemod_data->src_filepath_length = 0;
	852	+ filemod_data->dst_filepath_length = 0;
	853	+
	854	+ void* payload = populate_var_metadata(&filemod_data->meta, task, pid,
	855	+ filemod_data->payload);
	856	+ payload = populate_cgroup_info(&filemod_data->cgroup_data, task, payload);
	857	+
	858	+ size_t len = read_absolute_file_path_from_dentry(old_dentry, payload);
	859	+ barrier_var(len);
	860	+ if (len <= MAX_FILEPATH_LENGTH) {
	861	+ barrier_var(len);
	862	+ payload += len;
	863	+ filemod_data->src_filepath_length = len;
	864	+ }
	865	+
	866	+ len = read_absolute_file_path_from_dentry(new_dentry, payload);
	867	+ barrier_var(len);
	868	+ if (len <= MAX_FILEPATH_LENGTH) {
	869	+ barrier_var(len);
	870	+ payload += len;
	871	+ filemod_data->dst_filepath_length = len;
	872	+ }
	873	+
	874	+ bpf_stats_pre_submit_var_perf_event(&stats_ctx, &filemod_data->meta);
	875	+ unsigned long data_len = payload - (void*)filemod_data;
	876	+ data_len = data_len > sizeof(filemod_data) ? sizeof(filemod_data) : data_len;
	877	+ bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, filemod_data, data_len);
	878	+out:
	879	+ bpf_stats_exit(&stats_ctx);
	880	+ return 0;
	881	+}
	882	+
	883	+SEC("kprobe/vfs_symlink")
	884	+int BPF_KPROBE(kprobe__vfs_symlink, struct inode* dir, struct dentry* dentry,
	885	+ const char* oldname)
	886	+{
	887	+ struct bpf_func_stats_ctx stats_ctx;
	888	+ bpf_stats_enter(&stats_ctx, profiler_bpf_vfs_symlink);
	889	+
	890	+ u32 dst_device_id = 0;
	891	+ u64 dst_file_ino = 0;
	892	+ if (!is_dentry_allowed_for_filemod(dentry, &dst_device_id, &dst_file_ino))
	893	+ goto out;
	894	+
	895	+ int zero = 0;
	896	+ struct var_filemod_data_t* filemod_data = bpf_map_lookup_elem(&data_heap, &zero);
	897	+ if (!filemod_data)
	898	+ goto out;
	899	+
	900	+ u32 pid = get_userspace_pid();
	901	+ struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	902	+
	903	+ filemod_data->meta.type = FILEMOD_EVENT;
	904	+ filemod_data->fmod_type = FMOD_SYMLINK;
	905	+ filemod_data->dst_flags = 0;
	906	+ filemod_data->src_inode = 0;
	907	+ filemod_data->dst_inode = dst_file_ino;
	908	+ filemod_data->src_device_id = 0;
	909	+ filemod_data->dst_device_id = dst_device_id;
	910	+ filemod_data->src_filepath_length = 0;
	911	+ filemod_data->dst_filepath_length = 0;
	912	+
	913	+ void* payload = populate_var_metadata(&filemod_data->meta, task, pid,
	914	+ filemod_data->payload);
	915	+ payload = populate_cgroup_info(&filemod_data->cgroup_data, task, payload);
	916	+
	917	+ size_t len = bpf_probe_read_str(payload, MAX_FILEPATH_LENGTH, oldname);
	918	+ barrier_var(len);
	919	+ if (len <= MAX_FILEPATH_LENGTH) {
	920	+ barrier_var(len);
	921	+ payload += len;
	922	+ filemod_data->src_filepath_length = len;
	923	+ }
	924	+ len = read_absolute_file_path_from_dentry(dentry, payload);
	925	+ barrier_var(len);
	926	+ if (len <= MAX_FILEPATH_LENGTH) {
	927	+ barrier_var(len);
	928	+ payload += len;
	929	+ filemod_data->dst_filepath_length = len;
	930	+ }
	931	+ bpf_stats_pre_submit_var_perf_event(&stats_ctx, &filemod_data->meta);
	932	+ unsigned long data_len = payload - (void*)filemod_data;
	933	+ data_len = data_len > sizeof(filemod_data) ? sizeof(filemod_data) : data_len;
	934	+ bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, filemod_data, data_len);
	935	+out:
	936	+ bpf_stats_exit(&stats_ctx);
	937	+ return 0;
	938	+}
	939	+
	940	+SEC("raw_tracepoint/sched_process_fork")
	941	+int raw_tracepoint__sched_process_fork(struct bpf_raw_tracepoint_args* ctx)
	942	+{
	943	+ struct bpf_func_stats_ctx stats_ctx;
	944	+ bpf_stats_enter(&stats_ctx, profiler_bpf_sched_process_fork);
	945	+
	946	+ int zero = 0;
	947	+ struct var_fork_data_t* fork_data = bpf_map_lookup_elem(&data_heap, &zero);
	948	+ if (!fork_data)
	949	+ goto out;
	950	+
	951	+ struct task_struct* parent = (struct task_struct*)ctx->args[0];
	952	+ struct task_struct* child = (struct task_struct*)ctx->args[1];
	953	+ fork_data->meta.type = FORK_EVENT;
	954	+
	955	+ void* payload = populate_var_metadata(&fork_data->meta, child,
	956	+ BPF_CORE_READ(child, pid), fork_data->payload);
	957	+ fork_data->parent_pid = BPF_CORE_READ(parent, pid);
	958	+ fork_data->parent_exec_id = BPF_CORE_READ(parent, self_exec_id);
	959	+ fork_data->parent_start_time = BPF_CORE_READ(parent, start_time);
	960	+ bpf_stats_pre_submit_var_perf_event(&stats_ctx, &fork_data->meta);
	961	+
	962	+ unsigned long data_len = payload - (void*)fork_data;
	963	+ data_len = data_len > sizeof(fork_data) ? sizeof(fork_data) : data_len;
	964	+ bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, fork_data, data_len);
	965	+out:
	966	+ bpf_stats_exit(&stats_ctx);
	967	+ return 0;
	968	+}
	969	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/profiler1.c+6−0

@@ -0,0 +1,6 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#define barrier_var(var) asm volatile("" : "=r"(var) : "0"(var))
	4	+#define UNROLL
	5	+#define INLINE __always_inline
	6	+#include "profiler.inc.h"

tools/testing/selftests/bpf/progs/profiler2.c+6−0

@@ -0,0 +1,6 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#define barrier_var(var) /**/
	4	+/* undef #define UNROLL */
	5	+#define INLINE /**/
	6	+#include "profiler.inc.h"

tools/testing/selftests/bpf/progs/profiler3.c+6−0

@@ -0,0 +1,6 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+#define barrier_var(var) /**/
	4	+#define UNROLL
	5	+#define INLINE __noinline
	6	+#include "profiler.inc.h"

tools/testing/selftests/bpf/progs/test_btf_map_in_map.c+43−0

@@ -41,6 +41,43 @@ struct outer_arr { · 40 unmodified lines
41	41	.values = { (void )&inner_map1, 0, (void )&inner_map2 },
42	42	};
43	43
	44	+struct inner_map_sz3 {
	45	+ __uint(type, BPF_MAP_TYPE_ARRAY);
	46	+ __uint(map_flags, BPF_F_INNER_MAP);
	47	+ __uint(max_entries, 3);
	48	+ __type(key, int);
	49	+ __type(value, int);
	50	+} inner_map3 SEC(".maps"),
	51	+ inner_map4 SEC(".maps");
	52	+
	53	+struct inner_map_sz4 {
	54	+ __uint(type, BPF_MAP_TYPE_ARRAY);
	55	+ __uint(map_flags, BPF_F_INNER_MAP);
	56	+ __uint(max_entries, 5);
	57	+ __type(key, int);
	58	+ __type(value, int);
	59	+} inner_map5 SEC(".maps");
	60	+
	61	+struct outer_arr_dyn {
	62	+ __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
	63	+ __uint(max_entries, 3);
	64	+ __uint(key_size, sizeof(int));
	65	+ __uint(value_size, sizeof(int));
	66	+ __array(values, struct {
	67	+ __uint(type, BPF_MAP_TYPE_ARRAY);
	68	+ __uint(map_flags, BPF_F_INNER_MAP);
	69	+ __uint(max_entries, 1);
	70	+ __type(key, int);
	71	+ __type(value, int);
	72	+ });
	73	+} outer_arr_dyn SEC(".maps") = {
	74	+ .values = {
	75	+ [0] = (void *)&inner_map3,
	76	+ [1] = (void *)&inner_map4,
	77	+ [2] = (void *)&inner_map5,
	78	+ },
	79	+};
	80	+
44	81	struct outer_hash {
45	82	__uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
46	83	__uint(max_entries, 5);
@@ -101,6 +138,12 @@ int handle__sys_enter(void *ctx) · 54 unmodified lines
101	138	val = input + 1;
102	139	bpf_map_update_elem(inner_map, &key, &val, 0);
103	140
	141	+ inner_map = bpf_map_lookup_elem(&outer_arr_dyn, &key);
	142	+ if (!inner_map)
	143	+ return 1;
	144	+ val = input + 2;
	145	+ bpf_map_update_elem(inner_map, &key, &val, 0);
	146	+
104	147	return 0;
105	148	}
106	149

tools/testing/selftests/bpf/progs/test_core_autosize.c+172−0

@@ -0,0 +1,172 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Facebook */
	3	+
	4	+#include <linux/bpf.h>
	5	+#include <stdint.h>
	6	+#include <bpf/bpf_helpers.h>
	7	+#include <bpf/bpf_core_read.h>
	8	+
	9	+char _license[] SEC("license") = "GPL";
	10	+
	11	+/* fields of exactly the same size */
	12	+struct test_struct___samesize {
	13	+ void *ptr;
	14	+ unsigned long long val1;
	15	+ unsigned int val2;
	16	+ unsigned short val3;
	17	+ unsigned char val4;
	18	+} __attribute((preserve_access_index));
	19	+
	20	+/* unsigned fields that have to be downsized by libbpf */
	21	+struct test_struct___downsize {
	22	+ void *ptr;
	23	+ unsigned long val1;
	24	+ unsigned long val2;
	25	+ unsigned long val3;
	26	+ unsigned long val4;
	27	+ /* total sz: 40 */
	28	+} __attribute__((preserve_access_index));
	29	+
	30	+/* fields with signed integers of wrong size, should be rejected */
	31	+struct test_struct___signed {
	32	+ void *ptr;
	33	+ long val1;
	34	+ long val2;
	35	+ long val3;
	36	+ long val4;
	37	+} __attribute((preserve_access_index));
	38	+
	39	+/* real layout and sizes according to test's (32-bit) BTF */
	40	+struct test_struct___real {
	41	+ unsigned int ptr; /* can't use `void `, it is always 8 byte in BPF target /
	42	+ unsigned int val2;
	43	+ unsigned long long val1;
	44	+ unsigned short val3;
	45	+ unsigned char val4;
	46	+ unsigned char _pad;
	47	+ /* total sz: 20 */
	48	+};
	49	+
	50	+struct test_struct___real input = {
	51	+ .ptr = 0x01020304,
	52	+ .val1 = 0x1020304050607080,
	53	+ .val2 = 0x0a0b0c0d,
	54	+ .val3 = 0xfeed,
	55	+ .val4 = 0xb9,
	56	+ ._pad = 0xff, /* make sure no accidental zeros are present */
	57	+};
	58	+
	59	+unsigned long long ptr_samesized = 0;
	60	+unsigned long long val1_samesized = 0;
	61	+unsigned long long val2_samesized = 0;
	62	+unsigned long long val3_samesized = 0;
	63	+unsigned long long val4_samesized = 0;
	64	+struct test_struct___real output_samesized = {};
	65	+
	66	+unsigned long long ptr_downsized = 0;
	67	+unsigned long long val1_downsized = 0;
	68	+unsigned long long val2_downsized = 0;
	69	+unsigned long long val3_downsized = 0;
	70	+unsigned long long val4_downsized = 0;
	71	+struct test_struct___real output_downsized = {};
	72	+
	73	+unsigned long long ptr_probed = 0;
	74	+unsigned long long val1_probed = 0;
	75	+unsigned long long val2_probed = 0;
	76	+unsigned long long val3_probed = 0;
	77	+unsigned long long val4_probed = 0;
	78	+
	79	+unsigned long long ptr_signed = 0;
	80	+unsigned long long val1_signed = 0;
	81	+unsigned long long val2_signed = 0;
	82	+unsigned long long val3_signed = 0;
	83	+unsigned long long val4_signed = 0;
	84	+struct test_struct___real output_signed = {};
	85	+
	86	+SEC("raw_tp/sys_exit")
	87	+int handle_samesize(void *ctx)
	88	+{
	89	+ struct test_struct___samesize in = (void )&input;
	90	+ struct test_struct___samesize out = (void )&output_samesized;
	91	+
	92	+ ptr_samesized = (unsigned long long)in->ptr;
	93	+ val1_samesized = in->val1;
	94	+ val2_samesized = in->val2;
	95	+ val3_samesized = in->val3;
	96	+ val4_samesized = in->val4;
	97	+
	98	+ out->ptr = in->ptr;
	99	+ out->val1 = in->val1;
	100	+ out->val2 = in->val2;
	101	+ out->val3 = in->val3;
	102	+ out->val4 = in->val4;
	103	+
	104	+ return 0;
	105	+}
	106	+
	107	+SEC("raw_tp/sys_exit")
	108	+int handle_downsize(void *ctx)
	109	+{
	110	+ struct test_struct___downsize in = (void )&input;
	111	+ struct test_struct___downsize out = (void )&output_downsized;
	112	+
	113	+ ptr_downsized = (unsigned long long)in->ptr;
	114	+ val1_downsized = in->val1;
	115	+ val2_downsized = in->val2;
	116	+ val3_downsized = in->val3;
	117	+ val4_downsized = in->val4;
	118	+
	119	+ out->ptr = in->ptr;
	120	+ out->val1 = in->val1;
	121	+ out->val2 = in->val2;
	122	+ out->val3 = in->val3;
	123	+ out->val4 = in->val4;
	124	+
	125	+ return 0;
	126	+}
	127	+
	128	+SEC("raw_tp/sys_enter")
	129	+int handle_probed(void *ctx)
	130	+{
	131	+ struct test_struct___downsize in = (void )&input;
	132	+ __u64 tmp;
	133	+
	134	+ tmp = 0;
	135	+ bpf_core_read(&tmp, bpf_core_field_size(in->ptr), &in->ptr);
	136	+ ptr_probed = tmp;
	137	+
	138	+ tmp = 0;
	139	+ bpf_core_read(&tmp, bpf_core_field_size(in->val1), &in->val1);
	140	+ val1_probed = tmp;
	141	+
	142	+ tmp = 0;
	143	+ bpf_core_read(&tmp, bpf_core_field_size(in->val2), &in->val2);
	144	+ val2_probed = tmp;
	145	+
	146	+ tmp = 0;
	147	+ bpf_core_read(&tmp, bpf_core_field_size(in->val3), &in->val3);
	148	+ val3_probed = tmp;
	149	+
	150	+ tmp = 0;
	151	+ bpf_core_read(&tmp, bpf_core_field_size(in->val4), &in->val4);
	152	+ val4_probed = tmp;
	153	+
	154	+ return 0;
	155	+}
	156	+
	157	+SEC("raw_tp/sys_enter")
	158	+int handle_signed(void *ctx)
	159	+{
	160	+ struct test_struct___signed in = (void )&input;
	161	+ struct test_struct___signed out = (void )&output_signed;
	162	+
	163	+ val2_signed = in->val2;
	164	+ val3_signed = in->val3;
	165	+ val4_signed = in->val4;
	166	+
	167	+ out->val2= in->val2;
	168	+ out->val3= in->val3;
	169	+ out->val4= in->val4;
	170	+
	171	+ return 0;
	172	+}

tools/testing/selftests/bpf/progs/test_ksyms_btf.c+55−0

@@ -0,0 +1,55 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2020 Google */
	3	+
	4	+#include "vmlinux.h"
	5	+
	6	+#include <bpf/bpf_helpers.h>
	7	+
	8	+__u64 out__runqueues_addr = -1;
	9	+__u64 out__bpf_prog_active_addr = -1;
	10	+
	11	+__u32 out__rq_cpu = -1; /* percpu struct fields */
	12	+int out__bpf_prog_active = -1; /* percpu int */
	13	+
	14	+__u32 out__this_rq_cpu = -1;
	15	+int out__this_bpf_prog_active = -1;
	16	+
	17	+__u32 out__cpu_0_rq_cpu = -1; /* cpu_rq(0)->cpu */
	18	+
	19	+extern const struct rq runqueues __ksym; /* struct type global var. */
	20	+extern const int bpf_prog_active __ksym; /* int type global var. */
	21	+
	22	+SEC("raw_tp/sys_enter")
	23	+int handler(const void *ctx)
	24	+{
	25	+ struct rq *rq;
	26	+ int *active;
	27	+ __u32 cpu;
	28	+
	29	+ out__runqueues_addr = (__u64)&runqueues;
	30	+ out__bpf_prog_active_addr = (__u64)&bpf_prog_active;
	31	+
	32	+ cpu = bpf_get_smp_processor_id();
	33	+
	34	+ /* test bpf_per_cpu_ptr() */
	35	+ rq = (struct rq *)bpf_per_cpu_ptr(&runqueues, cpu);
	36	+ if (rq)
	37	+ out__rq_cpu = rq->cpu;
	38	+ active = (int *)bpf_per_cpu_ptr(&bpf_prog_active, cpu);
	39	+ if (active)
	40	+ out__bpf_prog_active = *active;
	41	+
	42	+ rq = (struct rq *)bpf_per_cpu_ptr(&runqueues, 0);
	43	+ if (rq) /* should always be valid, but we can't spare the check. */
	44	+ out__cpu_0_rq_cpu = rq->cpu;
	45	+
	46	+ /* test bpf_this_cpu_ptr */
	47	+ rq = (struct rq *)bpf_this_cpu_ptr(&runqueues);
	48	+ out__this_rq_cpu = rq->cpu;
	49	+ active = (int *)bpf_this_cpu_ptr(&bpf_prog_active);
	50	+ out__this_bpf_prog_active = *active;
	51	+
	52	+ return 0;
	53	+}
	54	+
	55	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c+2−2

@@ -304,10 +304,10 @@ int misc_estab(struct bpf_sock_ops *skops) · 303 unmodified lines
304	304	passive_lport_n = __bpf_htons(passive_lport_h);
305	305	bpf_setsockopt(skops, SOL_TCP, TCP_SAVE_SYN,
306	306	&true_val, sizeof(true_val));
307		− set_hdr_cb_flags(skops);
	307	+ set_hdr_cb_flags(skops, 0);
308	308	break;
309	309	case BPF_SOCK_OPS_TCP_CONNECT_CB:
310		− set_hdr_cb_flags(skops);
	310	+ set_hdr_cb_flags(skops, 0);
311	311	break;
312	312	case BPF_SOCK_OPS_PARSE_HDR_OPT_CB:
313	313	return handle_parse_hdr(skops);

tools/testing/selftests/bpf/progs/test_sockmap_kern.h+25−9

@@ -131,39 +131,55 @@ int bpf_prog2(struct __sk_buff *skb) · 130 unmodified lines
131	131
132	132	}
133	133
134		−SEC("sk_skb3")
135		−int bpf_prog3(struct __sk_buff *skb)
	134	+static inline void bpf_write_pass(struct __sk_buff *skb, int offset)
136	135	{
137		− const int one = 1;
138		− int err, *f, ret = SK_PASS;
	136	+ int err = bpf_skb_pull_data(skb, 6 + offset);
139	137	void *data_end;
140	138	char *c;
141	139
142		− err = bpf_skb_pull_data(skb, 19);
143	140	if (err)
144		− goto tls_out;
	141	+ return;
145	142
146	143	c = (char *)(long)skb->data;
147	144	data_end = (void *)(long)skb->data_end;
148	145
149		− if (c + 18 < data_end)
150		− memcpy(&c[13], "PASS", 4);
	146	+ if (c + 5 + offset < data_end)
	147	+ memcpy(c + offset, "PASS", 4);
	148	+}
	149	+
	150	+SEC("sk_skb3")
	151	+int bpf_prog3(struct __sk_buff *skb)
	152	+{
	153	+ int err, *f, ret = SK_PASS;
	154	+ const int one = 1;
	155	+
151	156	f = bpf_map_lookup_elem(&sock_skb_opts, &one);
152	157	if (f && *f) {
153	158	__u64 flags = 0;
154	159
155	160	ret = 0;
156	161	flags = *f;
	162	+
	163	+ err = bpf_skb_adjust_room(skb, -13, 0, 0);
	164	+ if (err)
	165	+ return SK_DROP;
	166	+ err = bpf_skb_adjust_room(skb, 4, 0, 0);
	167	+ if (err)
	168	+ return SK_DROP;
	169	+ bpf_write_pass(skb, 0);
157	170	#ifdef SOCKMAP
158	171	return bpf_sk_redirect_map(skb, &tls_sock_map, ret, flags);
159	172	#else
160	173	return bpf_sk_redirect_hash(skb, &tls_sock_map, &ret, flags);
161	174	#endif
162	175	}
163		−
164	176	f = bpf_map_lookup_elem(&sock_skb_opts, &one);
165	177	if (f && *f)
166	178	ret = SK_DROP;
	179	+ err = bpf_skb_adjust_room(skb, 4, 0, 0);
	180	+ if (err)
	181	+ return SK_DROP;
	182	+ bpf_write_pass(skb, 13);
167	183	tls_out:
168	184	return ret;
169	185	}

tools/testing/selftests/bpf/progs/test_tc_neigh.c+22−18

@@ -13,17 +13,10 @@ · 12 unmodified lines
13	13	#include <bpf/bpf_helpers.h>
14	14	#include <bpf/bpf_endian.h>
15	15
16		−#ifndef barrier_data
17		−# define barrier_data(ptr) asm volatile("": :"r"(ptr) :"memory")
18		−#endif
19		−
20	16	#ifndef ctx_ptr
21	17	# define ctx_ptr(field) (void *)(long)(field)
22	18	#endif
23	19
24		−#define dst_to_src_tmp 0xeeddddeeU
25		−#define src_to_dst_tmp 0xeeffffeeU
26		−
27	20	#define ip4_src 0xac100164 /* 172.16.1.100 */
28	21	#define ip4_dst 0xac100264 /* 172.16.2.100 */
29	22
@@ -39,6 +32,18 @@ · 9 unmodified lines
39	32	a.s6_addr32[3] == b.s6_addr32[3])
40	33	#endif
41	34
	35	+enum {
	36	+ dev_src,
	37	+ dev_dst,
	38	+};
	39	+
	40	+struct bpf_map_def SEC("maps") ifindex_map = {
	41	+ .type = BPF_MAP_TYPE_ARRAY,
	42	+ .key_size = sizeof(int),
	43	+ .value_size = sizeof(int),
	44	+ .max_entries = 2,
	45	+};
	46	+
42	47	static __always_inline bool is_remote_ep_v4(struct __sk_buff *skb,
43	48	__be32 addr)
44	49	{
@@ -73,7 +78,14 @@ static __always_inline bool is_remote_ep_v6(struct __sk_buff *skb, · 28 unmodified lines
73	78	return v6_equal(ip6h->daddr, addr);
74	79	}
75	80
76		−SEC("chk_neigh") int tc_chk(struct __sk_buff *skb)
	81	+static __always_inline int get_dev_ifindex(int which)
	82	+{
	83	+ int *ifindex = bpf_map_lookup_elem(&ifindex_map, &which);
	84	+
	85	+ return ifindex ? *ifindex : 0;
	86	+}
	87	+
	88	+SEC("chk_egress") int tc_chk(struct __sk_buff *skb)
77	89	{
78	90	void *data_end = ctx_ptr(skb->data_end);
79	91	void *data = ctx_ptr(skb->data);
@@ -87,7 +99,6 @@ SEC("chk_neigh") int tc_chk(struct __sk_buff *skb) · 7 unmodified lines
87	99
88	100	SEC("dst_ingress") int tc_dst(struct __sk_buff *skb)
89	101	{
90		− int idx = dst_to_src_tmp;
91	102	__u8 zero[ETH_ALEN * 2];
92	103	bool redirect = false;
93	104
@@ -103,19 +114,15 @@ SEC("dst_ingress") int tc_dst(struct __sk_buff *skb) · 9 unmodified lines
103	114	if (!redirect)
104	115	return TC_ACT_OK;
105	116
106		− barrier_data(&idx);
107		− idx = bpf_ntohl(idx);
108		−
109	117	__builtin_memset(&zero, 0, sizeof(zero));
110	118	if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0)
111	119	return TC_ACT_SHOT;
112	120
113		− return bpf_redirect_neigh(idx, 0);
	121	+ return bpf_redirect_neigh(get_dev_ifindex(dev_src), 0);
114	122	}
115	123
116	124	SEC("src_ingress") int tc_src(struct __sk_buff *skb)
117	125	{
118		− int idx = src_to_dst_tmp;
119	126	__u8 zero[ETH_ALEN * 2];
120	127	bool redirect = false;
121	128
@@ -131,14 +138,11 @@ SEC("src_ingress") int tc_src(struct __sk_buff *skb) · 9 unmodified lines
131	138	if (!redirect)
132	139	return TC_ACT_OK;
133	140
134		− barrier_data(&idx);
135		− idx = bpf_ntohl(idx);
136		−
137	141	__builtin_memset(&zero, 0, sizeof(zero));
138	142	if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0)
139	143	return TC_ACT_SHOT;
140	144
141		− return bpf_redirect_neigh(idx, 0);
	145	+ return bpf_redirect_neigh(get_dev_ifindex(dev_dst), 0);
142	146	}
143	147
144	148	char __license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_tc_peer.c+45−0

@@ -0,0 +1,45 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+#include <stdint.h>
	3	+#include <stdbool.h>
	4	+
	5	+#include <linux/bpf.h>
	6	+#include <linux/stddef.h>
	7	+#include <linux/pkt_cls.h>
	8	+
	9	+#include <bpf/bpf_helpers.h>
	10	+
	11	+enum {
	12	+ dev_src,
	13	+ dev_dst,
	14	+};
	15	+
	16	+struct bpf_map_def SEC("maps") ifindex_map = {
	17	+ .type = BPF_MAP_TYPE_ARRAY,
	18	+ .key_size = sizeof(int),
	19	+ .value_size = sizeof(int),
	20	+ .max_entries = 2,
	21	+};
	22	+
	23	+static __always_inline int get_dev_ifindex(int which)
	24	+{
	25	+ int *ifindex = bpf_map_lookup_elem(&ifindex_map, &which);
	26	+
	27	+ return ifindex ? *ifindex : 0;
	28	+}
	29	+
	30	+SEC("chk_egress") int tc_chk(struct __sk_buff *skb)
	31	+{
	32	+ return TC_ACT_SHOT;
	33	+}
	34	+
	35	+SEC("dst_ingress") int tc_dst(struct __sk_buff *skb)
	36	+{
	37	+ return bpf_redirect_peer(get_dev_ifindex(dev_src), 0);
	38	+}
	39	+
	40	+SEC("src_ingress") int tc_src(struct __sk_buff *skb)
	41	+{
	42	+ return bpf_redirect_peer(get_dev_ifindex(dev_dst), 0);
	43	+}
	44	+
	45	+char __license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_tcp_hdr_options.c+5−2

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21
22	22	__u8 test_kind = TCPOPT_EXP;
23	23	__u16 test_magic = 0xeB9F;
	24	+__u32 inherit_cb_flags = 0;
24	25
25	26	struct bpf_test_option passive_synack_out = {};
26	27	struct bpf_test_option passive_fin_out = {};
@@ -467,6 +468,8 @@ static int handle_passive_estab(struct bpf_sock_ops *skops) · 440 unmodified lines
467	468	struct tcphdr *th;
468	469	int err;
469	470
	471	+ inherit_cb_flags = skops->bpf_sock_ops_cb_flags;
	472	+
470	473	err = load_option(skops, &passive_estab_in, true);
471	474	if (err == -ENOENT) {
472	475	/* saved_syn is not found. It was in syncookie mode.
@@ -600,10 +603,10 @@ int estab(struct bpf_sock_ops *skops) · 127 unmodified lines
600	603	case BPF_SOCK_OPS_TCP_LISTEN_CB:
601	604	bpf_setsockopt(skops, SOL_TCP, TCP_SAVE_SYN,
602	605	&true_val, sizeof(true_val));
603		− set_hdr_cb_flags(skops);
	606	+ set_hdr_cb_flags(skops, BPF_SOCK_OPS_STATE_CB_FLAG);
604	607	break;
605	608	case BPF_SOCK_OPS_TCP_CONNECT_CB:
606		− set_hdr_cb_flags(skops);
	609	+ set_hdr_cb_flags(skops, 0);
607	610	break;
608	611	case BPF_SOCK_OPS_PARSE_HDR_OPT_CB:
609	612	return handle_parse_hdr(skops);

tools/testing/selftests/bpf/test_sockmap.c+45−36

@@ -86,6 +86,7 @@ int txmsg_ktls_skb_redir; · 85 unmodified lines
86	86	int ktls;
87	87	int peek_flag;
88	88	int skb_use_parser;
	89	+int txmsg_omit_skb_parser;
89	90
90	91	static const struct option long_options[] = {
91	92	{"help", no_argument, NULL, 'h' },
@@ -111,6 +112,7 @@ static const struct option long_options[] = { · 19 unmodified lines
111	112	{"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
112	113	{"ktls", no_argument, &ktls, 1 },
113	114	{"peek", no_argument, &peek_flag, 1 },
	115	+ {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
114	116	{"whitelist", required_argument, NULL, 'n' },
115	117	{"blacklist", required_argument, NULL, 'b' },
116	118	{0, 0, NULL, 0 }
@@ -175,6 +177,7 @@ static void test_reset(void) · 58 unmodified lines
175	177	txmsg_apply = txmsg_cork = 0;
176	178	txmsg_ingress = txmsg_redir_skb = 0;
177	179	txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
	180	+ txmsg_omit_skb_parser = 0;
178	181	skb_use_parser = 0;
179	182	}
180	183
@@ -518,28 +521,13 @@ static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz) · 337 unmodified lines
518	521	if (i == 0 && txmsg_ktls_skb) {
519	522	if (msg->msg_iov[i].iov_len < 4)
520	523	return -EIO;
521		− if (txmsg_ktls_skb_redir) {
522		− if (memcmp(&d[13], "PASS", 4) != 0) {
523		− fprintf(stderr,
524		− "detected redirect ktls_skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", i, 0, d[13], d[14], d[15], d[16]);
525		− return -EIO;
526		− }
527		− d[13] = 0;
528		− d[14] = 1;
529		− d[15] = 2;
530		− d[16] = 3;
531		− j = 13;
532		− } else if (txmsg_ktls_skb) {
533		− if (memcmp(d, "PASS", 4) != 0) {
534		− fprintf(stderr,
535		− "detected ktls_skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", i, 0, d[0], d[1], d[2], d[3]);
536		− return -EIO;
537		− }
538		− d[0] = 0;
539		− d[1] = 1;
540		− d[2] = 2;
541		− d[3] = 3;
	524	+ if (memcmp(d, "PASS", 4) != 0) {
	525	+ fprintf(stderr,
	526	+ "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
	527	+ i, 0, d[0], d[1], d[2], d[3]);
	528	+ return -EIO;
542	529	}
	530	+ j = 4; /* advance index past PASS header */
543	531	}
544	532
545	533	for (; j < msg->msg_iov[i].iov_len && size; j++) {
@@ -927,13 +915,15 @@ static int run_options(struct sockmap_options *options, int cg_fd, int test) · 381 unmodified lines
927	915	goto run;
928	916
929	917	/* Attach programs to sockmap */
930		− err = bpf_prog_attach(prog_fd[0], map_fd[0],
931		− BPF_SK_SKB_STREAM_PARSER, 0);
932		− if (err) {
933		− fprintf(stderr,
934		− "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
935		− prog_fd[0], map_fd[0], err, strerror(errno));
936		− return err;
	918	+ if (!txmsg_omit_skb_parser) {
	919	+ err = bpf_prog_attach(prog_fd[0], map_fd[0],
	920	+ BPF_SK_SKB_STREAM_PARSER, 0);
	921	+ if (err) {
	922	+ fprintf(stderr,
	923	+ "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
	924	+ prog_fd[0], map_fd[0], err, strerror(errno));
	925	+ return err;
	926	+ }
937	927	}
938	928
939	929	err = bpf_prog_attach(prog_fd[1], map_fd[0],
@@ -946,13 +936,15 @@ static int run_options(struct sockmap_options *options, int cg_fd, int test) · 6 unmodified lines
946	936
947	937	/* Attach programs to TLS sockmap */
948	938	if (txmsg_ktls_skb) {
949		− err = bpf_prog_attach(prog_fd[0], map_fd[8],
950		− BPF_SK_SKB_STREAM_PARSER, 0);
951		− if (err) {
952		− fprintf(stderr,
953		− "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
954		− prog_fd[0], map_fd[8], err, strerror(errno));
955		− return err;
	939	+ if (!txmsg_omit_skb_parser) {
	940	+ err = bpf_prog_attach(prog_fd[0], map_fd[8],
	941	+ BPF_SK_SKB_STREAM_PARSER, 0);
	942	+ if (err) {
	943	+ fprintf(stderr,
	944	+ "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
	945	+ prog_fd[0], map_fd[8], err, strerror(errno));
	946	+ return err;
	947	+ }
956	948	}
957	949
958	950	err = bpf_prog_attach(prog_fd[2], map_fd[8],
@@ -1480,12 +1472,29 @@ static void test_txmsg_skb(int cgrp, struct sockmap_options *opt) · 521 unmodified lines
1480	1472	txmsg_ktls_skb_drop = 0;
1481	1473	txmsg_ktls_skb_redir = 1;
1482	1474	test_exec(cgrp, opt);
	1475	+ txmsg_ktls_skb_redir = 0;
	1476	+
	1477	+ /* Tests that omit skb_parser */
	1478	+ txmsg_omit_skb_parser = 1;
	1479	+ ktls = 0;
	1480	+ txmsg_ktls_skb = 0;
	1481	+ test_exec(cgrp, opt);
	1482	+
	1483	+ txmsg_ktls_skb_drop = 1;
	1484	+ test_exec(cgrp, opt);
	1485	+ txmsg_ktls_skb_drop = 0;
	1486	+
	1487	+ txmsg_ktls_skb_redir = 1;
	1488	+ test_exec(cgrp, opt);
	1489	+
	1490	+ ktls = 1;
	1491	+ test_exec(cgrp, opt);
	1492	+ txmsg_omit_skb_parser = 0;
1483	1493
1484	1494	opt->data_test = data;
1485	1495	ktls = k;
1486	1496	}
1487	1497
1488		−
1489	1498	/* Test cork with hung data. This tests poor usage patterns where
1490	1499	* cork can leave data on the ring if user program is buggy and
1491	1500	* doesn't flush them somehow. They do take some time however

tools/testing/selftests/bpf/test_tc_neigh.sh+0−168

@@ -1,168 +0,0 @@
1		−#!/bin/bash
2		−# SPDX-License-Identifier: GPL-2.0
3		−#
4		−# This test sets up 3 netns (src <-> fwd <-> dst). There is no direct veth link
5		−# between src and dst. The netns fwd has veth links to each src and dst. The
6		−# client is in src and server in dst. The test installs a TC BPF program to each
7		−# host facing veth in fwd which calls into bpf_redirect_peer() to perform the
8		−# neigh addr population and redirect; it also installs a dropper prog on the
9		−# egress side to drop skbs if neigh addrs were not populated.
10		−
11		−if [[ $EUID -ne 0 ]]; then
12		− echo "This script must be run as root"
13		− echo "FAIL"
14		− exit 1
15		−fi
16		−
17		−# check that nc, dd, ping, ping6 and timeout are present
18		−command -v nc >/dev/null 2>&1 \|\| \
19		− { echo >&2 "nc is not available"; exit 1; }
20		−command -v dd >/dev/null 2>&1 \|\| \
21		− { echo >&2 "dd is not available"; exit 1; }
22		−command -v timeout >/dev/null 2>&1 \|\| \
23		− { echo >&2 "timeout is not available"; exit 1; }
24		−command -v ping >/dev/null 2>&1 \|\| \
25		− { echo >&2 "ping is not available"; exit 1; }
26		−command -v ping6 >/dev/null 2>&1 \|\| \
27		− { echo >&2 "ping6 is not available"; exit 1; }
28		−
29		−readonly GREEN='\033[0;92m'
30		−readonly RED='\033[0;31m'
31		−readonly NC='\033[0m' # No Color
32		−
33		−readonly PING_ARG="-c 3 -w 10 -q"
34		−
35		−readonly TIMEOUT=10
36		−
37		−readonly NS_SRC="ns-src-$(mktemp -u XXXXXX)"
38		−readonly NS_FWD="ns-fwd-$(mktemp -u XXXXXX)"
39		−readonly NS_DST="ns-dst-$(mktemp -u XXXXXX)"
40		−
41		−readonly IP4_SRC="172.16.1.100"
42		−readonly IP4_DST="172.16.2.100"
43		−
44		−readonly IP6_SRC="::1:dead:beef:cafe"
45		−readonly IP6_DST="::2:dead:beef:cafe"
46		−
47		−readonly IP4_SLL="169.254.0.1"
48		−readonly IP4_DLL="169.254.0.2"
49		−readonly IP4_NET="169.254.0.0"
50		−
51		−cleanup()
52		−{
53		− ip netns del ${NS_SRC}
54		− ip netns del ${NS_FWD}
55		− ip netns del ${NS_DST}
56		−}
57		−
58		−trap cleanup EXIT
59		−
60		−set -e
61		−
62		−ip netns add "${NS_SRC}"
63		−ip netns add "${NS_FWD}"
64		−ip netns add "${NS_DST}"
65		−
66		−ip link add veth_src type veth peer name veth_src_fwd
67		−ip link add veth_dst type veth peer name veth_dst_fwd
68		−
69		−ip link set veth_src netns ${NS_SRC}
70		−ip link set veth_src_fwd netns ${NS_FWD}
71		−
72		−ip link set veth_dst netns ${NS_DST}
73		−ip link set veth_dst_fwd netns ${NS_FWD}
74		−
75		−ip -netns ${NS_SRC} addr add ${IP4_SRC}/32 dev veth_src
76		−ip -netns ${NS_DST} addr add ${IP4_DST}/32 dev veth_dst
77		−
78		−# The fwd netns automatically get a v6 LL address / routes, but also needs v4
79		−# one in order to start ARP probing. IP4_NET route is added to the endpoints
80		−# so that the ARP processing will reply.
81		−
82		−ip -netns ${NS_FWD} addr add ${IP4_SLL}/32 dev veth_src_fwd
83		−ip -netns ${NS_FWD} addr add ${IP4_DLL}/32 dev veth_dst_fwd
84		−
85		−ip -netns ${NS_SRC} addr add ${IP6_SRC}/128 dev veth_src nodad
86		−ip -netns ${NS_DST} addr add ${IP6_DST}/128 dev veth_dst nodad
87		−
88		−ip -netns ${NS_SRC} link set dev veth_src up
89		−ip -netns ${NS_FWD} link set dev veth_src_fwd up
90		−
91		−ip -netns ${NS_DST} link set dev veth_dst up
92		−ip -netns ${NS_FWD} link set dev veth_dst_fwd up
93		−
94		−ip -netns ${NS_SRC} route add ${IP4_DST}/32 dev veth_src scope global
95		−ip -netns ${NS_SRC} route add ${IP4_NET}/16 dev veth_src scope global
96		−ip -netns ${NS_FWD} route add ${IP4_SRC}/32 dev veth_src_fwd scope global
97		−
98		−ip -netns ${NS_SRC} route add ${IP6_DST}/128 dev veth_src scope global
99		−ip -netns ${NS_FWD} route add ${IP6_SRC}/128 dev veth_src_fwd scope global
100		−
101		−ip -netns ${NS_DST} route add ${IP4_SRC}/32 dev veth_dst scope global
102		−ip -netns ${NS_DST} route add ${IP4_NET}/16 dev veth_dst scope global
103		−ip -netns ${NS_FWD} route add ${IP4_DST}/32 dev veth_dst_fwd scope global
104		−
105		−ip -netns ${NS_DST} route add ${IP6_SRC}/128 dev veth_dst scope global
106		−ip -netns ${NS_FWD} route add ${IP6_DST}/128 dev veth_dst_fwd scope global
107		−
108		−fmac_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/address)
109		−fmac_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/address)
110		−
111		−ip -netns ${NS_SRC} neigh add ${IP4_DST} dev veth_src lladdr $fmac_src
112		−ip -netns ${NS_DST} neigh add ${IP4_SRC} dev veth_dst lladdr $fmac_dst
113		−
114		−ip -netns ${NS_SRC} neigh add ${IP6_DST} dev veth_src lladdr $fmac_src
115		−ip -netns ${NS_DST} neigh add ${IP6_SRC} dev veth_dst lladdr $fmac_dst
116		−
117		−veth_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/ifindex \| awk '{printf "%08x\n", $1}')
118		−veth_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/ifindex \| awk '{printf "%08x\n", $1}')
119		−
120		−xxd -p < test_tc_neigh.o \| sed "s/eeddddee/$veth_src/g" \| xxd -r -p > test_tc_neigh.x.o
121		−xxd -p < test_tc_neigh.x.o \| sed "s/eeffffee/$veth_dst/g" \| xxd -r -p > test_tc_neigh.y.o
122		−
123		−ip netns exec ${NS_FWD} tc qdisc add dev veth_src_fwd clsact
124		−ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd ingress bpf da obj test_tc_neigh.y.o sec src_ingress
125		−ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh
126		−
127		−ip netns exec ${NS_FWD} tc qdisc add dev veth_dst_fwd clsact
128		−ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd ingress bpf da obj test_tc_neigh.y.o sec dst_ingress
129		−ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh
130		−
131		−rm -f test_tc_neigh.x.o test_tc_neigh.y.o
132		−
133		−ip netns exec ${NS_DST} bash -c "nc -4 -l -p 9004 &"
134		−ip netns exec ${NS_DST} bash -c "nc -6 -l -p 9006 &"
135		−
136		−set +e
137		−
138		−TEST="TCPv4 connectivity test"
139		−ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP4_DST}/9004"
140		−if [ $? -ne 0 ]; then
141		− echo -e "${TEST}: ${RED}FAIL${NC}"
142		− exit 1
143		−fi
144		−echo -e "${TEST}: ${GREEN}PASS${NC}"
145		−
146		−TEST="TCPv6 connectivity test"
147		−ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP6_DST}/9006"
148		−if [ $? -ne 0 ]; then
149		− echo -e "${TEST}: ${RED}FAIL${NC}"
150		− exit 1
151		−fi
152		−echo -e "${TEST}: ${GREEN}PASS${NC}"
153		−
154		−TEST="ICMPv4 connectivity test"
155		−ip netns exec ${NS_SRC} ping $PING_ARG ${IP4_DST}
156		−if [ $? -ne 0 ]; then
157		− echo -e "${TEST}: ${RED}FAIL${NC}"
158		− exit 1
159		−fi
160		−echo -e "${TEST}: ${GREEN}PASS${NC}"
161		−
162		−TEST="ICMPv6 connectivity test"
163		−ip netns exec ${NS_SRC} ping6 $PING_ARG ${IP6_DST}
164		−if [ $? -ne 0 ]; then
165		− echo -e "${TEST}: ${RED}FAIL${NC}"
166		− exit 1
167		−fi
168		−echo -e "${TEST}: ${GREEN}PASS${NC}"

tools/testing/selftests/bpf/test_tc_redirect.sh+204−0

@@ -0,0 +1,204 @@
	1	+#!/bin/bash
	2	+# SPDX-License-Identifier: GPL-2.0
	3	+#
	4	+# This test sets up 3 netns (src <-> fwd <-> dst). There is no direct veth link
	5	+# between src and dst. The netns fwd has veth links to each src and dst. The
	6	+# client is in src and server in dst. The test installs a TC BPF program to each
	7	+# host facing veth in fwd which calls into i) bpf_redirect_neigh() to perform the
	8	+# neigh addr population and redirect or ii) bpf_redirect_peer() for namespace
	9	+# switch from ingress side; it also installs a checker prog on the egress side
	10	+# to drop unexpected traffic.
	11	+
	12	+if [[ $EUID -ne 0 ]]; then
	13	+ echo "This script must be run as root"
	14	+ echo "FAIL"
	15	+ exit 1
	16	+fi
	17	+
	18	+# check that needed tools are present
	19	+command -v nc >/dev/null 2>&1 \|\| \
	20	+ { echo >&2 "nc is not available"; exit 1; }
	21	+command -v dd >/dev/null 2>&1 \|\| \
	22	+ { echo >&2 "dd is not available"; exit 1; }
	23	+command -v timeout >/dev/null 2>&1 \|\| \
	24	+ { echo >&2 "timeout is not available"; exit 1; }
	25	+command -v ping >/dev/null 2>&1 \|\| \
	26	+ { echo >&2 "ping is not available"; exit 1; }
	27	+command -v ping6 >/dev/null 2>&1 \|\| \
	28	+ { echo >&2 "ping6 is not available"; exit 1; }
	29	+command -v perl >/dev/null 2>&1 \|\| \
	30	+ { echo >&2 "perl is not available"; exit 1; }
	31	+command -v jq >/dev/null 2>&1 \|\| \
	32	+ { echo >&2 "jq is not available"; exit 1; }
	33	+command -v bpftool >/dev/null 2>&1 \|\| \
	34	+ { echo >&2 "bpftool is not available"; exit 1; }
	35	+
	36	+readonly GREEN='\033[0;92m'
	37	+readonly RED='\033[0;31m'
	38	+readonly NC='\033[0m' # No Color
	39	+
	40	+readonly PING_ARG="-c 3 -w 10 -q"
	41	+
	42	+readonly TIMEOUT=10
	43	+
	44	+readonly NS_SRC="ns-src-$(mktemp -u XXXXXX)"
	45	+readonly NS_FWD="ns-fwd-$(mktemp -u XXXXXX)"
	46	+readonly NS_DST="ns-dst-$(mktemp -u XXXXXX)"
	47	+
	48	+readonly IP4_SRC="172.16.1.100"
	49	+readonly IP4_DST="172.16.2.100"
	50	+
	51	+readonly IP6_SRC="::1:dead:beef:cafe"
	52	+readonly IP6_DST="::2:dead:beef:cafe"
	53	+
	54	+readonly IP4_SLL="169.254.0.1"
	55	+readonly IP4_DLL="169.254.0.2"
	56	+readonly IP4_NET="169.254.0.0"
	57	+
	58	+netns_cleanup()
	59	+{
	60	+ ip netns del ${NS_SRC}
	61	+ ip netns del ${NS_FWD}
	62	+ ip netns del ${NS_DST}
	63	+}
	64	+
	65	+netns_setup()
	66	+{
	67	+ ip netns add "${NS_SRC}"
	68	+ ip netns add "${NS_FWD}"
	69	+ ip netns add "${NS_DST}"
	70	+
	71	+ ip link add veth_src type veth peer name veth_src_fwd
	72	+ ip link add veth_dst type veth peer name veth_dst_fwd
	73	+
	74	+ ip link set veth_src netns ${NS_SRC}
	75	+ ip link set veth_src_fwd netns ${NS_FWD}
	76	+
	77	+ ip link set veth_dst netns ${NS_DST}
	78	+ ip link set veth_dst_fwd netns ${NS_FWD}
	79	+
	80	+ ip -netns ${NS_SRC} addr add ${IP4_SRC}/32 dev veth_src
	81	+ ip -netns ${NS_DST} addr add ${IP4_DST}/32 dev veth_dst
	82	+
	83	+ # The fwd netns automatically get a v6 LL address / routes, but also
	84	+ # needs v4 one in order to start ARP probing. IP4_NET route is added
	85	+ # to the endpoints so that the ARP processing will reply.
	86	+
	87	+ ip -netns ${NS_FWD} addr add ${IP4_SLL}/32 dev veth_src_fwd
	88	+ ip -netns ${NS_FWD} addr add ${IP4_DLL}/32 dev veth_dst_fwd
	89	+
	90	+ ip -netns ${NS_SRC} addr add ${IP6_SRC}/128 dev veth_src nodad
	91	+ ip -netns ${NS_DST} addr add ${IP6_DST}/128 dev veth_dst nodad
	92	+
	93	+ ip -netns ${NS_SRC} link set dev veth_src up
	94	+ ip -netns ${NS_FWD} link set dev veth_src_fwd up
	95	+
	96	+ ip -netns ${NS_DST} link set dev veth_dst up
	97	+ ip -netns ${NS_FWD} link set dev veth_dst_fwd up
	98	+
	99	+ ip -netns ${NS_SRC} route add ${IP4_DST}/32 dev veth_src scope global
	100	+ ip -netns ${NS_SRC} route add ${IP4_NET}/16 dev veth_src scope global
	101	+ ip -netns ${NS_FWD} route add ${IP4_SRC}/32 dev veth_src_fwd scope global
	102	+
	103	+ ip -netns ${NS_SRC} route add ${IP6_DST}/128 dev veth_src scope global
	104	+ ip -netns ${NS_FWD} route add ${IP6_SRC}/128 dev veth_src_fwd scope global
	105	+
	106	+ ip -netns ${NS_DST} route add ${IP4_SRC}/32 dev veth_dst scope global
	107	+ ip -netns ${NS_DST} route add ${IP4_NET}/16 dev veth_dst scope global
	108	+ ip -netns ${NS_FWD} route add ${IP4_DST}/32 dev veth_dst_fwd scope global
	109	+
	110	+ ip -netns ${NS_DST} route add ${IP6_SRC}/128 dev veth_dst scope global
	111	+ ip -netns ${NS_FWD} route add ${IP6_DST}/128 dev veth_dst_fwd scope global
	112	+
	113	+ fmac_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/address)
	114	+ fmac_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/address)
	115	+
	116	+ ip -netns ${NS_SRC} neigh add ${IP4_DST} dev veth_src lladdr $fmac_src
	117	+ ip -netns ${NS_DST} neigh add ${IP4_SRC} dev veth_dst lladdr $fmac_dst
	118	+
	119	+ ip -netns ${NS_SRC} neigh add ${IP6_DST} dev veth_src lladdr $fmac_src
	120	+ ip -netns ${NS_DST} neigh add ${IP6_SRC} dev veth_dst lladdr $fmac_dst
	121	+}
	122	+
	123	+netns_test_connectivity()
	124	+{
	125	+ set +e
	126	+
	127	+ ip netns exec ${NS_DST} bash -c "nc -4 -l -p 9004 &"
	128	+ ip netns exec ${NS_DST} bash -c "nc -6 -l -p 9006 &"
	129	+
	130	+ TEST="TCPv4 connectivity test"
	131	+ ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP4_DST}/9004"
	132	+ if [ $? -ne 0 ]; then
	133	+ echo -e "${TEST}: ${RED}FAIL${NC}"
	134	+ exit 1
	135	+ fi
	136	+ echo -e "${TEST}: ${GREEN}PASS${NC}"
	137	+
	138	+ TEST="TCPv6 connectivity test"
	139	+ ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP6_DST}/9006"
	140	+ if [ $? -ne 0 ]; then
	141	+ echo -e "${TEST}: ${RED}FAIL${NC}"
	142	+ exit 1
	143	+ fi
	144	+ echo -e "${TEST}: ${GREEN}PASS${NC}"
	145	+
	146	+ TEST="ICMPv4 connectivity test"
	147	+ ip netns exec ${NS_SRC} ping $PING_ARG ${IP4_DST}
	148	+ if [ $? -ne 0 ]; then
	149	+ echo -e "${TEST}: ${RED}FAIL${NC}"
	150	+ exit 1
	151	+ fi
	152	+ echo -e "${TEST}: ${GREEN}PASS${NC}"
	153	+
	154	+ TEST="ICMPv6 connectivity test"
	155	+ ip netns exec ${NS_SRC} ping6 $PING_ARG ${IP6_DST}
	156	+ if [ $? -ne 0 ]; then
	157	+ echo -e "${TEST}: ${RED}FAIL${NC}"
	158	+ exit 1
	159	+ fi
	160	+ echo -e "${TEST}: ${GREEN}PASS${NC}"
	161	+
	162	+ set -e
	163	+}
	164	+
	165	+hex_mem_str()
	166	+{
	167	+ perl -e 'print join(" ", unpack("(H2)8", pack("L", @ARGV)))' $1
	168	+}
	169	+
	170	+netns_setup_bpf()
	171	+{
	172	+ local obj=$1
	173	+
	174	+ ip netns exec ${NS_FWD} tc qdisc add dev veth_src_fwd clsact
	175	+ ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd ingress bpf da obj $obj sec src_ingress
	176	+ ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd egress bpf da obj $obj sec chk_egress
	177	+
	178	+ ip netns exec ${NS_FWD} tc qdisc add dev veth_dst_fwd clsact
	179	+ ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd ingress bpf da obj $obj sec dst_ingress
	180	+ ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd egress bpf da obj $obj sec chk_egress
	181	+
	182	+ veth_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/ifindex)
	183	+ veth_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/ifindex)
	184	+
	185	+ progs=$(ip netns exec ${NS_FWD} bpftool net --json \| jq -r '.[] \| .tc \| map(.id) \| .[]')
	186	+ for prog in $progs; do
	187	+ map=$(bpftool prog show id $prog --json \| jq -r '.map_ids \| .? \| .[]')
	188	+ if [ ! -z "$map" ]; then
	189	+ bpftool map update id $map key hex $(hex_mem_str 0) value hex $(hex_mem_str $veth_src)
	190	+ bpftool map update id $map key hex $(hex_mem_str 1) value hex $(hex_mem_str $veth_dst)
	191	+ fi
	192	+ done
	193	+}
	194	+
	195	+trap netns_cleanup EXIT
	196	+set -e
	197	+
	198	+netns_setup
	199	+netns_setup_bpf test_tc_neigh.o
	200	+netns_test_connectivity
	201	+netns_cleanup
	202	+netns_setup
	203	+netns_setup_bpf test_tc_peer.o
	204	+netns_test_connectivity

tools/testing/selftests/bpf/test_tcp_hdr_options.h+3−2

@@ -110,12 +110,13 @@ static inline void clear_hdr_cb_flags(struct bpf_sock_ops *skops) · 109 unmodified lines
110	110	BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG));
111	111	}
112	112
113		−static inline void set_hdr_cb_flags(struct bpf_sock_ops *skops)
	113	+static inline void set_hdr_cb_flags(struct bpf_sock_ops *skops, __u32 extra)
114	114	{
115	115	bpf_sock_ops_cb_flags_set(skops,
116	116	skops->bpf_sock_ops_cb_flags \|
117	117	BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG \|
118		− BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG);
	118	+ BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG \|
	119	+ extra);
119	120	}
120	121	static inline void
121	122	clear_parse_all_hdr_cb_flags(struct bpf_sock_ops *skops)

tools/testing/selftests/bpf/trace_helpers.c+27−0

@@ -90,6 +90,33 @@ long ksym_get_addr(const char *name) · 89 unmodified lines
90	90	return 0;
91	91	}
92	92
	93	+/* open kallsyms and read symbol addresses on the fly. Without caching all symbols,
	94	+ * this is faster than load + find.
	95	+ */
	96	+int kallsyms_find(const char sym, unsigned long long addr)
	97	+{
	98	+ char type, name[500];
	99	+ unsigned long long value;
	100	+ int err = 0;
	101	+ FILE *f;
	102	+
	103	+ f = fopen("/proc/kallsyms", "r");
	104	+ if (!f)
	105	+ return -EINVAL;
	106	+
	107	+ while (fscanf(f, "%llx %c %499s%*[^\n]\n", &value, &type, name) > 0) {
	108	+ if (strcmp(name, sym) == 0) {
	109	+ *addr = value;
	110	+ goto out;
	111	+ }
	112	+ }
	113	+ err = -ENOENT;
	114	+
	115	+out:
	116	+ fclose(f);
	117	+ return err;
	118	+}
	119	+
93	120	void read_trace_pipe(void)
94	121	{
95	122	int trace_fd;

tools/testing/selftests/bpf/trace_helpers.h+4−0

@@ -12,6 +12,10 @@ struct ksym { · 11 unmodified lines
12	12	int load_kallsyms(void);
13	13	struct ksym *ksym_search(long key);
14	14	long ksym_get_addr(const char *name);
	15	+
	16	+/* open kallsyms and find addresses on the fly, faster than load + search. */
	17	+int kallsyms_find(const char sym, unsigned long long addr);
	18	+
15	19	void read_trace_pipe(void);
16	20
17	21	#endif

tools/testing/selftests/bpf/verifier/basic.c+1−1

@@ -2,7 +2,7 @@ · 1 unmodified line
2	2	"empty prog",
3	3	.insns = {
4	4	},
5		− .errstr = "unknown opcode 00",
	5	+ .errstr = "last insn is not an exit or jmp",
6	6	.result = REJECT,
7	7	},
8	8	{

tools/testing/selftests/bpf/verifier/direct_packet_access.c+1−1

@@ -529,7 +529,7 @@ · 528 unmodified lines
529	529	},
530	530	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
531	531	.result = REJECT,
532		− .errstr = "invalid access to packet, off=0 size=8, R5(id=1,off=0,r=0)",
	532	+ .errstr = "invalid access to packet, off=0 size=8, R5(id=2,off=0,r=0)",
533	533	.flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
534	534	},
535	535	{

tools/testing/selftests/bpf/verifier/ld_imm64.c+0−8

@@ -50,14 +50,6 @@ · 49 unmodified lines
50	50	.errstr = "invalid bpf_ld_imm64 insn",
51	51	.result = REJECT,
52	52	},
53		−{
54		− "test5 ld_imm64",
55		− .insns = {
56		− BPF_RAW_INSN(BPF_LD \| BPF_IMM \| BPF_DW, 0, 0, 0, 0),
57		− },
58		− .errstr = "invalid bpf_ld_imm64 insn",
59		− .result = REJECT,
60		−},
61	53	{
62	54	"test6 ld_imm64",
63	55	.insns = {

tools/testing/selftests/bpf/verifier/regalloc.c+243−0

@@ -0,0 +1,243 @@
	1	+{
	2	+ "regalloc basic",
	3	+ .insns = {
	4	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	5	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	6	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	7	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	8	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	9	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	10	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 8),
	11	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	12	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	13	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	14	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 20, 4),
	15	+ BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 3),
	16	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	17	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	18	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	19	+ BPF_EXIT_INSN(),
	20	+ },
	21	+ .fixup_map_hash_48b = { 4 },
	22	+ .result = ACCEPT,
	23	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	24	+},
	25	+{
	26	+ "regalloc negative",
	27	+ .insns = {
	28	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	29	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	30	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	31	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	32	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	33	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	34	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 8),
	35	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	36	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	37	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	38	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 24, 4),
	39	+ BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 3),
	40	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	41	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	42	+ BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_7, 0),
	43	+ BPF_EXIT_INSN(),
	44	+ },
	45	+ .fixup_map_hash_48b = { 4 },
	46	+ .result = REJECT,
	47	+ .errstr = "invalid access to map value, value_size=48 off=48 size=1",
	48	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	49	+},
	50	+{
	51	+ "regalloc src_reg mark",
	52	+ .insns = {
	53	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	54	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	55	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	56	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	57	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	58	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	59	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 9),
	60	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	61	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	62	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	63	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 20, 5),
	64	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	65	+ BPF_JMP_REG(BPF_JSGE, BPF_REG_3, BPF_REG_2, 3),
	66	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	67	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	68	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	69	+ BPF_EXIT_INSN(),
	70	+ },
	71	+ .fixup_map_hash_48b = { 4 },
	72	+ .result = ACCEPT,
	73	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	74	+},
	75	+{
	76	+ "regalloc src_reg negative",
	77	+ .insns = {
	78	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	79	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	80	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	81	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	82	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	83	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	84	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 9),
	85	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	86	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	87	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	88	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 22, 5),
	89	+ BPF_MOV64_IMM(BPF_REG_3, 0),
	90	+ BPF_JMP_REG(BPF_JSGE, BPF_REG_3, BPF_REG_2, 3),
	91	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	92	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	93	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	94	+ BPF_EXIT_INSN(),
	95	+ },
	96	+ .fixup_map_hash_48b = { 4 },
	97	+ .result = REJECT,
	98	+ .errstr = "invalid access to map value, value_size=48 off=44 size=8",
	99	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	100	+},
	101	+{
	102	+ "regalloc and spill",
	103	+ .insns = {
	104	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	105	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	106	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	107	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	108	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	109	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	110	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 11),
	111	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	112	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	113	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	114	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 20, 7),
	115	+ /* r0 has upper bound that should propagate into r2 */
	116	+ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -8), /* spill r2 */
	117	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	118	+ BPF_MOV64_IMM(BPF_REG_2, 0), /* clear r0 and r2 */
	119	+ BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_10, -8), /* fill r3 */
	120	+ BPF_JMP_REG(BPF_JSGE, BPF_REG_0, BPF_REG_3, 2),
	121	+ /* r3 has lower and upper bounds */
	122	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_3),
	123	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	124	+ BPF_EXIT_INSN(),
	125	+ },
	126	+ .fixup_map_hash_48b = { 4 },
	127	+ .result = ACCEPT,
	128	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	129	+},
	130	+{
	131	+ "regalloc and spill negative",
	132	+ .insns = {
	133	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	134	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	135	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	136	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	137	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	138	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	139	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 11),
	140	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	141	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	142	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	143	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 48, 7),
	144	+ /* r0 has upper bound that should propagate into r2 */
	145	+ BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -8), /* spill r2 */
	146	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	147	+ BPF_MOV64_IMM(BPF_REG_2, 0), /* clear r0 and r2 */
	148	+ BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_10, -8), /* fill r3 */
	149	+ BPF_JMP_REG(BPF_JSGE, BPF_REG_0, BPF_REG_3, 2),
	150	+ /* r3 has lower and upper bounds */
	151	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_3),
	152	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	153	+ BPF_EXIT_INSN(),
	154	+ },
	155	+ .fixup_map_hash_48b = { 4 },
	156	+ .result = REJECT,
	157	+ .errstr = "invalid access to map value, value_size=48 off=48 size=8",
	158	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	159	+},
	160	+{
	161	+ "regalloc three regs",
	162	+ .insns = {
	163	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	164	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	165	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	166	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	167	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	168	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	169	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10),
	170	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	171	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	172	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	173	+ BPF_MOV64_REG(BPF_REG_4, BPF_REG_2),
	174	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 12, 5),
	175	+ BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 4),
	176	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	177	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	178	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_4),
	179	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	180	+ BPF_EXIT_INSN(),
	181	+ },
	182	+ .fixup_map_hash_48b = { 4 },
	183	+ .result = ACCEPT,
	184	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	185	+},
	186	+{
	187	+ "regalloc after call",
	188	+ .insns = {
	189	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	190	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	191	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	192	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	193	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	194	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	195	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10),
	196	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	197	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	198	+ BPF_MOV64_REG(BPF_REG_8, BPF_REG_0),
	199	+ BPF_MOV64_REG(BPF_REG_9, BPF_REG_0),
	200	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, 6),
	201	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_8, 20, 4),
	202	+ BPF_JMP_IMM(BPF_JSLT, BPF_REG_9, 0, 3),
	203	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_8),
	204	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_9),
	205	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	206	+ BPF_EXIT_INSN(),
	207	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	208	+ BPF_EXIT_INSN(),
	209	+ },
	210	+ .fixup_map_hash_48b = { 4 },
	211	+ .result = ACCEPT,
	212	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	213	+},
	214	+{
	215	+ "regalloc in callee",
	216	+ .insns = {
	217	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	218	+ BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	219	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	220	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	221	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	222	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	223	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 6),
	224	+ BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	225	+ BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	226	+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
	227	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	228	+ BPF_MOV64_REG(BPF_REG_3, BPF_REG_7),
	229	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, 1),
	230	+ BPF_EXIT_INSN(),
	231	+ BPF_JMP_IMM(BPF_JSGT, BPF_REG_1, 20, 5),
	232	+ BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 4),
	233	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_3, BPF_REG_1),
	234	+ BPF_ALU64_REG(BPF_ADD, BPF_REG_3, BPF_REG_2),
	235	+ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0),
	236	+ BPF_EXIT_INSN(),
	237	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	238	+ BPF_EXIT_INSN(),
	239	+ },
	240	+ .fixup_map_hash_48b = { 4 },
	241	+ .result = ACCEPT,
	242	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	243	+},