lael-labs/linux

Documentation/bpf/bpf_devel_QA.rst+2−2

@@ -60,13 +60,13 @@ Q: Where can I find patches currently under discussion for BPF subsystem? · 59 unmodified lines
60	A: All patches that are Cc'ed to netdev are queued for review under netdev	60	A: All patches that are Cc'ed to netdev are queued for review under netdev
61	patchwork project:	61	patchwork project:
62		62
63	http://patchwork.ozlabs.org/project/netdev/list/	63	https://patchwork.kernel.org/project/netdevbpf/list/
64		64
65	Those patches which target BPF, are assigned to a 'bpf' delegate for	65	Those patches which target BPF, are assigned to a 'bpf' delegate for
66	further processing from BPF maintainers. The current queue with	66	further processing from BPF maintainers. The current queue with
67	patches under review can be found at:	67	patches under review can be found at:
68		68
69	https://patchwork.ozlabs.org/project/netdev/list/?delegate=77147	69	https://patchwork.kernel.org/project/netdevbpf/list/?delegate=121173
70		70
71	Once the patches have been reviewed by the BPF community as a whole	71	Once the patches have been reviewed by the BPF community as a whole
72	and approved by the BPF maintainers, their status in patchwork will be	72	and approved by the BPF maintainers, their status in patchwork will be

MAINTAINERS+1−1

@@ -3263,7 +3263,7 @@ M: Daniel Borkmann <daniel@iogearbox.net> · 3262 unmodified lines
3263	R: Martin KaFai Lau <kafai@fb.com>	3263	R: Martin KaFai Lau <kafai@fb.com>
3264	R: Song Liu <songliubraving@fb.com>	3264	R: Song Liu <songliubraving@fb.com>
3265	R: Yonghong Song <yhs@fb.com>	3265	R: Yonghong Song <yhs@fb.com>
3266	R: Andrii Nakryiko <andriin@fb.com>	3266	R: Andrii Nakryiko <andrii@kernel.org>
3267	R: John Fastabend <john.fastabend@gmail.com>	3267	R: John Fastabend <john.fastabend@gmail.com>
3268	R: KP Singh <kpsingh@chromium.org>	3268	R: KP Singh <kpsingh@chromium.org>
3269	L: netdev@vger.kernel.org	3269	L: netdev@vger.kernel.org

drivers/net/veth.c+9−0

 	return smp_processor_id() % dev->real_num_rx_queues;
 }
+static struct net_device *veth_peer_dev(struct net_device *dev)
+{
+	struct veth_priv *priv = netdev_priv(dev);
+	/* Callers must be under RCU read side. */
+	return rcu_dereference(priv->peer);
+}
 static int veth_xdp_xmit(struct net_device *dev, int n,
 			 struct xdp_frame **frames,
 			 u32 flags, bool ndo_xmit)
 	.ndo_set_rx_headroom	= veth_set_rx_headroom,
 	.ndo_bpf		= veth_xdp,
 	.ndo_xdp_xmit		= veth_ndo_xdp_xmit,
+	.ndo_get_peer_dev	= veth_peer_dev,
 };
 #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \

include/linux/bpf.h+7−1

 	void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file,
 				int fd);
 	void (*map_fd_put_ptr)(void *ptr);
-	u32 (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
+	int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf);
 	u32 (*map_fd_sys_lookup_elem)(void *ptr);
 	void (*map_seq_show_elem)(struct bpf_map *map, void *key,
 				  struct seq_file *m);
 	ARG_PTR_TO_ALLOC_MEM_OR_NULL,	/* pointer to dynamically allocated memory or NULL */
 	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */
 	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
+	ARG_PTR_TO_PERCPU_BTF_ID,	/* pointer to in-kernel percpu type */
 	__BPF_ARG_TYPE_MAX,
 };
 	RET_PTR_TO_SOCK_COMMON_OR_NULL,	/* returns a pointer to a sock_common or NULL */
 	RET_PTR_TO_ALLOC_MEM_OR_NULL,	/* returns a pointer to dynamically allocated memory or NULL */
 	RET_PTR_TO_BTF_ID_OR_NULL,	/* returns a pointer to a btf_id or NULL */
+	RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
+	RET_PTR_TO_MEM_OR_BTF_ID,	/* returns a pointer to a valid memory or a btf_id */
 };
 /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
 	PTR_TO_RDONLY_BUF_OR_NULL, /* reg points to a readonly buffer or NULL */
 	PTR_TO_RDWR_BUF,	 /* reg points to a read/write buffer */
 	PTR_TO_RDWR_BUF_OR_NULL, /* reg points to a read/write buffer or NULL */
+	PTR_TO_PERCPU_BTF_ID,	 /* reg points to a percpu kernel variable */
 };
 /* The information passed from prog-specific *_is_valid_access
 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
 extern const struct bpf_func_proto bpf_copy_from_user_proto;
 extern const struct bpf_func_proto bpf_snprintf_btf_proto;
+extern const struct bpf_func_proto bpf_per_cpu_ptr_proto;
+extern const struct bpf_func_proto bpf_this_cpu_ptr_proto;
 const struct bpf_func_proto *bpf_tracing_func_proto(
 	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_verifier.h+7−0

@@ -308,6 +308,13 @@ struct bpf_insn_aux_data { · 307 unmodified lines
308	u32 map_index; /* index into used_maps[] */	308	u32 map_index; /* index into used_maps[] */
309	u32 map_off; /* offset from value base address */	309	u32 map_off; /* offset from value base address */
310	};	310	};
		311	struct {
		312	enum bpf_reg_type reg_type; /* type of pseudo_btf_id */
		313	union {
		314	u32 btf_id; /* btf_id for struct typed var */
		315	u32 mem_size; /* mem_size for non-struct typed var */
		316	};
		317	} btf_var;
311	};	318	};
312	u64 map_key_state; /* constant (32 bit) key tracking for maps */	319	u64 map_key_state; /* constant (32 bit) key tracking for maps */
313	int ctx_field_size; /* the ctx field size for load insn, maybe 0 */	320	int ctx_field_size; /* the ctx field size for load insn, maybe 0 */

include/linux/btf.h+26−0

@@ -110,6 +110,11 @@ btf_resolve_size(const struct btf btf, const struct btf_type type, · 109 unmodified lines
110	i < btf_type_vlen(struct_type); \	110	i < btf_type_vlen(struct_type); \
111	i++, member++)	111	i++, member++)
112		112
		113	#define for_each_vsi(i, datasec_type, member) \
		114	for (i = 0, member = btf_type_var_secinfo(datasec_type); \
		115	i < btf_type_vlen(datasec_type); \
		116	i++, member++)
		117
113	static inline bool btf_type_is_ptr(const struct btf_type *t)	118	static inline bool btf_type_is_ptr(const struct btf_type *t)
114	{	119	{
115	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;	120	return BTF_INFO_KIND(t->info) == BTF_KIND_PTR;
@@ -145,6 +150,21 @@ static inline bool btf_type_is_func_proto(const struct btf_type *t) · 29 unmodified lines
145	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;	150	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
146	}	151	}
147		152
		153	static inline bool btf_type_is_var(const struct btf_type *t)
		154	{
		155	return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
		156	}
		157
		158	/* union is only a special case of struct:
		159	* all its offsetof(member) == 0
		160	*/
		161	static inline bool btf_type_is_struct(const struct btf_type *t)
		162	{
		163	u8 kind = BTF_INFO_KIND(t->info);
		164
		165	return kind == BTF_KIND_STRUCT \|\| kind == BTF_KIND_UNION;
		166	}
		167
148	static inline u16 btf_type_vlen(const struct btf_type *t)	168	static inline u16 btf_type_vlen(const struct btf_type *t)
149	{	169	{
150	return BTF_INFO_VLEN(t->info);	170	return BTF_INFO_VLEN(t->info);
@@ -179,6 +199,12 @@ static inline const struct btf_member btf_type_member(const struct btf_type t) · 28 unmodified lines
179	return (const struct btf_member *)(t + 1);	199	return (const struct btf_member *)(t + 1);
180	}	200	}
181		201
		202	static inline const struct btf_var_secinfo *btf_type_var_secinfo(
		203	const struct btf_type *t)
		204	{
		205	return (const struct btf_var_secinfo *)(t + 1);
		206	}
		207
182	#ifdef CONFIG_BPF_SYSCALL	208	#ifdef CONFIG_BPF_SYSCALL
183	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);	209	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);
184	const char btf_name_by_offset(const struct btf btf, u32 offset);	210	const char btf_name_by_offset(const struct btf btf, u32 offset);

include/linux/netdevice.h+4−0

  * int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm *p,
  *			 int cmd);
  *	Add, change, delete or get information on an IPv4 tunnel.
+ * struct net_device *(*ndo_get_peer_dev)(struct net_device *dev);
+ *	If a device is paired with a peer device, return the peer instance.
+ *	The caller must be under RCU read context.
  */
 struct net_device_ops {
 	int			(*ndo_init)(struct net_device *dev);
 	struct devlink_port *	(*ndo_get_devlink_port)(struct net_device *dev);
 	int			(*ndo_tunnel_ctl)(struct net_device *dev,
 						  struct ip_tunnel_parm *p, int cmd);
+	struct net_device *	(*ndo_get_peer_dev)(struct net_device *dev);
 };
 /**

include/linux/skmsg.h+2−0

@@ -308,6 +308,8 @@ struct sk_psock sk_psock_init(struct sock sk, int node); · 307 unmodified lines
308	int sk_psock_init_strp(struct sock sk, struct sk_psock psock);	308	int sk_psock_init_strp(struct sock sk, struct sk_psock psock);
309	void sk_psock_start_strp(struct sock sk, struct sk_psock psock);	309	void sk_psock_start_strp(struct sock sk, struct sk_psock psock);
310	void sk_psock_stop_strp(struct sock sk, struct sk_psock psock);	310	void sk_psock_stop_strp(struct sock sk, struct sk_psock psock);
		311	void sk_psock_start_verdict(struct sock sk, struct sk_psock psock);
		312	void sk_psock_stop_verdict(struct sock sk, struct sk_psock psock);
311		313
312	int sk_psock_msg_verdict(struct sock sk, struct sk_psock psock,	314	int sk_psock_msg_verdict(struct sock sk, struct sk_psock psock,
313	struct sk_msg *msg);	315	struct sk_msg *msg);

include/net/tcp.h+0−33

@@ -2228,34 +2228,6 @@ int __tcp_bpf_recvmsg(struct sock sk, struct sk_psock psock, · 2227 unmodified lines
2228	#endif /* CONFIG_NET_SOCK_MSG */	2228	#endif /* CONFIG_NET_SOCK_MSG */
2229		2229
2230	#ifdef CONFIG_CGROUP_BPF	2230	#ifdef CONFIG_CGROUP_BPF
2231	/* Copy the listen sk's HDR_OPT_CB flags to its child.
2232	*
2233	* During 3-Way-HandShake, the synack is usually sent from
2234	* the listen sk with the HDR_OPT_CB flags set so that
2235	* bpf-prog will be called to write the BPF hdr option.
2236	*
2237	* In fastopen, the child sk is used to send synack instead
2238	* of the listen sk. Thus, inheriting the HDR_OPT_CB flags
2239	* from the listen sk gives the bpf-prog a chance to write
2240	* BPF hdr option in the synack pkt during fastopen.
2241	*
2242	* Both fastopen and non-fastopen child will inherit the
2243	* HDR_OPT_CB flags to keep the bpf-prog having a consistent
2244	* behavior when deciding to clear this cb flags (or not)
2245	* during the PASSIVE_ESTABLISHED_CB.
2246	*
2247	* In the future, other cb flags could be inherited here also.
2248	*/
2249	static inline void bpf_skops_init_child(const struct sock *sk,
2250	struct sock *child)
2251	{
2252	tcp_sk(child)->bpf_sock_ops_cb_flags =
2253	tcp_sk(sk)->bpf_sock_ops_cb_flags &
2254	(BPF_SOCK_OPS_PARSE_ALL_HDR_OPT_CB_FLAG \|
2255	BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG \|
2256	BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG);
2257	}
2258
2259	static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,	2231	static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2260	struct sk_buff *skb,	2232	struct sk_buff *skb,
2261	unsigned int end_offset)	2233	unsigned int end_offset)
@@ -2264,11 +2236,6 @@ static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops, · 2 unmodified lines
2264	skops->skb_data_end = skb->data + end_offset;	2236	skops->skb_data_end = skb->data + end_offset;
2265	}	2237	}
2266	#else	2238	#else
2267	static inline void bpf_skops_init_child(const struct sock *sk,
2268	struct sock *child)
2269	{
2270	}
2271
2272	static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,	2239	static inline void bpf_skops_init_skb(struct bpf_sock_ops_kern *skops,
2273	struct sk_buff *skb,	2240	struct sk_buff *skb,
2274	unsigned int end_offset)	2241	unsigned int end_offset)

include/uapi/linux/bpf.h+87−14

@@ -356,18 +356,36 @@ enum bpf_link_type { · 355 unmodified lines
356	#define BPF_F_SLEEPABLE (1U << 4)	356	#define BPF_F_SLEEPABLE (1U << 4)
357		357
358	/* When BPF ldimm64's insn[0].src_reg != 0 then this can have	358	/* When BPF ldimm64's insn[0].src_reg != 0 then this can have
359	* two extensions:	359	* the following extensions:
360	*	360	*
361	* insn[0].src_reg: BPF_PSEUDO_MAP_FD BPF_PSEUDO_MAP_VALUE	361	* insn[0].src_reg: BPF_PSEUDO_MAP_FD
362	* insn[0].imm: map fd map fd	362	* insn[0].imm: map fd
363	* insn[1].imm: 0 offset into value	363	* insn[1].imm: 0
364	* insn[0].off: 0 0	364	* insn[0].off: 0
365	* insn[1].off: 0 0	365	* insn[1].off: 0
366	* ldimm64 rewrite: address of map address of map[0]+offset	366	* ldimm64 rewrite: address of map
367	* verifier type: CONST_PTR_TO_MAP PTR_TO_MAP_VALUE	367	* verifier type: CONST_PTR_TO_MAP
368	*/	368	*/
369	#define BPF_PSEUDO_MAP_FD 1	369	#define BPF_PSEUDO_MAP_FD 1
		370	/* insn[0].src_reg: BPF_PSEUDO_MAP_VALUE
		371	* insn[0].imm: map fd
		372	* insn[1].imm: offset into value
		373	* insn[0].off: 0
		374	* insn[1].off: 0
		375	* ldimm64 rewrite: address of map[0]+offset
		376	* verifier type: PTR_TO_MAP_VALUE
		377	*/
370	#define BPF_PSEUDO_MAP_VALUE 2	378	#define BPF_PSEUDO_MAP_VALUE 2
		379	/* insn[0].src_reg: BPF_PSEUDO_BTF_ID
		380	* insn[0].imm: kernel btd id of VAR
		381	* insn[1].imm: 0
		382	* insn[0].off: 0
		383	* insn[1].off: 0
		384	* ldimm64 rewrite: address of the kernel variable
		385	* verifier type: PTR_TO_BTF_ID or PTR_TO_MEM, depending on whether the var
		386	* is struct/union.
		387	*/
		388	#define BPF_PSEUDO_BTF_ID 3
371		389
372	/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative	390	/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
373	* offset to another bpf function	391	* offset to another bpf function
@@ -417,6 +435,9 @@ enum { · 43 unmodified lines
417		435
418	/* Share perf_event among processes */	436	/* Share perf_event among processes */
419	BPF_F_PRESERVE_ELEMS = (1U << 11),	437	BPF_F_PRESERVE_ELEMS = (1U << 11),
		438
		439	/* Create a map that is suitable to be an inner map with dynamic max entries */
		440	BPF_F_INNER_MAP = (1U << 12),
420	};	441	};
421		442
422	/* Flags for BPF_PROG_QUERY. */	443	/* Flags for BPF_PROG_QUERY. */
@@ -1680,7 +1701,7 @@ union bpf_attr { · 1257 unmodified lines
1680	* TCP_CONGESTION, TCP_BPF_IW,	1701	* TCP_CONGESTION, TCP_BPF_IW,
1681	* TCP_BPF_SNDCWND_CLAMP, TCP_SAVE_SYN,	1702	* TCP_BPF_SNDCWND_CLAMP, TCP_SAVE_SYN,
1682	* TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT,	1703	* TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT,
1683	* TCP_SYNCNT, TCP_USER_TIMEOUT.	1704	* TCP_SYNCNT, TCP_USER_TIMEOUT, TCP_NOTSENT_LOWAT.
1684	* * IPPROTO_IP, which supports optname IP_TOS.	1705	* * IPPROTO_IP, which supports optname IP_TOS.
1685	* * IPPROTO_IPV6, which supports optname IPV6_TCLASS.	1706	* * IPPROTO_IPV6, which supports optname IPV6_TCLASS.
1686	* Return	1707	* Return
@@ -2235,7 +2256,7 @@ union bpf_attr { · 548 unmodified lines
2235	* Description	2256	* Description
2236	* This helper is used in programs implementing policies at the	2257	* This helper is used in programs implementing policies at the
2237	* skb socket level. If the sk_buff skb is allowed to pass (i.e.	2258	* skb socket level. If the sk_buff skb is allowed to pass (i.e.
2238	* if the verdeict eBPF program returns SK_PASS), redirect it	2259	* if the verdict eBPF program returns SK_PASS), redirect it
2239	* to the socket referenced by map (of type	2260	* to the socket referenced by map (of type
2240	* BPF_MAP_TYPE_SOCKHASH) using hash key. Both ingress and	2261	* BPF_MAP_TYPE_SOCKHASH) using hash key. Both ingress and
2241	* egress interfaces can be used for redirection. The	2262	* egress interfaces can be used for redirection. The
@@ -3661,10 +3682,59 @@ union bpf_attr { · 1419 unmodified lines
3661	* Redirect the packet to another net device of index ifindex	3682	* Redirect the packet to another net device of index ifindex
3662	* and fill in L2 addresses from neighboring subsystem. This helper	3683	* and fill in L2 addresses from neighboring subsystem. This helper
3663	* is somewhat similar to bpf_redirect\ (), except that it	3684	* is somewhat similar to bpf_redirect\ (), except that it
3664	* fills in e.g. MAC addresses based on the L3 information from	3685	* populates L2 addresses as well, meaning, internally, the helper
3665	* the packet. This helper is supported for IPv4 and IPv6 protocols.	3686	* performs a FIB lookup based on the skb's networking header to
		3687	* get the address of the next hop and then relies on the neighbor
		3688	* lookup for the L2 address of the nexthop.
		3689	*
		3690	* The flags argument is reserved and must be 0. The helper is
		3691	* currently only supported for tc BPF program types, and enabled
		3692	* for IPv4 and IPv6 protocols.
		3693	* Return
		3694	* The helper returns TC_ACT_REDIRECT on success or
		3695	* TC_ACT_SHOT on error.
		3696	*
		3697	* void bpf_per_cpu_ptr(const void percpu_ptr, u32 cpu)
		3698	* Description
		3699	* Take a pointer to a percpu ksym, percpu_ptr, and return a
		3700	* pointer to the percpu kernel variable on cpu. A ksym is an
		3701	* extern variable decorated with '__ksym'. For ksym, there is a
		3702	* global var (either static or global) defined of the same name
		3703	* in the kernel. The ksym is percpu if the global var is percpu.
		3704	* The returned pointer points to the global percpu var on cpu.
		3705	*
		3706	* bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the
		3707	* kernel, except that bpf_per_cpu_ptr() may return NULL. This
		3708	* happens if cpu is larger than nr_cpu_ids. The caller of
		3709	* bpf_per_cpu_ptr() must check the returned value.
		3710	* Return
		3711	* A pointer pointing to the kernel percpu variable on cpu, or
		3712	* NULL, if cpu is invalid.
		3713	*
		3714	* void bpf_this_cpu_ptr(const void percpu_ptr)
		3715	* Description
		3716	* Take a pointer to a percpu ksym, percpu_ptr, and return a
		3717	* pointer to the percpu kernel variable on this cpu. See the
		3718	* description of 'ksym' in bpf_per_cpu_ptr\ ().
		3719	*
		3720	* bpf_this_cpu_ptr() has the same semantic as this_cpu_ptr() in
		3721	* the kernel. Different from bpf_per_cpu_ptr\ (), it would
		3722	* never return NULL.
		3723	* Return
		3724	* A pointer pointing to the kernel percpu variable on this cpu.
		3725	*
		3726	* long bpf_redirect_peer(u32 ifindex, u64 flags)
		3727	* Description
		3728	* Redirect the packet to another net device of index ifindex.
		3729	* This helper is somewhat similar to bpf_redirect\ (), except
		3730	* that the redirection happens to the ifindex' peer device and
		3731	* the netns switch takes place from ingress to ingress without
		3732	* going through the CPU's backlog queue.
		3733	*
3666	* The flags argument is reserved and must be 0. The helper is	3734	* The flags argument is reserved and must be 0. The helper is
3667	* currently only supported for tc BPF program types.	3735	* currently only supported for tc BPF program types at the ingress
		3736	* hook and for veth device types. The peer device must reside in a
		3737	* different network namespace.
3668	* Return	3738	* Return
3669	* The helper returns TC_ACT_REDIRECT on success or	3739	* The helper returns TC_ACT_REDIRECT on success or
3670	* TC_ACT_SHOT on error.	3740	* TC_ACT_SHOT on error.
@@ -3823,6 +3893,9 @@ union bpf_attr { · 152 unmodified lines
3823	FN(seq_printf_btf), \	3893	FN(seq_printf_btf), \
3824	FN(skb_cgroup_classid), \	3894	FN(skb_cgroup_classid), \
3825	FN(redirect_neigh), \	3895	FN(redirect_neigh), \
		3896	FN(bpf_per_cpu_ptr), \
		3897	FN(bpf_this_cpu_ptr), \
		3898	FN(redirect_peer), \
3826	/* */	3899	/* */
3827		3900
3828	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	3901	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

kernel/bpf/arraymap.c+11−6

 #define ARRAY_CREATE_FLAG_MASK \
 	(BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \
-	 BPF_F_PRESERVE_ELEMS)
+	 BPF_F_PRESERVE_ELEMS | BPF_F_INNER_MAP)
 static void bpf_array_free_percpu(struct bpf_array *array)
 {
 		return -EINVAL;
 	if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
-	    attr->map_flags & BPF_F_MMAPABLE)
+	    attr->map_flags & (BPF_F_MMAPABLE | BPF_F_INNER_MAP))
 		return -EINVAL;
 	if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
 }
 /* emit BPF instructions equivalent to C code of array_map_lookup_elem() */
-static u32 array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
+static int array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf)
 {
 	struct bpf_array *array = container_of(map, struct bpf_array, map);
 	struct bpf_insn *insn = insn_buf;
 	const int map_ptr = BPF_REG_1;
 	const int index = BPF_REG_2;
+	if (map->map_flags & BPF_F_INNER_MAP)
+		return -EOPNOTSUPP;
 	*insn++ = BPF_ALU64_IMM(BPF_ADD, map_ptr, offsetof(struct bpf_array, value));
 	*insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0);
 	if (!map->bypass_spec_v1) {
 static bool array_map_meta_equal(const struct bpf_map *meta0,
 				 const struct bpf_map *meta1)
 {
-	return meta0->max_entries == meta1->max_entries &&
+	if (!bpf_map_meta_equal(meta0, meta1))
-		bpf_map_meta_equal(meta0, meta1);
+		return false;
+	return meta0->map_flags & BPF_F_INNER_MAP ? true :
+	       meta0->max_entries == meta1->max_entries;
 }
 struct bpf_iter_seq_array_map_info {
 	return READ_ONCE(*inner_map);
 }
-static u32 array_of_map_gen_lookup(struct bpf_map *map,
+static int array_of_map_gen_lookup(struct bpf_map *map,
 				   struct bpf_insn *insn_buf)
 {
 	struct bpf_array *array = container_of(map, struct bpf_array, map);

kernel/bpf/btf.c+0−25

@@ -188,11 +188,6 @@ · 187 unmodified lines
188	i < btf_type_vlen(struct_type); \	188	i < btf_type_vlen(struct_type); \
189	i++, member++)	189	i++, member++)
190		190
191	#define for_each_vsi(i, struct_type, member) \
192	for (i = 0, member = btf_type_var_secinfo(struct_type); \
193	i < btf_type_vlen(struct_type); \
194	i++, member++)
195
196	#define for_each_vsi_from(i, from, struct_type, member) \	191	#define for_each_vsi_from(i, from, struct_type, member) \
197	for (i = from, member = btf_type_var_secinfo(struct_type) + from; \	192	for (i = from, member = btf_type_var_secinfo(struct_type) + from; \
198	i < btf_type_vlen(struct_type); \	193	i < btf_type_vlen(struct_type); \
@@ -440,16 +435,6 @@ static bool btf_type_nosize_or_null(const struct btf_type *t) · 241 unmodified lines
440	return !t \|\| btf_type_nosize(t);	435	return !t \|\| btf_type_nosize(t);
441	}	436	}
442		437
443	/* union is only a special case of struct:
444	* all its offsetof(member) == 0
445	*/
446	static bool btf_type_is_struct(const struct btf_type *t)
447	{
448	u8 kind = BTF_INFO_KIND(t->info);
449
450	return kind == BTF_KIND_STRUCT \|\| kind == BTF_KIND_UNION;
451	}
452
453	static bool __btf_type_is_struct(const struct btf_type *t)	438	static bool __btf_type_is_struct(const struct btf_type *t)
454	{	439	{
455	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;	440	return BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT;
@@ -460,11 +445,6 @@ static bool btf_type_is_array(const struct btf_type *t) · 4 unmodified lines
460	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;	445	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
461	}	446	}
462		447
463	static bool btf_type_is_var(const struct btf_type *t)
464	{
465	return BTF_INFO_KIND(t->info) == BTF_KIND_VAR;
466	}
467
468	static bool btf_type_is_datasec(const struct btf_type *t)	448	static bool btf_type_is_datasec(const struct btf_type *t)
469	{	449	{
470	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;	450	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
@@ -613,11 +593,6 @@ static const struct btf_var btf_type_var(const struct btf_type t) · 142 unmodified lines
613	return (const struct btf_var *)(t + 1);	593	return (const struct btf_var *)(t + 1);
614	}	594	}
615		595
616	static const struct btf_var_secinfo btf_type_var_secinfo(const struct btf_type t)
617	{
618	return (const struct btf_var_secinfo *)(t + 1);
619	}
620
621	static const struct btf_kind_operations btf_type_ops(const struct btf_type t)	596	static const struct btf_kind_operations btf_type_ops(const struct btf_type t)
622	{	597	{
623	return kind_ops[BTF_INFO_KIND(t->info)];	598	return kind_ops[BTF_INFO_KIND(t->info)];

kernel/bpf/hashtab.c+3−3

@@ -612,7 +612,7 @@ static void htab_map_lookup_elem(struct bpf_map map, void *key) · 611 unmodified lines
612	* bpf_prog	612	* bpf_prog
613	* __htab_map_lookup_elem	613	* __htab_map_lookup_elem
614	*/	614	*/
615	static u32 htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)	615	static int htab_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
616	{	616	{
617	struct bpf_insn *insn = insn_buf;	617	struct bpf_insn *insn = insn_buf;
618	const int ret = BPF_REG_0;	618	const int ret = BPF_REG_0;
@@ -651,7 +651,7 @@ static void htab_lru_map_lookup_elem_sys(struct bpf_map map, void *key) · 32 unmodified lines
651	return __htab_lru_map_lookup_elem(map, key, false);	651	return __htab_lru_map_lookup_elem(map, key, false);
652	}	652	}
653		653
654	static u32 htab_lru_map_gen_lookup(struct bpf_map *map,	654	static int htab_lru_map_gen_lookup(struct bpf_map *map,
655	struct bpf_insn *insn_buf)	655	struct bpf_insn *insn_buf)
656	{	656	{
657	struct bpf_insn *insn = insn_buf;	657	struct bpf_insn *insn = insn_buf;
@@ -2070,7 +2070,7 @@ static void htab_of_map_lookup_elem(struct bpf_map map, void *key) · 1412 unmodified lines
2070	return READ_ONCE(*inner_map);	2070	return READ_ONCE(*inner_map);
2071	}	2071	}
2072		2072
2073	static u32 htab_of_map_gen_lookup(struct bpf_map *map,	2073	static int htab_of_map_gen_lookup(struct bpf_map *map,
2074	struct bpf_insn *insn_buf)	2074	struct bpf_insn *insn_buf)
2075	{	2075	{
2076	struct bpf_insn *insn = insn_buf;	2076	struct bpf_insn *insn = insn_buf;

kernel/bpf/helpers.c+32−0

@@ -623,6 +623,34 @@ const struct bpf_func_proto bpf_copy_from_user_proto = { · 622 unmodified lines
623	.arg3_type = ARG_ANYTHING,	623	.arg3_type = ARG_ANYTHING,
624	};	624	};
625		625
		626	BPF_CALL_2(bpf_per_cpu_ptr, const void *, ptr, u32, cpu)
		627	{
		628	if (cpu >= nr_cpu_ids)
		629	return (unsigned long)NULL;
		630
		631	return (unsigned long)per_cpu_ptr((const void __percpu *)ptr, cpu);
		632	}
		633
		634	const struct bpf_func_proto bpf_per_cpu_ptr_proto = {
		635	.func = bpf_per_cpu_ptr,
		636	.gpl_only = false,
		637	.ret_type = RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL,
		638	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
		639	.arg2_type = ARG_ANYTHING,
		640	};
		641
		642	BPF_CALL_1(bpf_this_cpu_ptr, const void *, percpu_ptr)
		643	{
		644	return (unsigned long)this_cpu_ptr((const void __percpu *)percpu_ptr);
		645	}
		646
		647	const struct bpf_func_proto bpf_this_cpu_ptr_proto = {
		648	.func = bpf_this_cpu_ptr,
		649	.gpl_only = false,
		650	.ret_type = RET_PTR_TO_MEM_OR_BTF_ID,
		651	.arg1_type = ARG_PTR_TO_PERCPU_BTF_ID,
		652	};
		653
626	const struct bpf_func_proto bpf_get_current_task_proto __weak;	654	const struct bpf_func_proto bpf_get_current_task_proto __weak;
627	const struct bpf_func_proto bpf_probe_read_user_proto __weak;	655	const struct bpf_func_proto bpf_probe_read_user_proto __weak;
628	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;	656	const struct bpf_func_proto bpf_probe_read_user_str_proto __weak;
@@ -689,6 +717,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 60 unmodified lines
689	return &bpf_snprintf_btf_proto;	717	return &bpf_snprintf_btf_proto;
690	case BPF_FUNC_jiffies64:	718	case BPF_FUNC_jiffies64:
691	return &bpf_jiffies64_proto;	719	return &bpf_jiffies64_proto;
		720	case BPF_FUNC_bpf_per_cpu_ptr:
		721	return &bpf_per_cpu_ptr_proto;
		722	case BPF_FUNC_bpf_this_cpu_ptr:
		723	return &bpf_this_cpu_ptr_proto;
692	default:	724	default:
693	break;	725	break;
694	}	726	}

kernel/bpf/percpu_freelist.c+96−5

@@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) · 16 unmodified lines
17	raw_spin_lock_init(&head->lock);	17	raw_spin_lock_init(&head->lock);
18	head->first = NULL;	18	head->first = NULL;
19	}	19	}
		20	raw_spin_lock_init(&s->extralist.lock);
		21	s->extralist.first = NULL;
20	return 0;	22	return 0;
21	}	23	}
22		24
@@ -40,12 +42,50 @@ static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, · 17 unmodified lines
40	raw_spin_unlock(&head->lock);	42	raw_spin_unlock(&head->lock);
41	}	43	}
42		44
		45	static inline bool pcpu_freelist_try_push_extra(struct pcpu_freelist *s,
		46	struct pcpu_freelist_node *node)
		47	{
		48	if (!raw_spin_trylock(&s->extralist.lock))
		49	return false;
		50
		51	pcpu_freelist_push_node(&s->extralist, node);
		52	raw_spin_unlock(&s->extralist.lock);
		53	return true;
		54	}
		55
		56	static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s,
		57	struct pcpu_freelist_node *node)
		58	{
		59	int cpu, orig_cpu;
		60
		61	orig_cpu = cpu = raw_smp_processor_id();
		62	while (1) {
		63	struct pcpu_freelist_head *head;
		64
		65	head = per_cpu_ptr(s->freelist, cpu);
		66	if (raw_spin_trylock(&head->lock)) {
		67	pcpu_freelist_push_node(head, node);
		68	raw_spin_unlock(&head->lock);
		69	return;
		70	}
		71	cpu = cpumask_next(cpu, cpu_possible_mask);
		72	if (cpu >= nr_cpu_ids)
		73	cpu = 0;
		74
		75	/* cannot lock any per cpu lock, try extralist */
		76	if (cpu == orig_cpu &&
		77	pcpu_freelist_try_push_extra(s, node))
		78	return;
		79	}
		80	}
		81
43	void __pcpu_freelist_push(struct pcpu_freelist *s,	82	void __pcpu_freelist_push(struct pcpu_freelist *s,
44	struct pcpu_freelist_node *node)	83	struct pcpu_freelist_node *node)
45	{	84	{
46	struct pcpu_freelist_head *head = this_cpu_ptr(s->freelist);	85	if (in_nmi())
47		86	___pcpu_freelist_push_nmi(s, node);
48	___pcpu_freelist_push(head, node);	87	else
		88	___pcpu_freelist_push(this_cpu_ptr(s->freelist), node);
49	}	89	}
50		90
51	void pcpu_freelist_push(struct pcpu_freelist *s,	91	void pcpu_freelist_push(struct pcpu_freelist *s,
@@ -81,7 +121,7 @@ again: · 29 unmodified lines
81	}	121	}
82	}	122	}
83		123
84	struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)	124	static struct pcpu_freelist_node ___pcpu_freelist_pop(struct pcpu_freelist s)
85	{	125	{
86	struct pcpu_freelist_head *head;	126	struct pcpu_freelist_head *head;
87	struct pcpu_freelist_node *node;	127	struct pcpu_freelist_node *node;
@@ -102,8 +142,59 @@ struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s) · 14 unmodified lines
102	if (cpu >= nr_cpu_ids)	142	if (cpu >= nr_cpu_ids)
103	cpu = 0;	143	cpu = 0;
104	if (cpu == orig_cpu)	144	if (cpu == orig_cpu)
105	return NULL;	145	break;
		146	}
		147
		148	/* per cpu lists are all empty, try extralist */
		149	raw_spin_lock(&s->extralist.lock);
		150	node = s->extralist.first;
		151	if (node)
		152	s->extralist.first = node->next;
		153	raw_spin_unlock(&s->extralist.lock);
		154	return node;
		155	}
		156
		157	static struct pcpu_freelist_node *
		158	___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
		159	{
		160	struct pcpu_freelist_head *head;
		161	struct pcpu_freelist_node *node;
		162	int orig_cpu, cpu;
		163
		164	orig_cpu = cpu = raw_smp_processor_id();
		165	while (1) {
		166	head = per_cpu_ptr(s->freelist, cpu);
		167	if (raw_spin_trylock(&head->lock)) {
		168	node = head->first;
		169	if (node) {
		170	head->first = node->next;
		171	raw_spin_unlock(&head->lock);
		172	return node;
		173	}
		174	raw_spin_unlock(&head->lock);
		175	}
		176	cpu = cpumask_next(cpu, cpu_possible_mask);
		177	if (cpu >= nr_cpu_ids)
		178	cpu = 0;
		179	if (cpu == orig_cpu)
		180	break;
106	}	181	}
		182
		183	/* cannot pop from per cpu lists, try extralist */
		184	if (!raw_spin_trylock(&s->extralist.lock))
		185	return NULL;
		186	node = s->extralist.first;
		187	if (node)
		188	s->extralist.first = node->next;
		189	raw_spin_unlock(&s->extralist.lock);
		190	return node;
		191	}
		192
		193	struct pcpu_freelist_node __pcpu_freelist_pop(struct pcpu_freelist s)
		194	{
		195	if (in_nmi())
		196	return ___pcpu_freelist_pop_nmi(s);
		197	return ___pcpu_freelist_pop(s);
107	}	198	}
108		199
109	struct pcpu_freelist_node pcpu_freelist_pop(struct pcpu_freelist s)	200	struct pcpu_freelist_node pcpu_freelist_pop(struct pcpu_freelist s)

kernel/bpf/percpu_freelist.h+1−0

@@ -13,6 +13,7 @@ struct pcpu_freelist_head { · 12 unmodified lines
13		13
14	struct pcpu_freelist {	14	struct pcpu_freelist {
15	struct pcpu_freelist_head __percpu *freelist;	15	struct pcpu_freelist_head __percpu *freelist;
		16	struct pcpu_freelist_head extralist;
16	};	17	};
17		18
18	struct pcpu_freelist_node {	19	struct pcpu_freelist_node {

kernel/bpf/syscall.c+3−1

 	used_maps_old = prog->aux->used_maps;
 	for (i = 0; i < prog->aux->used_map_cnt; i++)
-		if (used_maps_old[i] == map)
+		if (used_maps_old[i] == map) {
+			bpf_map_put(map);
 			goto out_unlock;
+		}
 	used_maps_new = kmalloc_array(prog->aux->used_map_cnt + 1,
 				      sizeof(used_maps_new[0]),

kernel/bpf/verifier.c+251−19

@@ -238,6 +238,8 @@ struct bpf_call_arg_meta { · 237 unmodified lines
238	u64 msize_max_value;	238	u64 msize_max_value;
239	int ref_obj_id;	239	int ref_obj_id;
240	int func_id;	240	int func_id;
		241	u32 btf_id;
		242	u32 ret_btf_id;
241	};	243	};
242		244
243	struct btf *btf_vmlinux;	245	struct btf *btf_vmlinux;
@@ -517,6 +519,7 @@ static const char * const reg_type_str[] = { · 273 unmodified lines
517	[PTR_TO_XDP_SOCK] = "xdp_sock",	519	[PTR_TO_XDP_SOCK] = "xdp_sock",
518	[PTR_TO_BTF_ID] = "ptr_",	520	[PTR_TO_BTF_ID] = "ptr_",
519	[PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",	521	[PTR_TO_BTF_ID_OR_NULL] = "ptr_or_null_",
		522	[PTR_TO_PERCPU_BTF_ID] = "percpu_ptr_",
520	[PTR_TO_MEM] = "mem",	523	[PTR_TO_MEM] = "mem",
521	[PTR_TO_MEM_OR_NULL] = "mem_or_null",	524	[PTR_TO_MEM_OR_NULL] = "mem_or_null",
522	[PTR_TO_RDONLY_BUF] = "rdonly_buf",	525	[PTR_TO_RDONLY_BUF] = "rdonly_buf",
@@ -583,7 +586,9 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 60 unmodified lines
583	/* reg->off should be 0 for SCALAR_VALUE */	586	/* reg->off should be 0 for SCALAR_VALUE */
584	verbose(env, "%lld", reg->var_off.value + reg->off);	587	verbose(env, "%lld", reg->var_off.value + reg->off);
585	} else {	588	} else {
586	if (t == PTR_TO_BTF_ID \|\| t == PTR_TO_BTF_ID_OR_NULL)	589	if (t == PTR_TO_BTF_ID \|\|
		590	t == PTR_TO_BTF_ID_OR_NULL \|\|
		591	t == PTR_TO_PERCPU_BTF_ID)
587	verbose(env, "%s", kernel_type_name(reg->btf_id));	592	verbose(env, "%s", kernel_type_name(reg->btf_id));
588	verbose(env, "(id=%d", reg->id);	593	verbose(env, "(id=%d", reg->id);
589	if (reg_type_may_be_refcounted_or_null(t))	594	if (reg_type_may_be_refcounted_or_null(t))
@@ -2204,6 +2209,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 1614 unmodified lines
2204	case PTR_TO_RDONLY_BUF_OR_NULL:	2209	case PTR_TO_RDONLY_BUF_OR_NULL:
2205	case PTR_TO_RDWR_BUF:	2210	case PTR_TO_RDWR_BUF:
2206	case PTR_TO_RDWR_BUF_OR_NULL:	2211	case PTR_TO_RDWR_BUF_OR_NULL:
		2212	case PTR_TO_PERCPU_BTF_ID:
2207	return true;	2213	return true;
2208	default:	2214	default:
2209	return false;	2215	return false;
@@ -2221,6 +2227,20 @@ static bool register_is_const(struct bpf_reg_state *reg) · 11 unmodified lines
2221	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);	2227	return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
2222	}	2228	}
2223		2229
		2230	static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
		2231	{
		2232	return tnum_is_unknown(reg->var_off) &&
		2233	reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
		2234	reg->umin_value == 0 && reg->umax_value == U64_MAX &&
		2235	reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
		2236	reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
		2237	}
		2238
		2239	static bool register_is_bounded(struct bpf_reg_state *reg)
		2240	{
		2241	return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
		2242	}
		2243
2224	static bool __is_pointer_value(bool allow_ptr_leaks,	2244	static bool __is_pointer_value(bool allow_ptr_leaks,
2225	const struct bpf_reg_state *reg)	2245	const struct bpf_reg_state *reg)
2226	{	2246	{
@@ -2272,7 +2292,7 @@ static int check_stack_write(struct bpf_verifier_env *env, · 45 unmodified lines
2272	if (value_regno >= 0)	2292	if (value_regno >= 0)
2273	reg = &cur->regs[value_regno];	2293	reg = &cur->regs[value_regno];
2274		2294
2275	if (reg && size == BPF_REG_SIZE && register_is_const(reg) &&	2295	if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) &&
2276	!register_is_null(reg) && env->bpf_capable) {	2296	!register_is_null(reg) && env->bpf_capable) {
2277	if (dst_reg != BPF_REG_FP) {	2297	if (dst_reg != BPF_REG_FP) {
2278	/* The backtracking logic can only recognize explicit	2298	/* The backtracking logic can only recognize explicit
@@ -2667,7 +2687,7 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, · 388 unmodified lines
2667	case BPF_PROG_TYPE_CGROUP_SKB:	2687	case BPF_PROG_TYPE_CGROUP_SKB:
2668	if (t == BPF_WRITE)	2688	if (t == BPF_WRITE)
2669	return false;	2689	return false;
2670	/* fallthrough */	2690	fallthrough;
2671		2691
2672	/* Program types with direct read + write access go here! */	2692	/* Program types with direct read + write access go here! */
2673	case BPF_PROG_TYPE_SCHED_CLS:	2693	case BPF_PROG_TYPE_SCHED_CLS:
@@ -3978,6 +3998,7 @@ static const struct bpf_reg_types sock_types = { · 1304 unmodified lines
3978	},	3998	},
3979	};	3999	};
3980		4000
		4001	#ifdef CONFIG_NET
3981	static const struct bpf_reg_types btf_id_sock_common_types = {	4002	static const struct bpf_reg_types btf_id_sock_common_types = {
3982	.types = {	4003	.types = {
3983	PTR_TO_SOCK_COMMON,	4004	PTR_TO_SOCK_COMMON,
@@ -3988,6 +4009,7 @@ static const struct bpf_reg_types btf_id_sock_common_types = { · 4 unmodified lines
3988	},	4009	},
3989	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],	4010	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
3990	};	4011	};
		4012	#endif
3991		4013
3992	static const struct bpf_reg_types mem_types = {	4014	static const struct bpf_reg_types mem_types = {
3993	.types = {	4015	.types = {
@@ -4017,6 +4039,7 @@ static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM } }; · 23 unmodified lines
4017	static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };	4039	static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
4018	static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };	4040	static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
4019	static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };	4041	static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
		4042	static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_PERCPU_BTF_ID } };
4020		4043
4021	static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {	4044	static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
4022	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,	4045	[ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
@@ -4030,7 +4053,9 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 7 unmodified lines
4030	[ARG_PTR_TO_CTX] = &context_types,	4053	[ARG_PTR_TO_CTX] = &context_types,
4031	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,	4054	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,
4032	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,	4055	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
		4056	#ifdef CONFIG_NET
4033	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,	4057	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
		4058	#endif
4034	[ARG_PTR_TO_SOCKET] = &fullsock_types,	4059	[ARG_PTR_TO_SOCKET] = &fullsock_types,
4035	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,	4060	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
4036	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,	4061	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
@@ -4042,6 +4067,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 5 unmodified lines
4042	[ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,	4067	[ARG_PTR_TO_ALLOC_MEM_OR_NULL] = &alloc_mem_types,
4043	[ARG_PTR_TO_INT] = &int_ptr_types,	4068	[ARG_PTR_TO_INT] = &int_ptr_types,
4044	[ARG_PTR_TO_LONG] = &int_ptr_types,	4069	[ARG_PTR_TO_LONG] = &int_ptr_types,
		4070	[ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
4045	};	4071	};
4046		4072
4047	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,	4073	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
@@ -4205,6 +4231,12 @@ skip_type_check: · 157 unmodified lines
4205	err = check_helper_mem_access(env, regno,	4231	err = check_helper_mem_access(env, regno,
4206	meta->map_ptr->value_size, false,	4232	meta->map_ptr->value_size, false,
4207	meta);	4233	meta);
		4234	} else if (arg_type == ARG_PTR_TO_PERCPU_BTF_ID) {
		4235	if (!reg->btf_id) {
		4236	verbose(env, "Helper has invalid btf_id in R%d\n", regno);
		4237	return -EACCES;
		4238	}
		4239	meta->ret_btf_id = reg->btf_id;
4208	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {	4240	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
4209	if (meta->func_id == BPF_FUNC_spin_lock) {	4241	if (meta->func_id == BPF_FUNC_spin_lock) {
4210	if (process_spin_lock(env, regno, true))	4242	if (process_spin_lock(env, regno, true))
@@ -5114,6 +5146,35 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn · 903 unmodified lines
5114	regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;	5146	regs[BPF_REG_0].type = PTR_TO_MEM_OR_NULL;
5115	regs[BPF_REG_0].id = ++env->id_gen;	5147	regs[BPF_REG_0].id = ++env->id_gen;
5116	regs[BPF_REG_0].mem_size = meta.mem_size;	5148	regs[BPF_REG_0].mem_size = meta.mem_size;
		5149	} else if (fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL \|\|
		5150	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID) {
		5151	const struct btf_type *t;
		5152
		5153	mark_reg_known_zero(env, regs, BPF_REG_0);
		5154	t = btf_type_skip_modifiers(btf_vmlinux, meta.ret_btf_id, NULL);
		5155	if (!btf_type_is_struct(t)) {
		5156	u32 tsize;
		5157	const struct btf_type *ret;
		5158	const char *tname;
		5159
		5160	/* resolve the type size of ksym. */
		5161	ret = btf_resolve_size(btf_vmlinux, t, &tsize);
		5162	if (IS_ERR(ret)) {
		5163	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
		5164	verbose(env, "unable to resolve the size of type '%s': %ld\n",
		5165	tname, PTR_ERR(ret));
		5166	return -EINVAL;
		5167	}
		5168	regs[BPF_REG_0].type =
		5169	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
		5170	PTR_TO_MEM : PTR_TO_MEM_OR_NULL;
		5171	regs[BPF_REG_0].mem_size = tsize;
		5172	} else {
		5173	regs[BPF_REG_0].type =
		5174	fn->ret_type == RET_PTR_TO_MEM_OR_BTF_ID ?
		5175	PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
		5176	regs[BPF_REG_0].btf_id = meta.ret_btf_id;
		5177	}
5117	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {	5178	} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {
5118	int ret_btf_id;	5179	int ret_btf_id;
5119		5180
@@ -5432,7 +5493,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 312 unmodified lines
5432	/* smin_val represents the known value */	5493	/* smin_val represents the known value */
5433	if (known && smin_val == 0 && opcode == BPF_ADD)	5494	if (known && smin_val == 0 && opcode == BPF_ADD)
5434	break;	5495	break;
5435	/* fall-through */	5496	fallthrough;
5436	case PTR_TO_PACKET_END:	5497	case PTR_TO_PACKET_END:
5437	case PTR_TO_SOCKET:	5498	case PTR_TO_SOCKET:
5438	case PTR_TO_SOCKET_OR_NULL:	5499	case PTR_TO_SOCKET_OR_NULL:
@@ -6389,6 +6450,11 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 950 unmodified lines
6389	src_reg = NULL;	6450	src_reg = NULL;
6390	if (dst_reg->type != SCALAR_VALUE)	6451	if (dst_reg->type != SCALAR_VALUE)
6391	ptr_reg = dst_reg;	6452	ptr_reg = dst_reg;
		6453	else
		6454	/* Make sure ID is cleared otherwise dst_reg min/max could be
		6455	* incorrectly propagated into other registers by find_equal_scalars()
		6456	*/
		6457	dst_reg->id = 0;
6392	if (BPF_SRC(insn->code) == BPF_X) {	6458	if (BPF_SRC(insn->code) == BPF_X) {
6393	src_reg = &regs[insn->src_reg];	6459	src_reg = &regs[insn->src_reg];
6394	if (src_reg->type != SCALAR_VALUE) {	6460	if (src_reg->type != SCALAR_VALUE) {
@@ -6522,6 +6588,12 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 127 unmodified lines
6522	/* case: R1 = R2	6588	/* case: R1 = R2
6523	* copy register state to dest reg	6589	* copy register state to dest reg
6524	*/	6590	*/
		6591	if (src_reg->type == SCALAR_VALUE && !src_reg->id)
		6592	/* Assign src and dst registers the same ID
		6593	* that will be used by find_equal_scalars()
		6594	* to propagate min/max range.
		6595	*/
		6596	src_reg->id = ++env->id_gen;
6525	dst_reg = src_reg;	6597	dst_reg = src_reg;
6526	dst_reg->live \|= REG_LIVE_WRITTEN;	6598	dst_reg->live \|= REG_LIVE_WRITTEN;
6527	dst_reg->subreg_def = DEF_NOT_SUBREG;	6599	dst_reg->subreg_def = DEF_NOT_SUBREG;
@@ -6534,6 +6606,11 @@ static int check_alu_op(struct bpf_verifier_env env, struct bpf_insn insn) · 6 unmodified lines
6534	return -EACCES;	6606	return -EACCES;
6535	} else if (src_reg->type == SCALAR_VALUE) {	6607	} else if (src_reg->type == SCALAR_VALUE) {
6536	dst_reg = src_reg;	6608	dst_reg = src_reg;
		6609	/* Make sure ID is cleared otherwise
		6610	* dst_reg min/max could be incorrectly
		6611	* propagated into src_reg by find_equal_scalars()
		6612	*/
		6613	dst_reg->id = 0;
6537	dst_reg->live \|= REG_LIVE_WRITTEN;	6614	dst_reg->live \|= REG_LIVE_WRITTEN;
6538	dst_reg->subreg_def = env->insn_idx + 1;	6615	dst_reg->subreg_def = env->insn_idx + 1;
6539	} else {	6616	} else {
@@ -7322,6 +7399,30 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn, · 782 unmodified lines
7322	return true;	7399	return true;
7323	}	7400	}
7324		7401
		7402	static void find_equal_scalars(struct bpf_verifier_state *vstate,
		7403	struct bpf_reg_state *known_reg)
		7404	{
		7405	struct bpf_func_state *state;
		7406	struct bpf_reg_state *reg;
		7407	int i, j;
		7408
		7409	for (i = 0; i <= vstate->curframe; i++) {
		7410	state = vstate->frame[i];
		7411	for (j = 0; j < MAX_BPF_REG; j++) {
		7412	reg = &state->regs[j];
		7413	if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
		7414	reg = known_reg;
		7415	}
		7416
		7417	bpf_for_each_spilled_reg(j, state, reg) {
		7418	if (!reg)
		7419	continue;
		7420	if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
		7421	reg = known_reg;
		7422	}
		7423	}
		7424	}
		7425
7325	static int check_cond_jmp_op(struct bpf_verifier_env *env,	7426	static int check_cond_jmp_op(struct bpf_verifier_env *env,
7326	struct bpf_insn insn, int insn_idx)	7427	struct bpf_insn insn, int insn_idx)
7327	{	7428	{
@@ -7450,6 +7551,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 122 unmodified lines
7450	reg_combine_min_max(&other_branch_regs[insn->src_reg],	7551	reg_combine_min_max(&other_branch_regs[insn->src_reg],
7451	&other_branch_regs[insn->dst_reg],	7552	&other_branch_regs[insn->dst_reg],
7452	src_reg, dst_reg, opcode);	7553	src_reg, dst_reg, opcode);
		7554	if (src_reg->id) {
		7555	find_equal_scalars(this_branch, src_reg);
		7556	find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
		7557	}
		7558
7453	}	7559	}
7454	} else if (dst_reg->type == SCALAR_VALUE) {	7560	} else if (dst_reg->type == SCALAR_VALUE) {
7455	reg_set_min_max(&other_branch_regs[insn->dst_reg],	7561	reg_set_min_max(&other_branch_regs[insn->dst_reg],
@@ -7457,6 +7563,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 1 unmodified line
7457	opcode, is_jmp32);	7563	opcode, is_jmp32);
7458	}	7564	}
7459		7565
		7566	if (dst_reg->type == SCALAR_VALUE && dst_reg->id) {
		7567	find_equal_scalars(this_branch, dst_reg);
		7568	find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
		7569	}
		7570
7460	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().	7571	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
7461	* NOTE: these optimizations below are related with pointer comparison	7572	* NOTE: these optimizations below are related with pointer comparison
7462	* which will never be JMP32.	7573	* which will never be JMP32.
@@ -7488,6 +7599,7 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 25 unmodified lines
7488	{	7599	{
7489	struct bpf_insn_aux_data *aux = cur_aux(env);	7600	struct bpf_insn_aux_data *aux = cur_aux(env);
7490	struct bpf_reg_state *regs = cur_regs(env);	7601	struct bpf_reg_state *regs = cur_regs(env);
		7602	struct bpf_reg_state *dst_reg;
7491	struct bpf_map *map;	7603	struct bpf_map *map;
7492	int err;	7604	int err;
7493		7605
@@ -7504,25 +7616,45 @@ static int check_ld_imm(struct bpf_verifier_env env, struct bpf_insn insn) · 10 unmodified lines
7504	if (err)	7616	if (err)
7505	return err;	7617	return err;
7506		7618
		7619	dst_reg = &regs[insn->dst_reg];
7507	if (insn->src_reg == 0) {	7620	if (insn->src_reg == 0) {
7508	u64 imm = ((u64)(insn + 1)->imm << 32) \| (u32)insn->imm;	7621	u64 imm = ((u64)(insn + 1)->imm << 32) \| (u32)insn->imm;
7509		7622
7510	regs[insn->dst_reg].type = SCALAR_VALUE;	7623	dst_reg->type = SCALAR_VALUE;
7511	__mark_reg_known(&regs[insn->dst_reg], imm);	7624	__mark_reg_known(&regs[insn->dst_reg], imm);
7512	return 0;	7625	return 0;
7513	}	7626	}
7514		7627
		7628	if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
		7629	mark_reg_known_zero(env, regs, insn->dst_reg);
		7630
		7631	dst_reg->type = aux->btf_var.reg_type;
		7632	switch (dst_reg->type) {
		7633	case PTR_TO_MEM:
		7634	dst_reg->mem_size = aux->btf_var.mem_size;
		7635	break;
		7636	case PTR_TO_BTF_ID:
		7637	case PTR_TO_PERCPU_BTF_ID:
		7638	dst_reg->btf_id = aux->btf_var.btf_id;
		7639	break;
		7640	default:
		7641	verbose(env, "bpf verifier is misconfigured\n");
		7642	return -EFAULT;
		7643	}
		7644	return 0;
		7645	}
		7646
7515	map = env->used_maps[aux->map_index];	7647	map = env->used_maps[aux->map_index];
7516	mark_reg_known_zero(env, regs, insn->dst_reg);	7648	mark_reg_known_zero(env, regs, insn->dst_reg);
7517	regs[insn->dst_reg].map_ptr = map;	7649	dst_reg->map_ptr = map;
7518		7650
7519	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {	7651	if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) {
7520	regs[insn->dst_reg].type = PTR_TO_MAP_VALUE;	7652	dst_reg->type = PTR_TO_MAP_VALUE;
7521	regs[insn->dst_reg].off = aux->map_off;	7653	dst_reg->off = aux->map_off;
7522	if (map_value_has_spin_lock(map))	7654	if (map_value_has_spin_lock(map))
7523	regs[insn->dst_reg].id = ++env->id_gen;	7655	dst_reg->id = ++env->id_gen;
7524	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {	7656	} else if (insn->src_reg == BPF_PSEUDO_MAP_FD) {
7525	regs[insn->dst_reg].type = CONST_PTR_TO_MAP;	7657	dst_reg->type = CONST_PTR_TO_MAP;
7526	} else {	7658	} else {
7527	verbose(env, "bpf verifier is misconfigured\n");	7659	verbose(env, "bpf verifier is misconfigured\n");
7528	return -EINVAL;	7660	return -EINVAL;
@@ -9424,6 +9556,92 @@ process_bpf_exit: · 1895 unmodified lines
9424	return 0;	9556	return 0;
9425	}	9557	}
9426		9558
		9559	/* replace pseudo btf_id with kernel symbol address */
		9560	static int check_pseudo_btf_id(struct bpf_verifier_env *env,
		9561	struct bpf_insn *insn,
		9562	struct bpf_insn_aux_data *aux)
		9563	{
		9564	u32 datasec_id, type, id = insn->imm;
		9565	const struct btf_var_secinfo *vsi;
		9566	const struct btf_type *datasec;
		9567	const struct btf_type *t;
		9568	const char *sym_name;
		9569	bool percpu = false;
		9570	u64 addr;
		9571	int i;
		9572
		9573	if (!btf_vmlinux) {
		9574	verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
		9575	return -EINVAL;
		9576	}
		9577
		9578	if (insn[1].imm != 0) {
		9579	verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n");
		9580	return -EINVAL;
		9581	}
		9582
		9583	t = btf_type_by_id(btf_vmlinux, id);
		9584	if (!t) {
		9585	verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
		9586	return -ENOENT;
		9587	}
		9588
		9589	if (!btf_type_is_var(t)) {
		9590	verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n",
		9591	id);
		9592	return -EINVAL;
		9593	}
		9594
		9595	sym_name = btf_name_by_offset(btf_vmlinux, t->name_off);
		9596	addr = kallsyms_lookup_name(sym_name);
		9597	if (!addr) {
		9598	verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
		9599	sym_name);
		9600	return -ENOENT;
		9601	}
		9602
		9603	datasec_id = btf_find_by_name_kind(btf_vmlinux, ".data..percpu",
		9604	BTF_KIND_DATASEC);
		9605	if (datasec_id > 0) {
		9606	datasec = btf_type_by_id(btf_vmlinux, datasec_id);
		9607	for_each_vsi(i, datasec, vsi) {
		9608	if (vsi->type == id) {
		9609	percpu = true;
		9610	break;
		9611	}
		9612	}
		9613	}
		9614
		9615	insn[0].imm = (u32)addr;
		9616	insn[1].imm = addr >> 32;
		9617
		9618	type = t->type;
		9619	t = btf_type_skip_modifiers(btf_vmlinux, type, NULL);
		9620	if (percpu) {
		9621	aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
		9622	aux->btf_var.btf_id = type;
		9623	} else if (!btf_type_is_struct(t)) {
		9624	const struct btf_type *ret;
		9625	const char *tname;
		9626	u32 tsize;
		9627
		9628	/* resolve the type size of ksym. */
		9629	ret = btf_resolve_size(btf_vmlinux, t, &tsize);
		9630	if (IS_ERR(ret)) {
		9631	tname = btf_name_by_offset(btf_vmlinux, t->name_off);
		9632	verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
		9633	tname, PTR_ERR(ret));
		9634	return -EINVAL;
		9635	}
		9636	aux->btf_var.reg_type = PTR_TO_MEM;
		9637	aux->btf_var.mem_size = tsize;
		9638	} else {
		9639	aux->btf_var.reg_type = PTR_TO_BTF_ID;
		9640	aux->btf_var.btf_id = type;
		9641	}
		9642	return 0;
		9643	}
		9644
9427	static int check_map_prealloc(struct bpf_map *map)	9645	static int check_map_prealloc(struct bpf_map *map)
9428	{	9646	{
9429	return (map->map_type != BPF_MAP_TYPE_HASH &&	9647	return (map->map_type != BPF_MAP_TYPE_HASH &&
@@ -9534,10 +9752,14 @@ static bool bpf_map_is_cgroup_storage(struct bpf_map *map) · 104 unmodified lines
9534	map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);	9752	map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
9535	}	9753	}
9536		9754
9537	/* look for pseudo eBPF instructions that access map FDs and	9755	/* find and rewrite pseudo imm in ld_imm64 instructions:
9538	* replace them with actual map pointers	9756	*
		9757	* 1. if it accesses map FD, replace it with actual map pointer.
		9758	* 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
		9759	*
		9760	* NOTE: btf_vmlinux is required for converting pseudo btf_id.
9539	*/	9761	*/
9540	static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)	9762	static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
9541	{	9763	{
9542	struct bpf_insn *insn = env->prog->insnsi;	9764	struct bpf_insn *insn = env->prog->insnsi;
9543	int insn_cnt = env->prog->len;	9765	int insn_cnt = env->prog->len;
@@ -9578,6 +9800,14 @@ static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) · 34 unmodified lines
9578	/* valid generic load 64-bit imm */	9800	/* valid generic load 64-bit imm */
9579	goto next_insn;	9801	goto next_insn;
9580		9802
		9803	if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
		9804	aux = &env->insn_aux_data[i];
		9805	err = check_pseudo_btf_id(env, insn, aux);
		9806	if (err)
		9807	return err;
		9808	goto next_insn;
		9809	}
		9810
9581	/* In final convert_pseudo_ld_imm64() step, this is	9811	/* In final convert_pseudo_ld_imm64() step, this is
9582	* converted into regular 64-bit imm load insn.	9812	* converted into regular 64-bit imm load insn.
9583	*/	9813	*/
@@ -10819,7 +11049,9 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 1235 unmodified lines
10819	if (insn->imm == BPF_FUNC_map_lookup_elem &&	11049	if (insn->imm == BPF_FUNC_map_lookup_elem &&
10820	ops->map_gen_lookup) {	11050	ops->map_gen_lookup) {
10821	cnt = ops->map_gen_lookup(map_ptr, insn_buf);	11051	cnt = ops->map_gen_lookup(map_ptr, insn_buf);
10822	if (cnt == 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {	11052	if (cnt == -EOPNOTSUPP)
		11053	goto patch_map_ops_generic;
		11054	if (cnt <= 0 \|\| cnt >= ARRAY_SIZE(insn_buf)) {
10823	verbose(env, "bpf verifier is misconfigured\n");	11055	verbose(env, "bpf verifier is misconfigured\n");
10824	return -EINVAL;	11056	return -EINVAL;
10825	}	11057	}
@@ -10849,7 +11081,7 @@ static int fixup_bpf_calls(struct bpf_verifier_env *env) · 23 unmodified lines
10849	(int ()(struct bpf_map map, void *value))NULL));	11081	(int ()(struct bpf_map map, void *value))NULL));
10850	BUILD_BUG_ON(!__same_type(ops->map_peek_elem,	11082	BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
10851	(int ()(struct bpf_map map, void *value))NULL));	11083	(int ()(struct bpf_map map, void *value))NULL));
10852		11084	patch_map_ops_generic:
10853	switch (insn->imm) {	11085	switch (insn->imm) {
10854	case BPF_FUNC_map_lookup_elem:	11086	case BPF_FUNC_map_lookup_elem:
10855	insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -	11087	insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
@@ -11633,10 +11865,6 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 777 unmodified lines
11633	if (is_priv)	11865	if (is_priv)
11634	env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;	11866	env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
11635		11867
11636	ret = replace_map_fd_with_map_ptr(env);
11637	if (ret < 0)
11638	goto skip_full_check;
11639
11640	if (bpf_prog_is_dev_bound(env->prog->aux)) {	11868	if (bpf_prog_is_dev_bound(env->prog->aux)) {
11641	ret = bpf_prog_offload_verifier_prep(env->prog);	11869	ret = bpf_prog_offload_verifier_prep(env->prog);
11642	if (ret)	11870	if (ret)
@@ -11662,6 +11890,10 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 19 unmodified lines
11662	if (ret)	11890	if (ret)
11663	goto skip_full_check;	11891	goto skip_full_check;
11664		11892
		11893	ret = resolve_pseudo_ldimm64(env);
		11894	if (ret < 0)
		11895	goto skip_full_check;
		11896
11665	ret = check_cfg(env);	11897	ret = check_cfg(env);
11666	if (ret < 0)	11898	if (ret < 0)
11667	goto skip_full_check;	11899	goto skip_full_check;

kernel/trace/bpf_trace.c+6−0

 		return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
 	case BPF_FUNC_snprintf_btf:
 		return &bpf_snprintf_btf_proto;
+	case BPF_FUNC_bpf_per_cpu_ptr:
+		return &bpf_per_cpu_ptr_proto;
+	case BPF_FUNC_bpf_this_cpu_ptr:
+		return &bpf_this_cpu_ptr_proto;
 	default:
 		return NULL;
 	}
 };
 const struct bpf_prog_ops raw_tracepoint_prog_ops = {
+#ifdef CONFIG_NET
 	.test_run = bpf_prog_test_run_raw_tp,
+#endif
 };
 const struct bpf_verifier_ops tracing_verifier_ops = {

net/core/dev.c+12−3

 static inline struct sk_buff *
 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
-		   struct net_device *orig_dev)
+		   struct net_device *orig_dev, bool *another)
 {
 #ifdef CONFIG_NET_CLS_ACT
 	struct mini_Qdisc *miniq = rcu_dereference_bh(skb->dev->miniq_ingress);
 		 * redirecting to another netdev
 		 */
 		__skb_push(skb, skb->mac_len);
-		skb_do_redirect(skb);
+		if (skb_do_redirect(skb) == -EAGAIN) {
+			__skb_pull(skb, skb->mac_len);
+			*another = true;
+			break;
+		}
 		return NULL;
 	case TC_ACT_CONSUMED:
 		return NULL;
 skip_taps:
 #ifdef CONFIG_NET_INGRESS
 	if (static_branch_unlikely(&ingress_needed_key)) {
-		skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev);
+		bool another = false;
+		skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
+					 &another);
+		if (another)
+			goto another_round;
 		if (!skb)
 			goto out;

net/core/filter.c+99−8

@@ -76,6 +76,7 @@ · 75 unmodified lines
76	#include <net/bpf_sk_storage.h>	76	#include <net/bpf_sk_storage.h>
77	#include <net/transp_v6.h>	77	#include <net/transp_v6.h>
78	#include <linux/btf_ids.h>	78	#include <linux/btf_ids.h>
		79	#include <net/tls.h>
79		80
80	static const struct bpf_func_proto *	81	static const struct bpf_func_proto *
81	bpf_sk_base_func_proto(enum bpf_func_id func_id);	82	bpf_sk_base_func_proto(enum bpf_func_id func_id);
@@ -2379,8 +2380,9 @@ out: · 2297 unmodified lines
2379		2380
2380	/* Internal, non-exposed redirect flags. */	2381	/* Internal, non-exposed redirect flags. */
2381	enum {	2382	enum {
2382	BPF_F_NEIGH = (1ULL << 1),	2383	BPF_F_NEIGH = (1ULL << 1),
2383	#define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH)	2384	BPF_F_PEER = (1ULL << 2),
		2385	#define BPF_F_REDIRECT_INTERNAL (BPF_F_NEIGH \| BPF_F_PEER)
2384	};	2386	};
2385		2387
2386	BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)	2388	BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
@@ -2429,19 +2431,35 @@ EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info); · 42 unmodified lines
2429	int skb_do_redirect(struct sk_buff *skb)	2431	int skb_do_redirect(struct sk_buff *skb)
2430	{	2432	{
2431	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);	2433	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
		2434	struct net *net = dev_net(skb->dev);
2432	struct net_device *dev;	2435	struct net_device *dev;
2433	u32 flags = ri->flags;	2436	u32 flags = ri->flags;
2434		2437
2435	dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->tgt_index);	2438	dev = dev_get_by_index_rcu(net, ri->tgt_index);
2436	ri->tgt_index = 0;	2439	ri->tgt_index = 0;
2437	if (unlikely(!dev)) {	2440	ri->flags = 0;
2438	kfree_skb(skb);	2441	if (unlikely(!dev))
2439	return -EINVAL;	2442	goto out_drop;
		2443	if (flags & BPF_F_PEER) {
		2444	const struct net_device_ops *ops = dev->netdev_ops;
		2445
		2446	if (unlikely(!ops->ndo_get_peer_dev \|\|
		2447	!skb_at_tc_ingress(skb)))
		2448	goto out_drop;
		2449	dev = ops->ndo_get_peer_dev(dev);
		2450	if (unlikely(!dev \|\|
		2451	!is_skb_forwardable(dev, skb) \|\|
		2452	net_eq(net, dev_net(dev))))
		2453	goto out_drop;
		2454	skb->dev = dev;
		2455	return -EAGAIN;
2440	}	2456	}
2441
2442	return flags & BPF_F_NEIGH ?	2457	return flags & BPF_F_NEIGH ?
2443	__bpf_redirect_neigh(skb, dev) :	2458	__bpf_redirect_neigh(skb, dev) :
2444	__bpf_redirect(skb, dev, flags);	2459	__bpf_redirect(skb, dev, flags);
		2460	out_drop:
		2461	kfree_skb(skb);
		2462	return -EINVAL;
2445	}	2463	}
2446		2464
2447	BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)	2465	BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
@@ -2465,6 +2483,27 @@ static const struct bpf_func_proto bpf_redirect_proto = { · 17 unmodified lines
2465	.arg2_type = ARG_ANYTHING,	2483	.arg2_type = ARG_ANYTHING,
2466	};	2484	};
2467		2485
		2486	BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags)
		2487	{
		2488	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
		2489
		2490	if (unlikely(flags))
		2491	return TC_ACT_SHOT;
		2492
		2493	ri->flags = BPF_F_PEER;
		2494	ri->tgt_index = ifindex;
		2495
		2496	return TC_ACT_REDIRECT;
		2497	}
		2498
		2499	static const struct bpf_func_proto bpf_redirect_peer_proto = {
		2500	.func = bpf_redirect_peer,
		2501	.gpl_only = false,
		2502	.ret_type = RET_INTEGER,
		2503	.arg1_type = ARG_ANYTHING,
		2504	.arg2_type = ARG_ANYTHING,
		2505	};
		2506
2468	BPF_CALL_2(bpf_redirect_neigh, u32, ifindex, u64, flags)	2507	BPF_CALL_2(bpf_redirect_neigh, u32, ifindex, u64, flags)
2469	{	2508	{
2470	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);	2509	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
@@ -3479,6 +3518,48 @@ static u32 __bpf_skb_max_len(const struct sk_buff *skb) · 1008 unmodified lines
3479	SKB_MAX_ALLOC;	3518	SKB_MAX_ALLOC;
3480	}	3519	}
3481		3520
		3521	BPF_CALL_4(sk_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
		3522	u32, mode, u64, flags)
		3523	{
		3524	u32 len_diff_abs = abs(len_diff);
		3525	bool shrink = len_diff < 0;
		3526	int ret = 0;
		3527
		3528	if (unlikely(flags \|\| mode))
		3529	return -EINVAL;
		3530	if (unlikely(len_diff_abs > 0xfffU))
		3531	return -EFAULT;
		3532
		3533	if (!shrink) {
		3534	ret = skb_cow(skb, len_diff);
		3535	if (unlikely(ret < 0))
		3536	return ret;
		3537	__skb_push(skb, len_diff_abs);
		3538	memset(skb->data, 0, len_diff_abs);
		3539	} else {
		3540	if (unlikely(!pskb_may_pull(skb, len_diff_abs)))
		3541	return -ENOMEM;
		3542	__skb_pull(skb, len_diff_abs);
		3543	}
		3544	bpf_compute_data_end_sk_skb(skb);
		3545	if (tls_sw_has_ctx_rx(skb->sk)) {
		3546	struct strp_msg *rxm = strp_msg(skb);
		3547
		3548	rxm->full_len += len_diff;
		3549	}
		3550	return ret;
		3551	}
		3552
		3553	static const struct bpf_func_proto sk_skb_adjust_room_proto = {
		3554	.func = sk_skb_adjust_room,
		3555	.gpl_only = false,
		3556	.ret_type = RET_INTEGER,
		3557	.arg1_type = ARG_PTR_TO_CTX,
		3558	.arg2_type = ARG_ANYTHING,
		3559	.arg3_type = ARG_ANYTHING,
		3560	.arg4_type = ARG_ANYTHING,
		3561	};
		3562
3482	BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,	3563	BPF_CALL_4(bpf_skb_adjust_room, struct sk_buff *, skb, s32, len_diff,
3483	u32, mode, u64, flags)	3564	u32, mode, u64, flags)
3484	{	3565	{
@@ -4784,6 +4865,10 @@ static int _bpf_setsockopt(struct sock *sk, int level, int optname, · 1299 unmodified lines
4784	else	4865	else
4785	icsk->icsk_user_timeout = val;	4866	icsk->icsk_user_timeout = val;
4786	break;	4867	break;
		4868	case TCP_NOTSENT_LOWAT:
		4869	tp->notsent_lowat = val;
		4870	sk->sk_write_space(sk);
		4871	break;
4787	default:	4872	default:
4788	ret = -EINVAL;	4873	ret = -EINVAL;
4789	}	4874	}
@@ -5149,7 +5234,6 @@ static int bpf_fib_set_fwd_params(struct bpf_fib_lookup *params, · 359 unmodified lines
5149	memcpy(params->smac, dev->dev_addr, ETH_ALEN);	5234	memcpy(params->smac, dev->dev_addr, ETH_ALEN);
5150	params->h_vlan_TCI = 0;	5235	params->h_vlan_TCI = 0;
5151	params->h_vlan_proto = 0;	5236	params->h_vlan_proto = 0;
5152	params->ifindex = dev->ifindex;
5153		5237
5154	return 0;	5238	return 0;
5155	}	5239	}
@@ -5246,6 +5330,7 @@ static int bpf_ipv4_fib_lookup(struct net net, struct bpf_fib_lookup params, · 90 unmodified lines
5246	dev = nhc->nhc_dev;	5330	dev = nhc->nhc_dev;
5247		5331
5248	params->rt_metric = res.fi->fib_priority;	5332	params->rt_metric = res.fi->fib_priority;
		5333	params->ifindex = dev->ifindex;
5249		5334
5250	/* xdp and cls_bpf programs are run in RCU-bh so	5335	/* xdp and cls_bpf programs are run in RCU-bh so
5251	* rcu_read_lock_bh is not needed here	5336	* rcu_read_lock_bh is not needed here
@@ -5371,6 +5456,7 @@ static int bpf_ipv6_fib_lookup(struct net net, struct bpf_fib_lookup params, · 119 unmodified lines
5371		5456
5372	dev = res.nh->fib_nh_dev;	5457	dev = res.nh->fib_nh_dev;
5373	params->rt_metric = res.f6i->fib6_metric;	5458	params->rt_metric = res.f6i->fib6_metric;
		5459	params->ifindex = dev->ifindex;
5374		5460
5375	/* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is	5461	/* xdp and cls_bpf programs are run in RCU-bh so rcu_read_lock_bh is
5376	* not needed here.	5462	* not needed here.
@@ -6745,6 +6831,7 @@ bool bpf_helper_changes_pkt_data(void *func) · 1368 unmodified lines
6745	func == bpf_skb_change_tail \|\|	6831	func == bpf_skb_change_tail \|\|
6746	func == sk_skb_change_tail \|\|	6832	func == sk_skb_change_tail \|\|
6747	func == bpf_skb_adjust_room \|\|	6833	func == bpf_skb_adjust_room \|\|
		6834	func == sk_skb_adjust_room \|\|
6748	func == bpf_skb_pull_data \|\|	6835	func == bpf_skb_pull_data \|\|
6749	func == sk_skb_pull_data \|\|	6836	func == sk_skb_pull_data \|\|
6750	func == bpf_clone_redirect \|\|	6837	func == bpf_clone_redirect \|\|
@@ -7005,6 +7092,8 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 254 unmodified lines
7005	return &bpf_redirect_proto;	7092	return &bpf_redirect_proto;
7006	case BPF_FUNC_redirect_neigh:	7093	case BPF_FUNC_redirect_neigh:
7007	return &bpf_redirect_neigh_proto;	7094	return &bpf_redirect_neigh_proto;
		7095	case BPF_FUNC_redirect_peer:
		7096	return &bpf_redirect_peer_proto;
7008	case BPF_FUNC_get_route_realm:	7097	case BPF_FUNC_get_route_realm:
7009	return &bpf_get_route_realm_proto;	7098	return &bpf_get_route_realm_proto;
7010	case BPF_FUNC_get_hash_recalc:	7099	case BPF_FUNC_get_hash_recalc:
@@ -7218,6 +7307,8 @@ sk_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 207 unmodified lines
7218	return &sk_skb_change_tail_proto;	7307	return &sk_skb_change_tail_proto;
7219	case BPF_FUNC_skb_change_head:	7308	case BPF_FUNC_skb_change_head:
7220	return &sk_skb_change_head_proto;	7309	return &sk_skb_change_head_proto;
		7310	case BPF_FUNC_skb_adjust_room:
		7311	return &sk_skb_adjust_room_proto;
7221	case BPF_FUNC_get_socket_cookie:	7312	case BPF_FUNC_get_socket_cookie:
7222	return &bpf_get_socket_cookie_proto;	7313	return &bpf_get_socket_cookie_proto;
7223	case BPF_FUNC_get_socket_uid:	7314	case BPF_FUNC_get_socket_uid:

net/core/skmsg.c+123−38

 static int sk_psock_handle_skb(struct sk_psock *psock, struct sk_buff *skb,
 			       u32 off, u32 len, bool ingress)
 {
-	if (ingress)
+	if (!ingress) {
-		return sk_psock_skb_ingress(psock, skb);
+		if (!sock_writeable(psock->sk))
-	else
+			return -EAGAIN;
 		return skb_send_sock_locked(psock->sk, skb, off, len);
+	}
+	return sk_psock_skb_ingress(psock, skb);
 }
 static void sk_psock_backlog(struct work_struct *work)
 	rcu_assign_sk_user_data(sk, NULL);
 	if (psock->progs.skb_parser)
 		sk_psock_stop_strp(sk, psock);
+	else if (psock->progs.skb_verdict)
+		sk_psock_stop_verdict(sk, psock);
 	write_unlock_bh(&sk->sk_callback_lock);
 	sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED);
 static int sk_psock_bpf_run(struct sk_psock *psock, struct bpf_prog *prog,
 			    struct sk_buff *skb)
 {
-	int ret;
-	skb->sk = psock->sk;
 	bpf_compute_data_end_sk_skb(skb);
-	ret = bpf_prog_run_pin_on_cpu(prog, skb);
+	return bpf_prog_run_pin_on_cpu(prog, skb);
-	/* strparser clones the skb before handing it to a upper layer,
-	 * meaning skb_orphan has been called. We NULL sk on the way out
-	 * to ensure we don't trigger a BUG_ON() in skb/sk operations
-	 * later and because we are not charging the memory of this skb
-	 * to any socket yet.
-	 */
-	skb->sk = NULL;
-	return ret;
 }
 static struct sk_psock *sk_psock_from_strp(struct strparser *strp)
 {
 	struct sk_psock *psock_other;
 	struct sock *sk_other;
-	bool ingress;
 	sk_other = tcp_skb_bpf_redirect_fetch(skb);
+	/* This error is a buggy BPF program, it returned a redirect
+	 * return code, but then didn't set a redirect interface.
+	 */
 	if (unlikely(!sk_other)) {
 		kfree_skb(skb);
 		return;
 	}
 	psock_other = sk_psock(sk_other);
+	/* This error indicates the socket is being torn down or had another
+	 * error that caused the pipe to break. We can't send a packet on
+	 * a socket that is in this state so we drop the skb.
+	 */
 	if (!psock_other || sock_flag(sk_other, SOCK_DEAD) ||
 	    !sk_psock_test_state(psock_other, SK_PSOCK_TX_ENABLED)) {
 		kfree_skb(skb);
 		return;
 	}
-	ingress = tcp_skb_bpf_ingress(skb);
+	skb_queue_tail(&psock_other->ingress_skb, skb);
-	if ((!ingress && sock_writeable(sk_other)) ||
+	schedule_work(&psock_other->work);
-	    (ingress &&
-	     atomic_read(&sk_other->sk_rmem_alloc) <=
-	     sk_other->sk_rcvbuf)) {
-		if (!ingress)
-			skb_set_owner_w(skb, sk_other);
-		skb_queue_tail(&psock_other->ingress_skb, skb);
-		schedule_work(&psock_other->work);
-	} else {
-		kfree_skb(skb);
-	}
 }
-static void sk_psock_tls_verdict_apply(struct sk_buff *skb, int verdict)
+static void sk_psock_tls_verdict_apply(struct sk_buff *skb, struct sock *sk, int verdict)
 {
 	switch (verdict) {
 	case __SK_REDIRECT:
+		skb_set_owner_r(skb, sk);
 		sk_psock_skb_redirect(skb);
 		break;
 	case __SK_PASS:
 	rcu_read_lock();
 	prog = READ_ONCE(psock->progs.skb_verdict);
 	if (likely(prog)) {
+		/* We skip full set_owner_r here because if we do a SK_PASS
+		 * or SK_DROP we can skip skb memory accounting and use the
+		 * TLS context.
+		 */
+		skb->sk = psock->sk;
 		tcp_skb_bpf_redirect_clear(skb);
 		ret = sk_psock_bpf_run(psock, prog, skb);
 		ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb));
+		skb->sk = NULL;
 	}
-	sk_psock_tls_verdict_apply(skb, ret);
+	sk_psock_tls_verdict_apply(skb, psock->sk, ret);
 	rcu_read_unlock();
 	return ret;
 }
 static void sk_psock_verdict_apply(struct sk_psock *psock,
 				   struct sk_buff *skb, int verdict)
 {
+	struct tcp_skb_cb *tcp;
 	struct sock *sk_other;
+	int err = -EIO;
 	switch (verdict) {
 	case __SK_PASS:
 		    !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {
 			goto out_free;
 		}
-		if (atomic_read(&sk_other->sk_rmem_alloc) <=
-		    sk_other->sk_rcvbuf) {
-			struct tcp_skb_cb *tcp = TCP_SKB_CB(skb);
-			tcp->bpf.flags |= BPF_F_INGRESS;
+		tcp = TCP_SKB_CB(skb);
+		tcp->bpf.flags |= BPF_F_INGRESS;
+		/* If the queue is empty then we can submit directly
+		 * into the msg queue. If its not empty we have to
+		 * queue work otherwise we may get OOO data. Otherwise,
+		 * if sk_psock_skb_ingress errors will be handled by
+		 * retrying later from workqueue.
+		 */
+		if (skb_queue_empty(&psock->ingress_skb)) {
+			err = sk_psock_skb_ingress(psock, skb);
+		}
+		if (err < 0) {
 			skb_queue_tail(&psock->ingress_skb, skb);
 			schedule_work(&psock->work);
-			break;
 		}
-		goto out_free;
+		break;
 	case __SK_REDIRECT:
 		sk_psock_skb_redirect(skb);
 		break;
 		kfree_skb(skb);
 		goto out;
 	}
+	skb_set_owner_r(skb, sk);
 	prog = READ_ONCE(psock->progs.skb_verdict);
 	if (likely(prog)) {
-		skb_orphan(skb);
 		tcp_skb_bpf_redirect_clear(skb);
 		ret = sk_psock_bpf_run(psock, prog, skb);
 		ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb));
 	rcu_read_lock();
 	prog = READ_ONCE(psock->progs.skb_parser);
-	if (likely(prog))
+	if (likely(prog)) {
+		skb->sk = psock->sk;
 		ret = sk_psock_bpf_run(psock, prog, skb);
+		skb->sk = NULL;
+	}
 	rcu_read_unlock();
 	return ret;
 }
 	rcu_read_unlock();
 }
+static int sk_psock_verdict_recv(read_descriptor_t *desc, struct sk_buff *skb,
+				 unsigned int offset, size_t orig_len)
+{
+	struct sock *sk = (struct sock *)desc->arg.data;
+	struct sk_psock *psock;
+	struct bpf_prog *prog;
+	int ret = __SK_DROP;
+	int len = skb->len;
+	/* clone here so sk_eat_skb() in tcp_read_sock does not drop our data */
+	skb = skb_clone(skb, GFP_ATOMIC);
+	if (!skb) {
+		desc->error = -ENOMEM;
+		return 0;
+	}
+	rcu_read_lock();
+	psock = sk_psock(sk);
+	if (unlikely(!psock)) {
+		len = 0;
+		kfree_skb(skb);
+		goto out;
+	}
+	skb_set_owner_r(skb, sk);
+	prog = READ_ONCE(psock->progs.skb_verdict);
+	if (likely(prog)) {
+		tcp_skb_bpf_redirect_clear(skb);
+		ret = sk_psock_bpf_run(psock, prog, skb);
+		ret = sk_psock_map_verd(ret, tcp_skb_bpf_redirect_fetch(skb));
+	}
+	sk_psock_verdict_apply(psock, skb, ret);
+out:
+	rcu_read_unlock();
+	return len;
+}
+static void sk_psock_verdict_data_ready(struct sock *sk)
+{
+	struct socket *sock = sk->sk_socket;
+	read_descriptor_t desc;
+	if (unlikely(!sock || !sock->ops || !sock->ops->read_sock))
+		return;
+	desc.arg.data = sk;
+	desc.error = 0;
+	desc.count = 1;
+	sock->ops->read_sock(sk, &desc, sk_psock_verdict_recv);
+}
 static void sk_psock_write_space(struct sock *sk)
 {
 	struct sk_psock *psock;
 	return strp_init(&psock->parser.strp, sk, &cb);
 }
+void sk_psock_start_verdict(struct sock *sk, struct sk_psock *psock)
+{
+	struct sk_psock_parser *parser = &psock->parser;
+	if (parser->enabled)
+		return;
+	parser->saved_data_ready = sk->sk_data_ready;
+	sk->sk_data_ready = sk_psock_verdict_data_ready;
+	sk->sk_write_space = sk_psock_write_space;
+	parser->enabled = true;
+}
 void sk_psock_start_strp(struct sock *sk, struct sk_psock *psock)
 {
 	struct sk_psock_parser *parser = &psock->parser;
 	strp_stop(&parser->strp);
 	parser->enabled = false;
 }
+void sk_psock_stop_verdict(struct sock *sk, struct sk_psock *psock)
+{
+	struct sk_psock_parser *parser = &psock->parser;
+	if (!parser->enabled)
+		return;
+	sk->sk_data_ready = parser->saved_data_ready;
+	parser->saved_data_ready = NULL;
+	parser->enabled = false;
+}

net/core/sock_map.c+23−14

 static void sock_map_del_link(struct sock *sk,
 			      struct sk_psock *psock, void *link_raw)
 {
+	bool strp_stop = false, verdict_stop = false;
 	struct sk_psock_link *link, *tmp;
-	bool strp_stop = false;
 	spin_lock_bh(&psock->link_lock);
 	list_for_each_entry_safe(link, tmp, &psock->link, list) {
 							     map);
 			if (psock->parser.enabled && stab->progs.skb_parser)
 				strp_stop = true;
+			if (psock->parser.enabled && stab->progs.skb_verdict)
+				verdict_stop = true;
 			list_del(&link->list);
 			sk_psock_free_link(link);
 		}
 	}
 	spin_unlock_bh(&psock->link_lock);
-	if (strp_stop) {
+	if (strp_stop || verdict_stop) {
 		write_lock_bh(&sk->sk_callback_lock);
-		sk_psock_stop_strp(sk, psock);
+		if (strp_stop)
+			sk_psock_stop_strp(sk, psock);
+		else
+			sk_psock_stop_verdict(sk, psock);
 		write_unlock_bh(&sk->sk_callback_lock);
 	}
 }
 {
 	struct bpf_prog *msg_parser, *skb_parser, *skb_verdict;
 	struct sk_psock *psock;
-	bool skb_progs;
 	int ret;
 	skb_verdict = READ_ONCE(progs->skb_verdict);
 	skb_parser = READ_ONCE(progs->skb_parser);
-	skb_progs = skb_parser && skb_verdict;
+	if (skb_verdict) {
-	if (skb_progs) {
 		skb_verdict = bpf_prog_inc_not_zero(skb_verdict);
 		if (IS_ERR(skb_verdict))
 			return PTR_ERR(skb_verdict);
+	}
+	if (skb_parser) {
 		skb_parser = bpf_prog_inc_not_zero(skb_parser);
 		if (IS_ERR(skb_parser)) {
 			bpf_prog_put(skb_verdict);
 	if (psock) {
 		if ((msg_parser && READ_ONCE(psock->progs.msg_parser)) ||
-		    (skb_progs  && READ_ONCE(psock->progs.skb_parser))) {
+		    (skb_parser  && READ_ONCE(psock->progs.skb_parser)) ||
+		    (skb_verdict && READ_ONCE(psock->progs.skb_verdict))) {
 			sk_psock_put(sk, psock);
 			ret = -EBUSY;
 			goto out_progs;
 		goto out_drop;
 	write_lock_bh(&sk->sk_callback_lock);
-	if (skb_progs && !psock->parser.enabled) {
+	if (skb_parser && skb_verdict && !psock->parser.enabled) {
 		ret = sk_psock_init_strp(sk, psock);
-		if (ret) {
+		if (ret)
-			write_unlock_bh(&sk->sk_callback_lock);
+			goto out_unlock_drop;
-			goto out_drop;
-		}
 		psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
 		psock_set_prog(&psock->progs.skb_parser, skb_parser);
 		sk_psock_start_strp(sk, psock);
+	} else if (!skb_parser && skb_verdict && !psock->parser.enabled) {
+		psock_set_prog(&psock->progs.skb_verdict, skb_verdict);
+		sk_psock_start_verdict(sk,psock);
 	}
 	write_unlock_bh(&sk->sk_callback_lock);
 	return 0;
+out_unlock_drop:
+	write_unlock_bh(&sk->sk_callback_lock);
 out_drop:
 	sk_psock_put(sk, psock);
 out_progs:
 	if (msg_parser)
 		bpf_prog_put(msg_parser);
 out:
-	if (skb_progs) {
+	if (skb_verdict)
 		bpf_prog_put(skb_verdict);
+	if (skb_parser)
 		bpf_prog_put(skb_parser);
-	}
 	return ret;
 }

net/ipv4/tcp_minisocks.c+0−1

@@ -548,7 +548,6 @@ struct sock tcp_create_openreq_child(const struct sock sk, · 547 unmodified lines
548	newtp->fastopen_req = NULL;	548	newtp->fastopen_req = NULL;
549	RCU_INIT_POINTER(newtp->fastopen_rsk, NULL);	549	RCU_INIT_POINTER(newtp->fastopen_rsk, NULL);
550		550
551	bpf_skops_init_child(sk, newsk);
552	tcp_bpf_clone(sk, newsk);	551	tcp_bpf_clone(sk, newsk);
553		552
554	__TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS);	553	__TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS);

net/xdp/xsk_buff_pool.c+0−3

 #include <net/xsk_buff_pool.h>
 #include <net/xdp_sock.h>
 #include <net/xdp_sock_drv.h>
-#include <linux/dma-direct.h>
-#include <linux/dma-noncoherent.h>
-#include <linux/swiotlb.h>
 #include "xsk_queue.h"
 #include "xdp_umem.h"

net/xdp/xsk_queue.h+4−0

 struct xdp_ring {
 	u32 producer ____cacheline_aligned_in_smp;
+	/* Hinder the adjacent cache prefetcher to prefetch the consumer
+	 * pointer if the producer pointer is touched and vice versa.
+	 */
+	u32 pad ____cacheline_aligned_in_smp;
 	u32 consumer ____cacheline_aligned_in_smp;
 	u32 flags;
 };

net/xdp/xskmap.c+1−1

@@ -132,7 +132,7 @@ static int xsk_map_get_next_key(struct bpf_map map, void key, void *next_key) · 131 unmodified lines
132	return 0;	132	return 0;
133	}	133	}
134		134
135	static u32 xsk_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)	135	static int xsk_map_gen_lookup(struct bpf_map map, struct bpf_insn insn_buf)
136	{	136	{
137	const int ret = BPF_REG_0, mp = BPF_REG_1, index = BPF_REG_2;	137	const int ret = BPF_REG_0, mp = BPF_REG_1, index = BPF_REG_2;
138	struct bpf_insn *insn = insn_buf;	138	struct bpf_insn *insn = insn_buf;

samples/bpf/Makefile+12−3

 per_socket_stats_example-objs := cookie_uid_helper_example.o
 xdp_redirect-objs := xdp_redirect_user.o
 xdp_redirect_map-objs := xdp_redirect_map_user.o
-xdp_redirect_cpu-objs := bpf_load.o xdp_redirect_cpu_user.o
+xdp_redirect_cpu-objs := xdp_redirect_cpu_user.o
-xdp_monitor-objs := bpf_load.o xdp_monitor_user.o
+xdp_monitor-objs := xdp_monitor_user.o
 xdp_rxq_info-objs := xdp_rxq_info_user.o
 syscall_tp-objs := syscall_tp_user.o
 cpustat-objs := cpustat_user.o
 #  make M=samples/bpf/ LLC=~/git/llvm/build/bin/llc CLANG=~/git/llvm/build/bin/clang
 LLC ?= llc
 CLANG ?= clang
+OPT ?= opt
+LLVM_DIS ?= llvm-dis
 LLVM_OBJCOPY ?= llvm-objcopy
 BTF_PAHOLE ?= pahole
 # asm/sysreg.h - inline assembly used by it is incompatible with llvm.
 # But, there is no easy way to fix it, so just exclude it since it is
 # useless for BPF samples.
+# below we use long chain of commands, clang | opt | llvm-dis | llc,
+# to generate final object file. 'clang' compiles the source into IR
+# with native target, e.g., x64, arm64, etc. 'opt' does bpf CORE IR builtin
+# processing (llvm12) and IR optimizations. 'llvm-dis' converts
+# 'opt' output to IR, and finally 'llc' generates bpf byte code.
 $(obj)/%.o: $(src)/%.c
 	@echo "  CLANG-bpf " $@
 	$(Q)$(CLANG) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) $(BPF_EXTRA_CFLAGS) \
 		-Wno-address-of-packed-member -Wno-tautological-compare \
 		-Wno-unknown-warning-option $(CLANG_ARCH_ARGS) \
 		-I$(srctree)/samples/bpf/ -include asm_goto_workaround.h \
-		-O2 -emit-llvm -c $< -o -| $(LLC) -march=bpf $(LLC_FLAGS) -filetype=obj -o $@
+		-O2 -emit-llvm -Xclang -disable-llvm-passes -c $< -o - | \
+		$(OPT) -O2 -mtriple=bpf-pc-linux | $(LLVM_DIS) | \
+		$(LLC) -march=bpf $(LLC_FLAGS) -filetype=obj -o $@
 ifeq ($(DWARF2BTF),y)
 	$(BTF_PAHOLE) -J $@
 endif

samples/bpf/hbm.c+2−1

 #include <errno.h>
 #include <fcntl.h>
 #include <linux/unistd.h>
+#include <linux/compiler.h>
 #include <linux/bpf.h>
 #include <bpf/bpf.h>
 					"Option -%c requires an argument.\n\n",
 					optopt);
 		case 'h':
-			fallthrough;
+			__fallthrough;
 		default:
 			Usage();
 			return 0;

samples/bpf/xdp_monitor_kern.c+30−30

@@ -6,21 +6,21 @@ · 5 unmodified lines
6	#include <uapi/linux/bpf.h>	6	#include <uapi/linux/bpf.h>
7	#include <bpf/bpf_helpers.h>	7	#include <bpf/bpf_helpers.h>
8		8
9	struct bpf_map_def SEC("maps") redirect_err_cnt = {	9	struct {
10	.type = BPF_MAP_TYPE_PERCPU_ARRAY,	10	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
11	.key_size = sizeof(u32),	11	__type(key, u32);
12	.value_size = sizeof(u64),	12	__type(value, u64);
13	.max_entries = 2,	13	__uint(max_entries, 2);
14	/* TODO: have entries for all possible errno's */	14	/* TODO: have entries for all possible errno's */
15	};	15	} redirect_err_cnt SEC(".maps");
16		16
17	#define XDP_UNKNOWN XDP_REDIRECT + 1	17	#define XDP_UNKNOWN XDP_REDIRECT + 1
18	struct bpf_map_def SEC("maps") exception_cnt = {	18	struct {
19	.type = BPF_MAP_TYPE_PERCPU_ARRAY,	19	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
20	.key_size = sizeof(u32),	20	__type(key, u32);
21	.value_size = sizeof(u64),	21	__type(value, u64);
22	.max_entries = XDP_UNKNOWN + 1,	22	__uint(max_entries, XDP_UNKNOWN + 1);
23	};	23	} exception_cnt SEC(".maps");
24		24
25	/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_redirect/format	25	/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_redirect/format
26	* Code in: kernel/include/trace/events/xdp.h	26	* Code in: kernel/include/trace/events/xdp.h
@@ -129,19 +129,19 @@ struct datarec { · 102 unmodified lines
129	};	129	};
130	#define MAX_CPUS 64	130	#define MAX_CPUS 64
131		131
132	struct bpf_map_def SEC("maps") cpumap_enqueue_cnt = {	132	struct {
133	.type = BPF_MAP_TYPE_PERCPU_ARRAY,	133	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
134	.key_size = sizeof(u32),	134	__type(key, u32);
135	.value_size = sizeof(struct datarec),	135	__type(value, struct datarec);
136	.max_entries = MAX_CPUS,	136	__uint(max_entries, MAX_CPUS);
137	};	137	} cpumap_enqueue_cnt SEC(".maps");
138		138
139	struct bpf_map_def SEC("maps") cpumap_kthread_cnt = {	139	struct {
140	.type = BPF_MAP_TYPE_PERCPU_ARRAY,	140	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
141	.key_size = sizeof(u32),	141	__type(key, u32);
142	.value_size = sizeof(struct datarec),	142	__type(value, struct datarec);
143	.max_entries = 1,	143	__uint(max_entries, 1);
144	};	144	} cpumap_kthread_cnt SEC(".maps");
145		145
146	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_enqueue/format	146	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_enqueue/format
147	* Code in: kernel/include/trace/events/xdp.h	147	* Code in: kernel/include/trace/events/xdp.h
@@ -210,12 +210,12 @@ int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx) · 62 unmodified lines
210	return 0;	210	return 0;
211	}	211	}
212		212
213	struct bpf_map_def SEC("maps") devmap_xmit_cnt = {	213	struct {
214	.type = BPF_MAP_TYPE_PERCPU_ARRAY,	214	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
215	.key_size = sizeof(u32),	215	__type(key, u32);
216	.value_size = sizeof(struct datarec),	216	__type(value, struct datarec);
217	.max_entries = 1,	217	__uint(max_entries, 1);
218	};	218	} devmap_xmit_cnt SEC(".maps");
219		219
220	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_devmap_xmit/format	220	/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_devmap_xmit/format
221	* Code in: kernel/include/trace/events/xdp.h	221	* Code in: kernel/include/trace/events/xdp.h

samples/bpf/xdp_monitor_user.c+120−39

@@ -26,12 +26,37 @@ static const char *__doc_err_only__= · 25 unmodified lines
26	#include <net/if.h>	26	#include <net/if.h>
27	#include <time.h>	27	#include <time.h>
28		28
		29	#include <signal.h>
29	#include <bpf/bpf.h>	30	#include <bpf/bpf.h>
30	#include "bpf_load.h"	31	#include <bpf/libbpf.h>
31	#include "bpf_util.h"	32	#include "bpf_util.h"
32		33
		34	enum map_type {
		35	REDIRECT_ERR_CNT,
		36	EXCEPTION_CNT,
		37	CPUMAP_ENQUEUE_CNT,
		38	CPUMAP_KTHREAD_CNT,
		39	DEVMAP_XMIT_CNT,
		40	};
		41
		42	static const char *const map_type_strings[] = {
		43	[REDIRECT_ERR_CNT] = "redirect_err_cnt",
		44	[EXCEPTION_CNT] = "exception_cnt",
		45	[CPUMAP_ENQUEUE_CNT] = "cpumap_enqueue_cnt",
		46	[CPUMAP_KTHREAD_CNT] = "cpumap_kthread_cnt",
		47	[DEVMAP_XMIT_CNT] = "devmap_xmit_cnt",
		48	};
		49
		50	#define NUM_MAP 5
		51	#define NUM_TP 8
		52
		53	static int tp_cnt;
		54	static int map_cnt;
33	static int verbose = 1;	55	static int verbose = 1;
34	static bool debug = false;	56	static bool debug = false;
		57	struct bpf_map *map_data[NUM_MAP] = {};
		58	struct bpf_link *tp_links[NUM_TP] = {};
		59	struct bpf_object *obj;
35		60
36	static const struct option long_options[] = {	61	static const struct option long_options[] = {
37	{"help", no_argument, NULL, 'h' },	62	{"help", no_argument, NULL, 'h' },
@@ -41,6 +66,16 @@ static const struct option long_options[] = { · 3 unmodified lines
41	{0, 0, NULL, 0 }	66	{0, 0, NULL, 0 }
42	};	67	};
43		68
		69	static void int_exit(int sig)
		70	{
		71	/* Detach tracepoints */
		72	while (tp_cnt)
		73	bpf_link__destroy(tp_links[--tp_cnt]);
		74
		75	bpf_object__close(obj);
		76	exit(0);
		77	}
		78
44	/* C standard specifies two constants, EXIT_SUCCESS(0) and EXIT_FAILURE(1) */	79	/* C standard specifies two constants, EXIT_SUCCESS(0) and EXIT_FAILURE(1) */
45	#define EXIT_FAIL_MEM 5	80	#define EXIT_FAIL_MEM 5
46		81
@@ -483,23 +518,23 @@ static bool stats_collect(struct stats_record *rec) · 436 unmodified lines
483	* this can happen by someone running perf-record -e	518	* this can happen by someone running perf-record -e
484	*/	519	*/
485		520
486	fd = map_data[0].fd; /* map0: redirect_err_cnt */	521	fd = bpf_map__fd(map_data[REDIRECT_ERR_CNT]);
487	for (i = 0; i < REDIR_RES_MAX; i++)	522	for (i = 0; i < REDIR_RES_MAX; i++)
488	map_collect_record_u64(fd, i, &rec->xdp_redirect[i]);	523	map_collect_record_u64(fd, i, &rec->xdp_redirect[i]);
489		524
490	fd = map_data[1].fd; /* map1: exception_cnt */	525	fd = bpf_map__fd(map_data[EXCEPTION_CNT]);
491	for (i = 0; i < XDP_ACTION_MAX; i++) {	526	for (i = 0; i < XDP_ACTION_MAX; i++) {
492	map_collect_record_u64(fd, i, &rec->xdp_exception[i]);	527	map_collect_record_u64(fd, i, &rec->xdp_exception[i]);
493	}	528	}
494		529
495	fd = map_data[2].fd; /* map2: cpumap_enqueue_cnt */	530	fd = bpf_map__fd(map_data[CPUMAP_ENQUEUE_CNT]);
496	for (i = 0; i < MAX_CPUS; i++)	531	for (i = 0; i < MAX_CPUS; i++)
497	map_collect_record(fd, i, &rec->xdp_cpumap_enqueue[i]);	532	map_collect_record(fd, i, &rec->xdp_cpumap_enqueue[i]);
498		533
499	fd = map_data[3].fd; /* map3: cpumap_kthread_cnt */	534	fd = bpf_map__fd(map_data[CPUMAP_KTHREAD_CNT]);
500	map_collect_record(fd, 0, &rec->xdp_cpumap_kthread);	535	map_collect_record(fd, 0, &rec->xdp_cpumap_kthread);
501		536
502	fd = map_data[4].fd; /* map4: devmap_xmit_cnt */	537	fd = bpf_map__fd(map_data[DEVMAP_XMIT_CNT]);
503	map_collect_record(fd, 0, &rec->xdp_devmap_xmit);	538	map_collect_record(fd, 0, &rec->xdp_devmap_xmit);
504		539
505	return true;	540	return true;
@@ -598,8 +633,8 @@ static void stats_poll(int interval, bool err_only) · 92 unmodified lines
598		633
599	/* TODO Need more advanced stats on error types */	634	/* TODO Need more advanced stats on error types */
600	if (verbose) {	635	if (verbose) {
601	printf(" - Stats map0: %s\n", map_data[0].name);	636	printf(" - Stats map0: %s\n", bpf_map__name(map_data[0]));
602	printf(" - Stats map1: %s\n", map_data[1].name);	637	printf(" - Stats map1: %s\n", bpf_map__name(map_data[1]));
603	printf("\n");	638	printf("\n");
604	}	639	}
605	fflush(stdout);	640	fflush(stdout);
@@ -618,44 +653,51 @@ static void stats_poll(int interval, bool err_only) · 12 unmodified lines
618		653
619	static void print_bpf_prog_info(void)	654	static void print_bpf_prog_info(void)
620	{	655	{
621	int i;	656	struct bpf_program *prog;
		657	struct bpf_map *map;
		658	int i = 0;
622		659
623	/* Prog info */	660	/* Prog info */
624	printf("Loaded BPF prog have %d bpf program(s)\n", prog_cnt);	661	printf("Loaded BPF prog have %d bpf program(s)\n", tp_cnt);
625	for (i = 0; i < prog_cnt; i++) {	662	bpf_object__for_each_program(prog, obj) {
626	printf(" - prog_fd[%d] = fd(%d)\n", i, prog_fd[i]);	663	printf(" - prog_fd[%d] = fd(%d)\n", i, bpf_program__fd(prog));
		664	i++;
627	}	665	}
628		666
		667	i = 0;
629	/* Maps info */	668	/* Maps info */
630	printf("Loaded BPF prog have %d map(s)\n", map_data_count);	669	printf("Loaded BPF prog have %d map(s)\n", map_cnt);
631	for (i = 0; i < map_data_count; i++) {	670	bpf_object__for_each_map(map, obj) {
632	char *name = map_data[i].name;	671	const char *name = bpf_map__name(map);
633	int fd = map_data[i].fd;	672	int fd = bpf_map__fd(map);
634		673
635	printf(" - map_data[%d] = fd(%d) name:%s\n", i, fd, name);	674	printf(" - map_data[%d] = fd(%d) name:%s\n", i, fd, name);
		675	i++;
636	}	676	}
637		677
638	/* Event info */	678	/* Event info */
639	printf("Searching for (max:%d) event file descriptor(s)\n", prog_cnt);	679	printf("Searching for (max:%d) event file descriptor(s)\n", tp_cnt);
640	for (i = 0; i < prog_cnt; i++) {	680	for (i = 0; i < tp_cnt; i++) {
641	if (event_fd[i] != -1)	681	int fd = bpf_link__fd(tp_links[i]);
642	printf(" - event_fd[%d] = fd(%d)\n", i, event_fd[i]);	682
		683	if (fd != -1)
		684	printf(" - event_fd[%d] = fd(%d)\n", i, fd);
643	}	685	}
644	}	686	}
645		687
646	int main(int argc, char **argv)	688	int main(int argc, char **argv)
647	{	689	{
648	struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};	690	struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
		691	struct bpf_program *prog;
649	int longindex = 0, opt;	692	int longindex = 0, opt;
650	int ret = EXIT_SUCCESS;	693	int ret = EXIT_FAILURE;
651	char bpf_obj_file[256];	694	enum map_type type;
		695	char filename[256];
652		696
653	/* Default settings: */	697	/* Default settings: */
654	bool errors_only = true;	698	bool errors_only = true;
655	int interval = 2;	699	int interval = 2;
656		700
657	snprintf(bpf_obj_file, sizeof(bpf_obj_file), "%s_kern.o", argv[0]);
658
659	/* Parse commands line args */	701	/* Parse commands line args */
660	while ((opt = getopt_long(argc, argv, "hDSs:",	702	while ((opt = getopt_long(argc, argv, "hDSs:",
661	long_options, &longindex)) != -1) {	703	long_options, &longindex)) != -1) {
@@ -672,40 +714,79 @@ int main(int argc, char **argv) · 10 unmodified lines
672	case 'h':	714	case 'h':
673	default:	715	default:
674	usage(argv);	716	usage(argv);
675	return EXIT_FAILURE;	717	return ret;
676	}	718	}
677	}	719	}
678		720
		721	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
679	if (setrlimit(RLIMIT_MEMLOCK, &r)) {	722	if (setrlimit(RLIMIT_MEMLOCK, &r)) {
680	perror("setrlimit(RLIMIT_MEMLOCK)");	723	perror("setrlimit(RLIMIT_MEMLOCK)");
681	return EXIT_FAILURE;	724	return ret;
682	}	725	}
683		726
684	if (load_bpf_file(bpf_obj_file)) {	727	/* Remove tracepoint program when program is interrupted or killed */
685	printf("ERROR - bpf_log_buf: %s", bpf_log_buf);	728	signal(SIGINT, int_exit);
686	return EXIT_FAILURE;	729	signal(SIGTERM, int_exit);
		730
		731	obj = bpf_object__open_file(filename, NULL);
		732	if (libbpf_get_error(obj)) {
		733	printf("ERROR: opening BPF object file failed\n");
		734	obj = NULL;
		735	goto cleanup;
		736	}
		737
		738	/* load BPF program */
		739	if (bpf_object__load(obj)) {
		740	printf("ERROR: loading BPF object file failed\n");
		741	goto cleanup;
		742	}
		743
		744	for (type = 0; type < NUM_MAP; type++) {
		745	map_data[type] =
		746	bpf_object__find_map_by_name(obj, map_type_strings[type]);
		747
		748	if (libbpf_get_error(map_data[type])) {
		749	printf("ERROR: finding a map in obj file failed\n");
		750	goto cleanup;
		751	}
		752	map_cnt++;
687	}	753	}
688	if (!prog_fd[0]) {	754
689	printf("ERROR - load_bpf_file: %s\n", strerror(errno));	755	bpf_object__for_each_program(prog, obj) {
690	return EXIT_FAILURE;	756	tp_links[tp_cnt] = bpf_program__attach(prog);
		757	if (libbpf_get_error(tp_links[tp_cnt])) {
		758	printf("ERROR: bpf_program__attach failed\n");
		759	tp_links[tp_cnt] = NULL;
		760	goto cleanup;
		761	}
		762	tp_cnt++;
691	}	763	}
692		764
693	if (debug) {	765	if (debug) {
694	print_bpf_prog_info();	766	print_bpf_prog_info();
695	}	767	}
696		768
697	/* Unload/stop tracepoint event by closing fd's */	769	/* Unload/stop tracepoint event by closing bpf_link's */
698	if (errors_only) {	770	if (errors_only) {
699	/* The prog_fd[i] and event_fd[i] depend on the	771	/* The bpf_link[i] depend on the order of
700	* order the functions was defined in _kern.c	772	* the functions was defined in _kern.c
701	*/	773	*/
702	close(event_fd[2]); /* tracepoint/xdp/xdp_redirect */	774	bpf_link__destroy(tp_links[2]); /* tracepoint/xdp/xdp_redirect */
703	close(prog_fd[2]); /* func: trace_xdp_redirect */	775	tp_links[2] = NULL;
704	close(event_fd[3]); /* tracepoint/xdp/xdp_redirect_map */	776
705	close(prog_fd[3]); /* func: trace_xdp_redirect_map */	777	bpf_link__destroy(tp_links[3]); /* tracepoint/xdp/xdp_redirect_map */
		778	tp_links[3] = NULL;
706	}	779	}
707		780
708	stats_poll(interval, errors_only);	781	stats_poll(interval, errors_only);
709		782
		783	ret = EXIT_SUCCESS;
		784
		785	cleanup:
		786	/* Detach tracepoints */
		787	while (tp_cnt)
		788	bpf_link__destroy(tp_links[--tp_cnt]);
		789
		790	bpf_object__close(obj);
710	return ret;	791	return ret;
711	}	792	}

samples/bpf/xdp_redirect_cpu_user.c+72−81

@@ -37,18 +37,35 @@ static __u32 prog_id; · 36 unmodified lines
37		37
38	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;	38	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
39	static int n_cpus;	39	static int n_cpus;
40	static int cpu_map_fd;	40
41	static int rx_cnt_map_fd;	41	enum map_type {
42	static int redirect_err_cnt_map_fd;	42	CPU_MAP,
43	static int cpumap_enqueue_cnt_map_fd;	43	RX_CNT,
44	static int cpumap_kthread_cnt_map_fd;	44	REDIRECT_ERR_CNT,
45	static int cpus_available_map_fd;	45	CPUMAP_ENQUEUE_CNT,
46	static int cpus_count_map_fd;	46	CPUMAP_KTHREAD_CNT,
47	static int cpus_iterator_map_fd;	47	CPUS_AVAILABLE,
48	static int exception_cnt_map_fd;	48	CPUS_COUNT,
		49	CPUS_ITERATOR,
		50	EXCEPTION_CNT,
		51	};
		52
		53	static const char *const map_type_strings[] = {
		54	[CPU_MAP] = "cpu_map",
		55	[RX_CNT] = "rx_cnt",
		56	[REDIRECT_ERR_CNT] = "redirect_err_cnt",
		57	[CPUMAP_ENQUEUE_CNT] = "cpumap_enqueue_cnt",
		58	[CPUMAP_KTHREAD_CNT] = "cpumap_kthread_cnt",
		59	[CPUS_AVAILABLE] = "cpus_available",
		60	[CPUS_COUNT] = "cpus_count",
		61	[CPUS_ITERATOR] = "cpus_iterator",
		62	[EXCEPTION_CNT] = "exception_cnt",
		63	};
49		64
50	#define NUM_TP 5	65	#define NUM_TP 5
51	struct bpf_link *tp_links[NUM_TP] = { 0 };	66	#define NUM_MAP 9
		67	struct bpf_link *tp_links[NUM_TP] = {};
		68	static int map_fds[NUM_MAP];
52	static int tp_cnt = 0;	69	static int tp_cnt = 0;
53		70
54	/* Exit return codes */	71	/* Exit return codes */
@@ -527,20 +544,20 @@ static void stats_collect(struct stats_record *rec) · 472 unmodified lines
527	{	544	{
528	int fd, i;	545	int fd, i;
529		546
530	fd = rx_cnt_map_fd;	547	fd = map_fds[RX_CNT];
531	map_collect_percpu(fd, 0, &rec->rx_cnt);	548	map_collect_percpu(fd, 0, &rec->rx_cnt);
532		549
533	fd = redirect_err_cnt_map_fd;	550	fd = map_fds[REDIRECT_ERR_CNT];
534	map_collect_percpu(fd, 1, &rec->redir_err);	551	map_collect_percpu(fd, 1, &rec->redir_err);
535		552
536	fd = cpumap_enqueue_cnt_map_fd;	553	fd = map_fds[CPUMAP_ENQUEUE_CNT];
537	for (i = 0; i < n_cpus; i++)	554	for (i = 0; i < n_cpus; i++)
538	map_collect_percpu(fd, i, &rec->enq[i]);	555	map_collect_percpu(fd, i, &rec->enq[i]);
539		556
540	fd = cpumap_kthread_cnt_map_fd;	557	fd = map_fds[CPUMAP_KTHREAD_CNT];
541	map_collect_percpu(fd, 0, &rec->kthread);	558	map_collect_percpu(fd, 0, &rec->kthread);
542		559
543	fd = exception_cnt_map_fd;	560	fd = map_fds[EXCEPTION_CNT];
544	map_collect_percpu(fd, 0, &rec->exception);	561	map_collect_percpu(fd, 0, &rec->exception);
545	}	562	}
546		563
@@ -565,7 +582,7 @@ static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value, · 18 unmodified lines
565	/* Add a CPU entry to cpumap, as this allocate a cpu entry in	582	/* Add a CPU entry to cpumap, as this allocate a cpu entry in
566	* the kernel for the cpu.	583	* the kernel for the cpu.
567	*/	584	*/
568	ret = bpf_map_update_elem(cpu_map_fd, &cpu, value, 0);	585	ret = bpf_map_update_elem(map_fds[CPU_MAP], &cpu, value, 0);
569	if (ret) {	586	if (ret) {
570	fprintf(stderr, "Create CPU entry failed (err:%d)\n", ret);	587	fprintf(stderr, "Create CPU entry failed (err:%d)\n", ret);
571	exit(EXIT_FAIL_BPF);	588	exit(EXIT_FAIL_BPF);
@@ -574,21 +591,21 @@ static int create_cpu_entry(__u32 cpu, struct bpf_cpumap_val *value, · 2 unmodified lines
574	/* Inform bpf_prog's that a new CPU is available to select	591	/* Inform bpf_prog's that a new CPU is available to select
575	* from via some control maps.	592	* from via some control maps.
576	*/	593	*/
577	ret = bpf_map_update_elem(cpus_available_map_fd, &avail_idx, &cpu, 0);	594	ret = bpf_map_update_elem(map_fds[CPUS_AVAILABLE], &avail_idx, &cpu, 0);
578	if (ret) {	595	if (ret) {
579	fprintf(stderr, "Add to avail CPUs failed\n");	596	fprintf(stderr, "Add to avail CPUs failed\n");
580	exit(EXIT_FAIL_BPF);	597	exit(EXIT_FAIL_BPF);
581	}	598	}
582		599
583	/* When not replacing/updating existing entry, bump the count */	600	/* When not replacing/updating existing entry, bump the count */
584	ret = bpf_map_lookup_elem(cpus_count_map_fd, &key, &curr_cpus_count);	601	ret = bpf_map_lookup_elem(map_fds[CPUS_COUNT], &key, &curr_cpus_count);
585	if (ret) {	602	if (ret) {
586	fprintf(stderr, "Failed reading curr cpus_count\n");	603	fprintf(stderr, "Failed reading curr cpus_count\n");
587	exit(EXIT_FAIL_BPF);	604	exit(EXIT_FAIL_BPF);
588	}	605	}
589	if (new) {	606	if (new) {
590	curr_cpus_count++;	607	curr_cpus_count++;
591	ret = bpf_map_update_elem(cpus_count_map_fd, &key,	608	ret = bpf_map_update_elem(map_fds[CPUS_COUNT], &key,
592	&curr_cpus_count, 0);	609	&curr_cpus_count, 0);
593	if (ret) {	610	if (ret) {
594	fprintf(stderr, "Failed write curr cpus_count\n");	611	fprintf(stderr, "Failed write curr cpus_count\n");
@@ -612,7 +629,7 @@ static void mark_cpus_unavailable(void) · 17 unmodified lines
612	int ret, i;	629	int ret, i;
613		630
614	for (i = 0; i < n_cpus; i++) {	631	for (i = 0; i < n_cpus; i++) {
615	ret = bpf_map_update_elem(cpus_available_map_fd, &i,	632	ret = bpf_map_update_elem(map_fds[CPUS_AVAILABLE], &i,
616	&invalid_cpu, 0);	633	&invalid_cpu, 0);
617	if (ret) {	634	if (ret) {
618	fprintf(stderr, "Failed marking CPU unavailable\n");	635	fprintf(stderr, "Failed marking CPU unavailable\n");
@@ -665,68 +682,37 @@ static void stats_poll(int interval, bool use_separators, char *prog_name, · 46 unmodified lines
665	free_stats_record(prev);	682	free_stats_record(prev);
666	}	683	}
667		684
668	static struct bpf_link * attach_tp(struct bpf_object *obj,	685	static int init_tracepoints(struct bpf_object *obj)
669	const char *tp_category,
670	const char* tp_name)
671	{	686	{
672	struct bpf_program *prog;	687	struct bpf_program *prog;
673	struct bpf_link *link;
674	char sec_name[PATH_MAX];
675	int len;
676		688
677	len = snprintf(sec_name, PATH_MAX, "tracepoint/%s/%s",	689	bpf_object__for_each_program(prog, obj) {
678	tp_category, tp_name);	690	if (bpf_program__is_tracepoint(prog) != true)
679	if (len < 0)	691	continue;
680	exit(EXIT_FAIL);
681		692
682	prog = bpf_object__find_program_by_title(obj, sec_name);	693	tp_links[tp_cnt] = bpf_program__attach(prog);
683	if (!prog) {	694	if (libbpf_get_error(tp_links[tp_cnt])) {
684	fprintf(stderr, "ERR: finding progsec: %s\n", sec_name);	695	tp_links[tp_cnt] = NULL;
685	exit(EXIT_FAIL_BPF);	696	return -EINVAL;
		697	}
		698	tp_cnt++;
686	}	699	}
687		700
688	link = bpf_program__attach_tracepoint(prog, tp_category, tp_name);	701	return 0;
689	if (libbpf_get_error(link))
690	exit(EXIT_FAIL_BPF);
691
692	return link;
693	}
694
695	static void init_tracepoints(struct bpf_object *obj) {
696	tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_redirect_err");
697	tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_redirect_map_err");
698	tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_exception");
699	tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_cpumap_enqueue");
700	tp_links[tp_cnt++] = attach_tp(obj, "xdp", "xdp_cpumap_kthread");
701	}	702	}
702		703
703	static int init_map_fds(struct bpf_object *obj)	704	static int init_map_fds(struct bpf_object *obj)
704	{	705	{
705	/* Maps updated by tracepoints */	706	enum map_type type;
706	redirect_err_cnt_map_fd =	707
707	bpf_object__find_map_fd_by_name(obj, "redirect_err_cnt");	708	for (type = 0; type < NUM_MAP; type++) {
708	exception_cnt_map_fd =	709	map_fds[type] =
709	bpf_object__find_map_fd_by_name(obj, "exception_cnt");	710	bpf_object__find_map_fd_by_name(obj,
710	cpumap_enqueue_cnt_map_fd =	711	map_type_strings[type]);
711	bpf_object__find_map_fd_by_name(obj, "cpumap_enqueue_cnt");	712
712	cpumap_kthread_cnt_map_fd =	713	if (map_fds[type] < 0)
713	bpf_object__find_map_fd_by_name(obj, "cpumap_kthread_cnt");	714	return -ENOENT;
714		715	}
715	/* Maps used by XDP */
716	rx_cnt_map_fd = bpf_object__find_map_fd_by_name(obj, "rx_cnt");
717	cpu_map_fd = bpf_object__find_map_fd_by_name(obj, "cpu_map");
718	cpus_available_map_fd =
719	bpf_object__find_map_fd_by_name(obj, "cpus_available");
720	cpus_count_map_fd = bpf_object__find_map_fd_by_name(obj, "cpus_count");
721	cpus_iterator_map_fd =
722	bpf_object__find_map_fd_by_name(obj, "cpus_iterator");
723
724	if (cpu_map_fd < 0 \|\| rx_cnt_map_fd < 0 \|\|
725	redirect_err_cnt_map_fd < 0 \|\| cpumap_enqueue_cnt_map_fd < 0 \|\|
726	cpumap_kthread_cnt_map_fd < 0 \|\| cpus_available_map_fd < 0 \|\|
727	cpus_count_map_fd < 0 \|\| cpus_iterator_map_fd < 0 \|\|
728	exception_cnt_map_fd < 0)
729	return -ENOENT;
730		716
731	return 0;	717	return 0;
732	}	718	}
@@ -795,13 +781,13 @@ int main(int argc, char **argv) · 62 unmodified lines
795	bool stress_mode = false;	781	bool stress_mode = false;
796	struct bpf_program *prog;	782	struct bpf_program *prog;
797	struct bpf_object *obj;	783	struct bpf_object *obj;
		784	int err = EXIT_FAIL;
798	char filename[256];	785	char filename[256];
799	int added_cpus = 0;	786	int added_cpus = 0;
800	int longindex = 0;	787	int longindex = 0;
801	int interval = 2;	788	int interval = 2;
802	int add_cpu = -1;	789	int add_cpu = -1;
803	int opt, err;	790	int opt, prog_fd;
804	int prog_fd;
805	int *cpu, i;	791	int *cpu, i;
806	__u32 qsize;	792	__u32 qsize;
807		793
@@ -824,24 +810,29 @@ int main(int argc, char **argv) · 16 unmodified lines
824	}	810	}
825		811
826	if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))	812	if (bpf_prog_load_xattr(&prog_load_attr, &obj, &prog_fd))
827	return EXIT_FAIL;	813	return err;
828		814
829	if (prog_fd < 0) {	815	if (prog_fd < 0) {
830	fprintf(stderr, "ERR: bpf_prog_load_xattr: %s\n",	816	fprintf(stderr, "ERR: bpf_prog_load_xattr: %s\n",
831	strerror(errno));	817	strerror(errno));
832	return EXIT_FAIL;	818	return err;
833	}	819	}
834	init_tracepoints(obj);	820
		821	if (init_tracepoints(obj) < 0) {
		822	fprintf(stderr, "ERR: bpf_program__attach failed\n");
		823	return err;
		824	}
		825
835	if (init_map_fds(obj) < 0) {	826	if (init_map_fds(obj) < 0) {
836	fprintf(stderr, "bpf_object__find_map_fd_by_name failed\n");	827	fprintf(stderr, "bpf_object__find_map_fd_by_name failed\n");
837	return EXIT_FAIL;	828	return err;
838	}	829	}
839	mark_cpus_unavailable();	830	mark_cpus_unavailable();
840		831
841	cpu = malloc(n_cpus * sizeof(int));	832	cpu = malloc(n_cpus * sizeof(int));
842	if (!cpu) {	833	if (!cpu) {
843	fprintf(stderr, "failed to allocate cpu array\n");	834	fprintf(stderr, "failed to allocate cpu array\n");
844	return EXIT_FAIL;	835	return err;
845	}	836	}
846	memset(cpu, 0, n_cpus * sizeof(int));	837	memset(cpu, 0, n_cpus * sizeof(int));
847		838
@@ -960,14 +951,12 @@ int main(int argc, char **argv) · 112 unmodified lines
960	prog = bpf_object__find_program_by_title(obj, prog_name);	951	prog = bpf_object__find_program_by_title(obj, prog_name);
961	if (!prog) {	952	if (!prog) {
962	fprintf(stderr, "bpf_object__find_program_by_title failed\n");	953	fprintf(stderr, "bpf_object__find_program_by_title failed\n");
963	err = EXIT_FAIL;
964	goto out;	954	goto out;
965	}	955	}
966		956
967	prog_fd = bpf_program__fd(prog);	957	prog_fd = bpf_program__fd(prog);
968	if (prog_fd < 0) {	958	if (prog_fd < 0) {
969	fprintf(stderr, "bpf_program__fd failed\n");	959	fprintf(stderr, "bpf_program__fd failed\n");
970	err = EXIT_FAIL;
971	goto out;	960	goto out;
972	}	961	}
973		962
@@ -986,6 +975,8 @@ int main(int argc, char **argv) · 12 unmodified lines
986		975
987	stats_poll(interval, use_separators, prog_name, mprog_name,	976	stats_poll(interval, use_separators, prog_name, mprog_name,
988	&value, stress_mode);	977	&value, stress_mode);
		978
		979	err = EXIT_OK;
989	out:	980	out:
990	free(cpu);	981	free(cpu);
991	return err;	982	return err;

samples/bpf/xdp_sample_pkts_kern.c+6−8

@@ -5,14 +5,12 @@ · 4 unmodified lines
5	#include <bpf/bpf_helpers.h>	5	#include <bpf/bpf_helpers.h>
6		6
7	#define SAMPLE_SIZE 64ul	7	#define SAMPLE_SIZE 64ul
8	#define MAX_CPUS 128	8
9		9	struct {
10	struct bpf_map_def SEC("maps") my_map = {	10	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
11	.type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,	11	__uint(key_size, sizeof(int));
12	.key_size = sizeof(int),	12	__uint(value_size, sizeof(u32));
13	.value_size = sizeof(u32),	13	} my_map SEC(".maps");
14	.max_entries = MAX_CPUS,
15	};
16		14
17	SEC("xdp_sample")	15	SEC("xdp_sample")
18	int xdp_sample_prog(struct xdp_md *ctx)	16	int xdp_sample_prog(struct xdp_md *ctx)

samples/bpf/xdp_sample_pkts_user.c+0−1

@@ -18,7 +18,6 @@ · 17 unmodified lines
18		18
19	#include "perf-sys.h"	19	#include "perf-sys.h"
20		20
21	#define MAX_CPUS 128
22	static int if_idx;	21	static int if_idx;
23	static char *if_name;	22	static char *if_name;
24	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;	23	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;

samples/bpf/xdpsock_user.c+290−64

 #include <linux/if_xdp.h>
 #include <linux/if_ether.h>
 #include <linux/ip.h>
+#include <linux/limits.h>
 #include <linux/udp.h>
 #include <arpa/inet.h>
 #include <locale.h>
 static u32 opt_pkt_fill_pattern = 0x12345678;
 static bool opt_extra_stats;
 static bool opt_quiet;
+static bool opt_app_stats;
+static const char *opt_irq_str = "";
+static u32 irq_no;
+static int irqs_at_init = -1;
 static int opt_poll;
 static int opt_interval = 1;
 static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP;
 static u32 opt_num_xsks = 1;
 static u32 prog_id;
-struct xsk_umem_info {
+struct xsk_ring_stats {
-	struct xsk_ring_prod fq;
-	struct xsk_ring_cons cq;
-	struct xsk_umem *umem;
-	void *buffer;
-};
-struct xsk_socket_info {
-	struct xsk_ring_cons rx;
-	struct xsk_ring_prod tx;
-	struct xsk_umem_info *umem;
-	struct xsk_socket *xsk;
 	unsigned long rx_npkts;
 	unsigned long tx_npkts;
 	unsigned long rx_dropped_npkts;
 	unsigned long prev_rx_full_npkts;
 	unsigned long prev_rx_fill_empty_npkts;
 	unsigned long prev_tx_empty_npkts;
+};
+struct xsk_driver_stats {
+	unsigned long intrs;
+	unsigned long prev_intrs;
+};
+struct xsk_app_stats {
+	unsigned long rx_empty_polls;
+	unsigned long fill_fail_polls;
+	unsigned long copy_tx_sendtos;
+	unsigned long tx_wakeup_sendtos;
+	unsigned long opt_polls;
+	unsigned long prev_rx_empty_polls;
+	unsigned long prev_fill_fail_polls;
+	unsigned long prev_copy_tx_sendtos;
+	unsigned long prev_tx_wakeup_sendtos;
+	unsigned long prev_opt_polls;
+};
+struct xsk_umem_info {
+	struct xsk_ring_prod fq;
+	struct xsk_ring_cons cq;
+	struct xsk_umem *umem;
+	void *buffer;
+};
+struct xsk_socket_info {
+	struct xsk_ring_cons rx;
+	struct xsk_ring_prod tx;
+	struct xsk_umem_info *umem;
+	struct xsk_socket *xsk;
+	struct xsk_ring_stats ring_stats;
+	struct xsk_app_stats app_stats;
+	struct xsk_driver_stats drv_stats;
 	u32 outstanding_tx;
 };
 		return err;
 	if (optlen == sizeof(struct xdp_statistics)) {
-		xsk->rx_dropped_npkts = stats.rx_dropped;
+		xsk->ring_stats.rx_dropped_npkts = stats.rx_dropped;
-		xsk->rx_invalid_npkts = stats.rx_invalid_descs;
+		xsk->ring_stats.rx_invalid_npkts = stats.rx_invalid_descs;
-		xsk->tx_invalid_npkts = stats.tx_invalid_descs;
+		xsk->ring_stats.tx_invalid_npkts = stats.tx_invalid_descs;
-		xsk->rx_full_npkts = stats.rx_ring_full;
+		xsk->ring_stats.rx_full_npkts = stats.rx_ring_full;
-		xsk->rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs;
+		xsk->ring_stats.rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs;
-		xsk->tx_empty_npkts = stats.tx_ring_empty_descs;
+		xsk->ring_stats.tx_empty_npkts = stats.tx_ring_empty_descs;
 		return 0;
 	}
 	return -EINVAL;
 }
+static void dump_app_stats(long dt)
+{
+	int i;
+	for (i = 0; i < num_socks && xsks[i]; i++) {
+		char *fmt = "%-18s %'-14.0f %'-14lu\n";
+		double rx_empty_polls_ps, fill_fail_polls_ps, copy_tx_sendtos_ps,
+				tx_wakeup_sendtos_ps, opt_polls_ps;
+		rx_empty_polls_ps = (xsks[i]->app_stats.rx_empty_polls -
+					xsks[i]->app_stats.prev_rx_empty_polls) * 1000000000. / dt;
+		fill_fail_polls_ps = (xsks[i]->app_stats.fill_fail_polls -
+					xsks[i]->app_stats.prev_fill_fail_polls) * 1000000000. / dt;
+		copy_tx_sendtos_ps = (xsks[i]->app_stats.copy_tx_sendtos -
+					xsks[i]->app_stats.prev_copy_tx_sendtos) * 1000000000. / dt;
+		tx_wakeup_sendtos_ps = (xsks[i]->app_stats.tx_wakeup_sendtos -
+					xsks[i]->app_stats.prev_tx_wakeup_sendtos)
+										* 1000000000. / dt;
+		opt_polls_ps = (xsks[i]->app_stats.opt_polls -
+					xsks[i]->app_stats.prev_opt_polls) * 1000000000. / dt;
+		printf("\n%-18s %-14s %-14s\n", "", "calls/s", "count");
+		printf(fmt, "rx empty polls", rx_empty_polls_ps, xsks[i]->app_stats.rx_empty_polls);
+		printf(fmt, "fill fail polls", fill_fail_polls_ps,
+							xsks[i]->app_stats.fill_fail_polls);
+		printf(fmt, "copy tx sendtos", copy_tx_sendtos_ps,
+							xsks[i]->app_stats.copy_tx_sendtos);
+		printf(fmt, "tx wakeup sendtos", tx_wakeup_sendtos_ps,
+							xsks[i]->app_stats.tx_wakeup_sendtos);
+		printf(fmt, "opt polls", opt_polls_ps, xsks[i]->app_stats.opt_polls);
+		xsks[i]->app_stats.prev_rx_empty_polls = xsks[i]->app_stats.rx_empty_polls;
+		xsks[i]->app_stats.prev_fill_fail_polls = xsks[i]->app_stats.fill_fail_polls;
+		xsks[i]->app_stats.prev_copy_tx_sendtos = xsks[i]->app_stats.copy_tx_sendtos;
+		xsks[i]->app_stats.prev_tx_wakeup_sendtos = xsks[i]->app_stats.tx_wakeup_sendtos;
+		xsks[i]->app_stats.prev_opt_polls = xsks[i]->app_stats.opt_polls;
+	}
+}
+static bool get_interrupt_number(void)
+{
+	FILE *f_int_proc;
+	char line[4096];
+	bool found = false;
+	f_int_proc = fopen("/proc/interrupts", "r");
+	if (f_int_proc == NULL) {
+		printf("Failed to open /proc/interrupts.\n");
+		return found;
+	}
+	while (!feof(f_int_proc) && !found) {
+		/* Make sure to read a full line at a time */
+		if (fgets(line, sizeof(line), f_int_proc) == NULL ||
+				line[strlen(line) - 1] != '\n') {
+			printf("Error reading from interrupts file\n");
+			break;
+		}
+		/* Extract interrupt number from line */
+		if (strstr(line, opt_irq_str) != NULL) {
+			irq_no = atoi(line);
+			found = true;
+			break;
+		}
+	}
+	fclose(f_int_proc);
+	return found;
+}
+static int get_irqs(void)
+{
+	char count_path[PATH_MAX];
+	int total_intrs = -1;
+	FILE *f_count_proc;
+	char line[4096];
+	snprintf(count_path, sizeof(count_path),
+		"/sys/kernel/irq/%i/per_cpu_count", irq_no);
+	f_count_proc = fopen(count_path, "r");
+	if (f_count_proc == NULL) {
+		printf("Failed to open %s\n", count_path);
+		return total_intrs;
+	}
+	if (fgets(line, sizeof(line), f_count_proc) == NULL ||
+			line[strlen(line) - 1] != '\n') {
+		printf("Error reading from %s\n", count_path);
+	} else {
+		static const char com[2] = ",";
+		char *token;
+		total_intrs = 0;
+		token = strtok(line, com);
+		while (token != NULL) {
+			/* sum up interrupts across all cores */
+			total_intrs += atoi(token);
+			token = strtok(NULL, com);
+		}
+	}
+	fclose(f_count_proc);
+	return total_intrs;
+}
+static void dump_driver_stats(long dt)
+{
+	int i;
+	for (i = 0; i < num_socks && xsks[i]; i++) {
+		char *fmt = "%-18s %'-14.0f %'-14lu\n";
+		double intrs_ps;
+		int n_ints = get_irqs();
+		if (n_ints < 0) {
+			printf("error getting intr info for intr %i\n", irq_no);
+			return;
+		}
+		xsks[i]->drv_stats.intrs = n_ints - irqs_at_init;
+		intrs_ps = (xsks[i]->drv_stats.intrs - xsks[i]->drv_stats.prev_intrs) *
+			 1000000000. / dt;
+		printf("\n%-18s %-14s %-14s\n", "", "intrs/s", "count");
+		printf(fmt, "irqs", intrs_ps, xsks[i]->drv_stats.intrs);
+		xsks[i]->drv_stats.prev_intrs = xsks[i]->drv_stats.intrs;
+	}
+}
 static void dump_stats(void)
 {
 	unsigned long now = get_nsecs();
 	prev_time = now;
 	for (i = 0; i < num_socks && xsks[i]; i++) {
-		char *fmt = "%-15s %'-11.0f %'-11lu\n";
+		char *fmt = "%-18s %'-14.0f %'-14lu\n";
 		double rx_pps, tx_pps, dropped_pps, rx_invalid_pps, full_pps, fill_empty_pps,
 			tx_invalid_pps, tx_empty_pps;
-		rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) *
+		rx_pps = (xsks[i]->ring_stats.rx_npkts - xsks[i]->ring_stats.prev_rx_npkts) *
 			 1000000000. / dt;
-		tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) *
+		tx_pps = (xsks[i]->ring_stats.tx_npkts - xsks[i]->ring_stats.prev_tx_npkts) *
 			 1000000000. / dt;
 		printf("\n sock%d@", i);
 		print_benchmark(false);
 		printf("\n");
-		printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
+		printf("%-18s %-14s %-14s %-14.2f\n", "", "pps", "pkts",
 		       dt / 1000000000.);
-		printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts);
+		printf(fmt, "rx", rx_pps, xsks[i]->ring_stats.rx_npkts);
-		printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts);
+		printf(fmt, "tx", tx_pps, xsks[i]->ring_stats.tx_npkts);
-		xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts;
+		xsks[i]->ring_stats.prev_rx_npkts = xsks[i]->ring_stats.rx_npkts;
-		xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts;
+		xsks[i]->ring_stats.prev_tx_npkts = xsks[i]->ring_stats.tx_npkts;
 		if (opt_extra_stats) {
 			if (!xsk_get_xdp_stats(xsk_socket__fd(xsks[i]->xsk), xsks[i])) {
-				dropped_pps = (xsks[i]->rx_dropped_npkts -
+				dropped_pps = (xsks[i]->ring_stats.rx_dropped_npkts -
-						xsks[i]->prev_rx_dropped_npkts) * 1000000000. / dt;
+						xsks[i]->ring_stats.prev_rx_dropped_npkts) *
-				rx_invalid_pps = (xsks[i]->rx_invalid_npkts -
+							1000000000. / dt;
-						xsks[i]->prev_rx_invalid_npkts) * 1000000000. / dt;
+				rx_invalid_pps = (xsks[i]->ring_stats.rx_invalid_npkts -
-				tx_invalid_pps = (xsks[i]->tx_invalid_npkts -
+						xsks[i]->ring_stats.prev_rx_invalid_npkts) *
-						xsks[i]->prev_tx_invalid_npkts) * 1000000000. / dt;
+							1000000000. / dt;
-				full_pps = (xsks[i]->rx_full_npkts -
+				tx_invalid_pps = (xsks[i]->ring_stats.tx_invalid_npkts -
-						xsks[i]->prev_rx_full_npkts) * 1000000000. / dt;
+						xsks[i]->ring_stats.prev_tx_invalid_npkts) *
-				fill_empty_pps = (xsks[i]->rx_fill_empty_npkts -
+							1000000000. / dt;
-						xsks[i]->prev_rx_fill_empty_npkts)
+				full_pps = (xsks[i]->ring_stats.rx_full_npkts -
-						* 1000000000. / dt;
+						xsks[i]->ring_stats.prev_rx_full_npkts) *
-				tx_empty_pps = (xsks[i]->tx_empty_npkts -
+							1000000000. / dt;
-						xsks[i]->prev_tx_empty_npkts) * 1000000000. / dt;
+				fill_empty_pps = (xsks[i]->ring_stats.rx_fill_empty_npkts -
+						xsks[i]->ring_stats.prev_rx_fill_empty_npkts) *
+							1000000000. / dt;
+				tx_empty_pps = (xsks[i]->ring_stats.tx_empty_npkts -
+						xsks[i]->ring_stats.prev_tx_empty_npkts) *
+							1000000000. / dt;
 				printf(fmt, "rx dropped", dropped_pps,
-				       xsks[i]->rx_dropped_npkts);
+				       xsks[i]->ring_stats.rx_dropped_npkts);
 				printf(fmt, "rx invalid", rx_invalid_pps,
-				       xsks[i]->rx_invalid_npkts);
+				       xsks[i]->ring_stats.rx_invalid_npkts);
 				printf(fmt, "tx invalid", tx_invalid_pps,
-				       xsks[i]->tx_invalid_npkts);
+				       xsks[i]->ring_stats.tx_invalid_npkts);
 				printf(fmt, "rx queue full", full_pps,
-				       xsks[i]->rx_full_npkts);
+				       xsks[i]->ring_stats.rx_full_npkts);
 				printf(fmt, "fill ring empty", fill_empty_pps,
-				       xsks[i]->rx_fill_empty_npkts);
+				       xsks[i]->ring_stats.rx_fill_empty_npkts);
 				printf(fmt, "tx ring empty", tx_empty_pps,
-				       xsks[i]->tx_empty_npkts);
+				       xsks[i]->ring_stats.tx_empty_npkts);
-				xsks[i]->prev_rx_dropped_npkts = xsks[i]->rx_dropped_npkts;
+				xsks[i]->ring_stats.prev_rx_dropped_npkts =
-				xsks[i]->prev_rx_invalid_npkts = xsks[i]->rx_invalid_npkts;
+					xsks[i]->ring_stats.rx_dropped_npkts;
-				xsks[i]->prev_tx_invalid_npkts = xsks[i]->tx_invalid_npkts;
+				xsks[i]->ring_stats.prev_rx_invalid_npkts =
-				xsks[i]->prev_rx_full_npkts = xsks[i]->rx_full_npkts;
+					xsks[i]->ring_stats.rx_invalid_npkts;
-				xsks[i]->prev_rx_fill_empty_npkts = xsks[i]->rx_fill_empty_npkts;
+				xsks[i]->ring_stats.prev_tx_invalid_npkts =
-				xsks[i]->prev_tx_empty_npkts = xsks[i]->tx_empty_npkts;
+					xsks[i]->ring_stats.tx_invalid_npkts;
+				xsks[i]->ring_stats.prev_rx_full_npkts =
+					xsks[i]->ring_stats.rx_full_npkts;
+				xsks[i]->ring_stats.prev_rx_fill_empty_npkts =
+					xsks[i]->ring_stats.rx_fill_empty_npkts;
+				xsks[i]->ring_stats.prev_tx_empty_npkts =
+					xsks[i]->ring_stats.tx_empty_npkts;
 			} else {
 				printf("%-15s\n", "Error retrieving extra stats");
 			}
 		}
 	}
+	if (opt_app_stats)
+		dump_app_stats(dt);
+	if (irq_no)
+		dump_driver_stats(dt);
 }
 static bool is_benchmark_done(void)
 	if (ret)
 		exit_with_error(-ret);
+	xsk->app_stats.rx_empty_polls = 0;
+	xsk->app_stats.fill_fail_polls = 0;
+	xsk->app_stats.copy_tx_sendtos = 0;
+	xsk->app_stats.tx_wakeup_sendtos = 0;
+	xsk->app_stats.opt_polls = 0;
+	xsk->app_stats.prev_rx_empty_polls = 0;
+	xsk->app_stats.prev_fill_fail_polls = 0;
+	xsk->app_stats.prev_copy_tx_sendtos = 0;
+	xsk->app_stats.prev_tx_wakeup_sendtos = 0;
+	xsk->app_stats.prev_opt_polls = 0;
 	return xsk;
 }
 	{"tx-pkt-pattern", required_argument, 0, 'P'},
 	{"extra-stats", no_argument, 0, 'x'},
 	{"quiet", no_argument, 0, 'Q'},
+	{"app-stats", no_argument, 0, 'a'},
+	{"irq-string", no_argument, 0, 'I'},
 	{0, 0, 0, 0}
 };
 		"  -P, --tx-pkt-pattern=nPacket fill pattern. Default: 0x%x\n"
 		"  -x, --extra-stats	Display extra statistics.\n"
 		"  -Q, --quiet          Do not display any stats.\n"
+		"  -a, --app-stats	Display application (syscall) statistics.\n"
+		"  -I, --irq-string	Display driver interrupt statistics for interface associated with irq-string.\n"
 		"\n";
 	fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE,
 		opt_batch_size, MIN_PKT_SIZE, MIN_PKT_SIZE,
 	opterr = 0;
 	for (;;) {
-		c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:xQ",
+		c = getopt_long(argc, argv, "Frtli:q:pSNn:czf:muMd:b:C:s:P:xQaI:",
 				long_options, &option_index);
 		if (c == -1)
 			break;
 			break;
 		case 'Q':
 			opt_quiet = 1;
+			break;
+		case 'a':
+			opt_app_stats = 1;
+			break;
+		case 'I':
+			opt_irq_str = optarg;
+			if (get_interrupt_number())
+				irqs_at_init = get_irqs();
+			if (irqs_at_init < 0) {
+				fprintf(stderr, "ERROR: Failed to get irqs for %s\n", opt_irq_str);
+				usage(basename(argv[0]));
+			}
 			break;
 		default:
 			usage(basename(argv[0]));
 	 * is driven by the NAPI loop. So as an optimization, we do not have to call
 	 * sendto() all the time in zero-copy mode for l2fwd.
 	 */
-	if (opt_xdp_bind_flags & XDP_COPY)
+	if (opt_xdp_bind_flags & XDP_COPY) {
+		xsk->app_stats.copy_tx_sendtos++;
 		kick_tx(xsk);
+	}
 	ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size :
 		xsk->outstanding_tx;
 		while (ret != rcvd) {
 			if (ret < 0)
 				exit_with_error(-ret);
-			if (xsk_ring_prod__needs_wakeup(&umem->fq))
+			if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
+				xsk->app_stats.fill_fail_polls++;
 				ret = poll(fds, num_socks, opt_timeout);
+			}
 			ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
 		}
 		xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
 		xsk_ring_cons__release(&xsk->umem->cq, rcvd);
 		xsk->outstanding_tx -= rcvd;
-		xsk->tx_npkts += rcvd;
+		xsk->ring_stats.tx_npkts += rcvd;
 	}
 }
 	if (!xsk->outstanding_tx)
 		return;
-	if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx))
+	if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) {
+		xsk->app_stats.tx_wakeup_sendtos++;
 		kick_tx(xsk);
+	}
 	rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
 	if (rcvd > 0) {
 		xsk_ring_cons__release(&xsk->umem->cq, rcvd);
 		xsk->outstanding_tx -= rcvd;
-		xsk->tx_npkts += rcvd;
+		xsk->ring_stats.tx_npkts += rcvd;
 	}
 }
 	rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);
 	if (!rcvd) {
-		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
+		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
+			xsk->app_stats.rx_empty_polls++;
 			ret = poll(fds, num_socks, opt_timeout);
+		}
 		return;
 	}
 	while (ret != rcvd) {
 		if (ret < 0)
 			exit_with_error(-ret);
-		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
+		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
+			xsk->app_stats.fill_fail_polls++;
 			ret = poll(fds, num_socks, opt_timeout);
+		}
 		ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
 	}
 	xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
 	xsk_ring_cons__release(&xsk->rx, rcvd);
-	xsk->rx_npkts += rcvd;
+	xsk->ring_stats.rx_npkts += rcvd;
 }
 static void rx_drop_all(void)
 	for (;;) {
 		if (opt_poll) {
+			for (i = 0; i < num_socks; i++)
+				xsks[i]->app_stats.opt_polls++;
 			ret = poll(fds, num_socks, opt_timeout);
 			if (ret <= 0)
 				continue;
 		int batch_size = get_batch_size(pkt_cnt);
 		if (opt_poll) {
+			for (i = 0; i < num_socks; i++)
+				xsks[i]->app_stats.opt_polls++;
 			ret = poll(fds, num_socks, opt_timeout);
 			if (ret <= 0)
 				continue;
 	rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx);
 	if (!rcvd) {
-		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
+		if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) {
+			xsk->app_stats.rx_empty_polls++;
 			ret = poll(fds, num_socks, opt_timeout);
+		}
 		return;
 	}
 		if (ret < 0)
 			exit_with_error(-ret);
 		complete_tx_l2fwd(xsk, fds);
-		if (xsk_ring_prod__needs_wakeup(&xsk->tx))
+		if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
+			xsk->app_stats.tx_wakeup_sendtos++;
 			kick_tx(xsk);
+		}
 		ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
 	}
 	xsk_ring_prod__submit(&xsk->tx, rcvd);
 	xsk_ring_cons__release(&xsk->rx, rcvd);
-	xsk->rx_npkts += rcvd;
+	xsk->ring_stats.rx_npkts += rcvd;
 	xsk->outstanding_tx += rcvd;
 }
 	for (;;) {
 		if (opt_poll) {
+			for (i = 0; i < num_socks; i++)
+				xsks[i]->app_stats.opt_polls++;
 			ret = poll(fds, num_socks, opt_timeout);
 			if (ret <= 0)
 				continue;

tools/include/uapi/linux/bpf.h+87−14

@@ -356,18 +356,36 @@ enum bpf_link_type { · 355 unmodified lines
356	#define BPF_F_SLEEPABLE (1U << 4)	356	#define BPF_F_SLEEPABLE (1U << 4)
357		357
358	/* When BPF ldimm64's insn[0].src_reg != 0 then this can have	358	/* When BPF ldimm64's insn[0].src_reg != 0 then this can have
359	* two extensions:	359	* the following extensions:
360	*	360	*
361	* insn[0].src_reg: BPF_PSEUDO_MAP_FD BPF_PSEUDO_MAP_VALUE	361	* insn[0].src_reg: BPF_PSEUDO_MAP_FD
362	* insn[0].imm: map fd map fd	362	* insn[0].imm: map fd
363	* insn[1].imm: 0 offset into value	363	* insn[1].imm: 0
364	* insn[0].off: 0 0	364	* insn[0].off: 0
365	* insn[1].off: 0 0	365	* insn[1].off: 0
366	* ldimm64 rewrite: address of map address of map[0]+offset	366	* ldimm64 rewrite: address of map
367	* verifier type: CONST_PTR_TO_MAP PTR_TO_MAP_VALUE	367	* verifier type: CONST_PTR_TO_MAP
368	*/	368	*/
369	#define BPF_PSEUDO_MAP_FD 1	369	#define BPF_PSEUDO_MAP_FD 1
		370	/* insn[0].src_reg: BPF_PSEUDO_MAP_VALUE
		371	* insn[0].imm: map fd
		372	* insn[1].imm: offset into value
		373	* insn[0].off: 0
		374	* insn[1].off: 0
		375	* ldimm64 rewrite: address of map[0]+offset
		376	* verifier type: PTR_TO_MAP_VALUE
		377	*/
370	#define BPF_PSEUDO_MAP_VALUE 2	378	#define BPF_PSEUDO_MAP_VALUE 2
		379	/* insn[0].src_reg: BPF_PSEUDO_BTF_ID
		380	* insn[0].imm: kernel btd id of VAR
		381	* insn[1].imm: 0
		382	* insn[0].off: 0
		383	* insn[1].off: 0
		384	* ldimm64 rewrite: address of the kernel variable
		385	* verifier type: PTR_TO_BTF_ID or PTR_TO_MEM, depending on whether the var
		386	* is struct/union.
		387	*/
		388	#define BPF_PSEUDO_BTF_ID 3
371		389
372	/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative	390	/* when bpf_call->src_reg == BPF_PSEUDO_CALL, bpf_call->imm == pc-relative
373	* offset to another bpf function	391	* offset to another bpf function
@@ -417,6 +435,9 @@ enum { · 43 unmodified lines
417		435
418	/* Share perf_event among processes */	436	/* Share perf_event among processes */
419	BPF_F_PRESERVE_ELEMS = (1U << 11),	437	BPF_F_PRESERVE_ELEMS = (1U << 11),
		438
		439	/* Create a map that is suitable to be an inner map with dynamic max entries */
		440	BPF_F_INNER_MAP = (1U << 12),
420	};	441	};
421		442
422	/* Flags for BPF_PROG_QUERY. */	443	/* Flags for BPF_PROG_QUERY. */
@@ -1680,7 +1701,7 @@ union bpf_attr { · 1257 unmodified lines
1680	* TCP_CONGESTION, TCP_BPF_IW,	1701	* TCP_CONGESTION, TCP_BPF_IW,
1681	* TCP_BPF_SNDCWND_CLAMP, TCP_SAVE_SYN,	1702	* TCP_BPF_SNDCWND_CLAMP, TCP_SAVE_SYN,
1682	* TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT,	1703	* TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT,
1683	* TCP_SYNCNT, TCP_USER_TIMEOUT.	1704	* TCP_SYNCNT, TCP_USER_TIMEOUT, TCP_NOTSENT_LOWAT.
1684	* * IPPROTO_IP, which supports optname IP_TOS.	1705	* * IPPROTO_IP, which supports optname IP_TOS.
1685	* * IPPROTO_IPV6, which supports optname IPV6_TCLASS.	1706	* * IPPROTO_IPV6, which supports optname IPV6_TCLASS.
1686	* Return	1707	* Return
@@ -2235,7 +2256,7 @@ union bpf_attr { · 548 unmodified lines
2235	* Description	2256	* Description
2236	* This helper is used in programs implementing policies at the	2257	* This helper is used in programs implementing policies at the
2237	* skb socket level. If the sk_buff skb is allowed to pass (i.e.	2258	* skb socket level. If the sk_buff skb is allowed to pass (i.e.
2238	* if the verdeict eBPF program returns SK_PASS), redirect it	2259	* if the verdict eBPF program returns SK_PASS), redirect it
2239	* to the socket referenced by map (of type	2260	* to the socket referenced by map (of type
2240	* BPF_MAP_TYPE_SOCKHASH) using hash key. Both ingress and	2261	* BPF_MAP_TYPE_SOCKHASH) using hash key. Both ingress and
2241	* egress interfaces can be used for redirection. The	2262	* egress interfaces can be used for redirection. The
@@ -3661,10 +3682,59 @@ union bpf_attr { · 1419 unmodified lines
3661	* Redirect the packet to another net device of index ifindex	3682	* Redirect the packet to another net device of index ifindex
3662	* and fill in L2 addresses from neighboring subsystem. This helper	3683	* and fill in L2 addresses from neighboring subsystem. This helper
3663	* is somewhat similar to bpf_redirect\ (), except that it	3684	* is somewhat similar to bpf_redirect\ (), except that it
3664	* fills in e.g. MAC addresses based on the L3 information from	3685	* populates L2 addresses as well, meaning, internally, the helper
3665	* the packet. This helper is supported for IPv4 and IPv6 protocols.	3686	* performs a FIB lookup based on the skb's networking header to
		3687	* get the address of the next hop and then relies on the neighbor
		3688	* lookup for the L2 address of the nexthop.
		3689	*
		3690	* The flags argument is reserved and must be 0. The helper is
		3691	* currently only supported for tc BPF program types, and enabled
		3692	* for IPv4 and IPv6 protocols.
		3693	* Return
		3694	* The helper returns TC_ACT_REDIRECT on success or
		3695	* TC_ACT_SHOT on error.
		3696	*
		3697	* void bpf_per_cpu_ptr(const void percpu_ptr, u32 cpu)
		3698	* Description
		3699	* Take a pointer to a percpu ksym, percpu_ptr, and return a
		3700	* pointer to the percpu kernel variable on cpu. A ksym is an
		3701	* extern variable decorated with '__ksym'. For ksym, there is a
		3702	* global var (either static or global) defined of the same name
		3703	* in the kernel. The ksym is percpu if the global var is percpu.
		3704	* The returned pointer points to the global percpu var on cpu.
		3705	*
		3706	* bpf_per_cpu_ptr() has the same semantic as per_cpu_ptr() in the
		3707	* kernel, except that bpf_per_cpu_ptr() may return NULL. This
		3708	* happens if cpu is larger than nr_cpu_ids. The caller of
		3709	* bpf_per_cpu_ptr() must check the returned value.
		3710	* Return
		3711	* A pointer pointing to the kernel percpu variable on cpu, or
		3712	* NULL, if cpu is invalid.
		3713	*
		3714	* void bpf_this_cpu_ptr(const void percpu_ptr)
		3715	* Description
		3716	* Take a pointer to a percpu ksym, percpu_ptr, and return a
		3717	* pointer to the percpu kernel variable on this cpu. See the
		3718	* description of 'ksym' in bpf_per_cpu_ptr\ ().
		3719	*
		3720	* bpf_this_cpu_ptr() has the same semantic as this_cpu_ptr() in
		3721	* the kernel. Different from bpf_per_cpu_ptr\ (), it would
		3722	* never return NULL.
		3723	* Return
		3724	* A pointer pointing to the kernel percpu variable on this cpu.
		3725	*
		3726	* long bpf_redirect_peer(u32 ifindex, u64 flags)
		3727	* Description
		3728	* Redirect the packet to another net device of index ifindex.
		3729	* This helper is somewhat similar to bpf_redirect\ (), except
		3730	* that the redirection happens to the ifindex' peer device and
		3731	* the netns switch takes place from ingress to ingress without
		3732	* going through the CPU's backlog queue.
		3733	*
3666	* The flags argument is reserved and must be 0. The helper is	3734	* The flags argument is reserved and must be 0. The helper is
3667	* currently only supported for tc BPF program types.	3735	* currently only supported for tc BPF program types at the ingress
		3736	* hook and for veth device types. The peer device must reside in a
		3737	* different network namespace.
3668	* Return	3738	* Return
3669	* The helper returns TC_ACT_REDIRECT on success or	3739	* The helper returns TC_ACT_REDIRECT on success or
3670	* TC_ACT_SHOT on error.	3740	* TC_ACT_SHOT on error.
@@ -3823,6 +3893,9 @@ union bpf_attr { · 152 unmodified lines
3823	FN(seq_printf_btf), \	3893	FN(seq_printf_btf), \
3824	FN(skb_cgroup_classid), \	3894	FN(skb_cgroup_classid), \
3825	FN(redirect_neigh), \	3895	FN(redirect_neigh), \
		3896	FN(bpf_per_cpu_ptr), \
		3897	FN(bpf_this_cpu_ptr), \
		3898	FN(redirect_peer), \
3826	/* */	3899	/* */
3827		3900
3828	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	3901	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

tools/lib/bpf/libbpf.c+289−59

@@ -390,6 +390,12 @@ struct extern_desc { · 389 unmodified lines
390	} kcfg;	390	} kcfg;
391	struct {	391	struct {
392	unsigned long long addr;	392	unsigned long long addr;
		393
		394	/* target btf_id of the corresponding kernel var. */
		395	int vmlinux_btf_id;
		396
		397	/* local btf_id of the ksym extern's type. */
		398	__u32 type_id;
393	} ksym;	399	} ksym;
394	};	400	};
395	};	401	};
@@ -2522,12 +2528,23 @@ static int bpf_object__load_vmlinux_btf(struct bpf_object *obj) · 2126 unmodified lines
2522	{	2528	{
2523	bool need_vmlinux_btf = false;	2529	bool need_vmlinux_btf = false;
2524	struct bpf_program *prog;	2530	struct bpf_program *prog;
2525	int err;	2531	int i, err;
2526		2532
2527	/* CO-RE relocations need kernel BTF */	2533	/* CO-RE relocations need kernel BTF */
2528	if (obj->btf_ext && obj->btf_ext->core_relo_info.len)	2534	if (obj->btf_ext && obj->btf_ext->core_relo_info.len)
2529	need_vmlinux_btf = true;	2535	need_vmlinux_btf = true;
2530		2536
		2537	/* Support for typed ksyms needs kernel BTF */
		2538	for (i = 0; i < obj->nr_extern; i++) {
		2539	const struct extern_desc *ext;
		2540
		2541	ext = &obj->externs[i];
		2542	if (ext->type == EXT_KSYM && ext->ksym.type_id) {
		2543	need_vmlinux_btf = true;
		2544	break;
		2545	}
		2546	}
		2547
2531	bpf_object__for_each_program(prog, obj) {	2548	bpf_object__for_each_program(prog, obj) {
2532	if (!prog->load)	2549	if (!prog->load)
2533	continue;	2550	continue;
@@ -3156,16 +3173,10 @@ static int bpf_object__collect_externs(struct bpf_object *obj) · 622 unmodified lines
3156	return -ENOTSUP;	3173	return -ENOTSUP;
3157	}	3174	}
3158	} else if (strcmp(sec_name, KSYMS_SEC) == 0) {	3175	} else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3159	const struct btf_type *vt;
3160
3161	ksym_sec = sec;	3176	ksym_sec = sec;
3162	ext->type = EXT_KSYM;	3177	ext->type = EXT_KSYM;
3163		3178	skip_mods_and_typedefs(obj->btf, t->type,
3164	vt = skip_mods_and_typedefs(obj->btf, t->type, NULL);	3179	&ext->ksym.type_id);
3165	if (!btf_is_void(vt)) {
3166	pr_warn("extern (ksym) '%s' is not typeless (void)\n", ext_name);
3167	return -ENOTSUP;
3168	}
3169	} else {	3180	} else {
3170	pr_warn("unrecognized extern section '%s'\n", sec_name);	3181	pr_warn("unrecognized extern section '%s'\n", sec_name);
3171	return -ENOTSUP;	3182	return -ENOTSUP;
@@ -4192,6 +4203,36 @@ static int bpf_object__create_map(struct bpf_object obj, struct bpf_map map) · 1020 unmodified lines
4192	return 0;	4203	return 0;
4193	}	4204	}
4194		4205
		4206	static int init_map_slots(struct bpf_map *map)
		4207	{
		4208	const struct bpf_map *targ_map;
		4209	unsigned int i;
		4210	int fd, err;
		4211
		4212	for (i = 0; i < map->init_slots_sz; i++) {
		4213	if (!map->init_slots[i])
		4214	continue;
		4215
		4216	targ_map = map->init_slots[i];
		4217	fd = bpf_map__fd(targ_map);
		4218	err = bpf_map_update_elem(map->fd, &i, &fd, 0);
		4219	if (err) {
		4220	err = -errno;
		4221	pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
		4222	map->name, i, targ_map->name,
		4223	fd, err);
		4224	return err;
		4225	}
		4226	pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
		4227	map->name, i, targ_map->name, fd);
		4228	}
		4229
		4230	zfree(&map->init_slots);
		4231	map->init_slots_sz = 0;
		4232
		4233	return 0;
		4234	}
		4235
4195	static int	4236	static int
4196	bpf_object__create_maps(struct bpf_object *obj)	4237	bpf_object__create_maps(struct bpf_object *obj)
4197	{	4238	{
@@ -4215,47 +4256,29 @@ bpf_object__create_maps(struct bpf_object *obj) · 17 unmodified lines
4215	if (map->fd >= 0) {	4256	if (map->fd >= 0) {
4216	pr_debug("map '%s': skipping creation (preset fd=%d)\n",	4257	pr_debug("map '%s': skipping creation (preset fd=%d)\n",
4217	map->name, map->fd);	4258	map->name, map->fd);
4218	continue;	4259	} else {
4219	}	4260	err = bpf_object__create_map(obj, map);
4220		4261	if (err)
4221	err = bpf_object__create_map(obj, map);
4222	if (err)
4223	goto err_out;
4224
4225	pr_debug("map '%s': created successfully, fd=%d\n", map->name,
4226	map->fd);
4227
4228	if (bpf_map__is_internal(map)) {
4229	err = bpf_object__populate_internal_map(obj, map);
4230	if (err < 0) {
4231	zclose(map->fd);
4232	goto err_out;	4262	goto err_out;
4233	}
4234	}
4235		4263
4236	if (map->init_slots_sz) {	4264	pr_debug("map '%s': created successfully, fd=%d\n",
4237	for (j = 0; j < map->init_slots_sz; j++) {	4265	map->name, map->fd);
4238	const struct bpf_map *targ_map;
4239	int fd;
4240		4266
4241	if (!map->init_slots[j])	4267	if (bpf_map__is_internal(map)) {
4242	continue;	4268	err = bpf_object__populate_internal_map(obj, map);
		4269	if (err < 0) {
		4270	zclose(map->fd);
		4271	goto err_out;
		4272	}
		4273	}
4243		4274
4244	targ_map = map->init_slots[j];	4275	if (map->init_slots_sz) {
4245	fd = bpf_map__fd(targ_map);	4276	err = init_map_slots(map);
4246	err = bpf_map_update_elem(map->fd, &j, &fd, 0);	4277	if (err < 0) {
4247	if (err) {	4278	zclose(map->fd);
4248	err = -errno;
4249	pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n",
4250	map->name, j, targ_map->name,
4251	fd, err);
4252	goto err_out;	4279	goto err_out;
4253	}	4280	}
4254	pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n",
4255	map->name, j, targ_map->name, fd);
4256	}	4281	}
4257	zfree(&map->init_slots);
4258	map->init_slots_sz = 0;
4259	}	4282	}
4260		4283
4261	if (map->pin_path && !map->pinned) {	4284	if (map->pin_path && !map->pinned) {
@@ -5017,16 +5040,19 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, · 755 unmodified lines
5017	static int bpf_core_calc_field_relo(const struct bpf_program *prog,	5040	static int bpf_core_calc_field_relo(const struct bpf_program *prog,
5018	const struct bpf_core_relo *relo,	5041	const struct bpf_core_relo *relo,
5019	const struct bpf_core_spec *spec,	5042	const struct bpf_core_spec *spec,
5020	__u32 val, bool validate)	5043	__u32 val, __u32 field_sz, __u32 *type_id,
		5044	bool *validate)
5021	{	5045	{
5022	const struct bpf_core_accessor *acc;	5046	const struct bpf_core_accessor *acc;
5023	const struct btf_type *t;	5047	const struct btf_type *t;
5024	__u32 byte_off, byte_sz, bit_off, bit_sz;	5048	__u32 byte_off, byte_sz, bit_off, bit_sz, field_type_id;
5025	const struct btf_member *m;	5049	const struct btf_member *m;
5026	const struct btf_type *mt;	5050	const struct btf_type *mt;
5027	bool bitfield;	5051	bool bitfield;
5028	__s64 sz;	5052	__s64 sz;
5029		5053
		5054	*field_sz = 0;
		5055
5030	if (relo->kind == BPF_FIELD_EXISTS) {	5056	if (relo->kind == BPF_FIELD_EXISTS) {
5031	*val = spec ? 1 : 0;	5057	*val = spec ? 1 : 0;
5032	return 0;	5058	return 0;
@@ -5042,6 +5068,12 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 9 unmodified lines
5042	if (!acc->name) {	5068	if (!acc->name) {
5043	if (relo->kind == BPF_FIELD_BYTE_OFFSET) {	5069	if (relo->kind == BPF_FIELD_BYTE_OFFSET) {
5044	*val = spec->bit_offset / 8;	5070	*val = spec->bit_offset / 8;
		5071	/* remember field size for load/store mem size */
		5072	sz = btf__resolve_size(spec->btf, acc->type_id);
		5073	if (sz < 0)
		5074	return -EINVAL;
		5075	*field_sz = sz;
		5076	*type_id = acc->type_id;
5045	} else if (relo->kind == BPF_FIELD_BYTE_SIZE) {	5077	} else if (relo->kind == BPF_FIELD_BYTE_SIZE) {
5046	sz = btf__resolve_size(spec->btf, acc->type_id);	5078	sz = btf__resolve_size(spec->btf, acc->type_id);
5047	if (sz < 0)	5079	if (sz < 0)
@@ -5058,7 +5090,7 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 10 unmodified lines
5058	}	5090	}
5059		5091
5060	m = btf_members(t) + acc->idx;	5092	m = btf_members(t) + acc->idx;
5061	mt = skip_mods_and_typedefs(spec->btf, m->type, NULL);	5093	mt = skip_mods_and_typedefs(spec->btf, m->type, &field_type_id);
5062	bit_off = spec->bit_offset;	5094	bit_off = spec->bit_offset;
5063	bit_sz = btf_member_bitfield_size(t, acc->idx);	5095	bit_sz = btf_member_bitfield_size(t, acc->idx);
5064		5096
@@ -5078,7 +5110,7 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 13 unmodified lines
5078	byte_off = bit_off / 8 / byte_sz * byte_sz;	5110	byte_off = bit_off / 8 / byte_sz * byte_sz;
5079	}	5111	}
5080	} else {	5112	} else {
5081	sz = btf__resolve_size(spec->btf, m->type);	5113	sz = btf__resolve_size(spec->btf, field_type_id);
5082	if (sz < 0)	5114	if (sz < 0)
5083	return -EINVAL;	5115	return -EINVAL;
5084	byte_sz = sz;	5116	byte_sz = sz;
@@ -5096,6 +5128,10 @@ static int bpf_core_calc_field_relo(const struct bpf_program *prog, · 11 unmodified lines
5096	switch (relo->kind) {	5128	switch (relo->kind) {
5097	case BPF_FIELD_BYTE_OFFSET:	5129	case BPF_FIELD_BYTE_OFFSET:
5098	*val = byte_off;	5130	*val = byte_off;
		5131	if (!bitfield) {
		5132	*field_sz = byte_sz;
		5133	*type_id = field_type_id;
		5134	}
5099	break;	5135	break;
5100	case BPF_FIELD_BYTE_SIZE:	5136	case BPF_FIELD_BYTE_SIZE:
5101	*val = byte_sz;	5137	*val = byte_sz;
@@ -5196,6 +5232,19 @@ struct bpf_core_relo_res · 94 unmodified lines
5196	bool poison;	5232	bool poison;
5197	/* some relocations can't be validated against orig_val */	5233	/* some relocations can't be validated against orig_val */
5198	bool validate;	5234	bool validate;
		5235	/* for field byte offset relocations or the forms:
		5236	* (T )(rX + <off>) = rY
		5237	* rX = (T )(rY + <off>),
		5238	* we remember original and resolved field size to adjust direct
		5239	* memory loads of pointers and integers; this is necessary for 32-bit
		5240	* host kernel architectures, but also allows to automatically
		5241	* relocate fields that were resized from, e.g., u32 to u64, etc.
		5242	*/
		5243	bool fail_memsz_adjust;
		5244	__u32 orig_sz;
		5245	__u32 orig_type_id;
		5246	__u32 new_sz;
		5247	__u32 new_type_id;
5199	};	5248	};
5200		5249
5201	/* Calculate original and target relocation values, given local and target	5250	/* Calculate original and target relocation values, given local and target
@@ -5217,10 +5266,56 @@ static int bpf_core_calc_relo(const struct bpf_program *prog, · 15 unmodified lines
5217	res->new_val = 0;	5266	res->new_val = 0;
5218	res->poison = false;	5267	res->poison = false;
5219	res->validate = true;	5268	res->validate = true;
		5269	res->fail_memsz_adjust = false;
		5270	res->orig_sz = res->new_sz = 0;
		5271	res->orig_type_id = res->new_type_id = 0;
5220		5272
5221	if (core_relo_is_field_based(relo->kind)) {	5273	if (core_relo_is_field_based(relo->kind)) {
5222	err = bpf_core_calc_field_relo(prog, relo, local_spec, &res->orig_val, &res->validate);	5274	err = bpf_core_calc_field_relo(prog, relo, local_spec,
5223	err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec, &res->new_val, NULL);	5275	&res->orig_val, &res->orig_sz,
		5276	&res->orig_type_id, &res->validate);
		5277	err = err ?: bpf_core_calc_field_relo(prog, relo, targ_spec,
		5278	&res->new_val, &res->new_sz,
		5279	&res->new_type_id, NULL);
		5280	if (err)
		5281	goto done;
		5282	/* Validate if it's safe to adjust load/store memory size.
		5283	* Adjustments are performed only if original and new memory
		5284	* sizes differ.
		5285	*/
		5286	res->fail_memsz_adjust = false;
		5287	if (res->orig_sz != res->new_sz) {
		5288	const struct btf_type orig_t, new_t;
		5289
		5290	orig_t = btf__type_by_id(local_spec->btf, res->orig_type_id);
		5291	new_t = btf__type_by_id(targ_spec->btf, res->new_type_id);
		5292
		5293	/* There are two use cases in which it's safe to
		5294	* adjust load/store's mem size:
		5295	* - reading a 32-bit kernel pointer, while on BPF
		5296	* size pointers are always 64-bit; in this case
		5297	* it's safe to "downsize" instruction size due to
		5298	* pointer being treated as unsigned integer with
		5299	* zero-extended upper 32-bits;
		5300	* - reading unsigned integers, again due to
		5301	* zero-extension is preserving the value correctly.
		5302	*
		5303	* In all other cases it's incorrect to attempt to
		5304	* load/store field because read value will be
		5305	* incorrect, so we poison relocated instruction.
		5306	*/
		5307	if (btf_is_ptr(orig_t) && btf_is_ptr(new_t))
		5308	goto done;
		5309	if (btf_is_int(orig_t) && btf_is_int(new_t) &&
		5310	btf_int_encoding(orig_t) != BTF_INT_SIGNED &&
		5311	btf_int_encoding(new_t) != BTF_INT_SIGNED)
		5312	goto done;
		5313
		5314	/* mark as invalid mem size adjustment, but this will
		5315	* only be checked for LDX/STX/ST insns
		5316	*/
		5317	res->fail_memsz_adjust = true;
		5318	}
5224	} else if (core_relo_is_type_based(relo->kind)) {	5319	} else if (core_relo_is_type_based(relo->kind)) {
5225	err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);	5320	err = bpf_core_calc_type_relo(relo, local_spec, &res->orig_val);
5226	err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);	5321	err = err ?: bpf_core_calc_type_relo(relo, targ_spec, &res->new_val);
@@ -5229,6 +5324,7 @@ static int bpf_core_calc_relo(const struct bpf_program *prog, · 2 unmodified lines
5229	err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);	5324	err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val);
5230	}	5325	}
5231		5326
		5327	done:
5232	if (err == -EUCLEAN) {	5328	if (err == -EUCLEAN) {
5233	/* EUCLEAN is used to signal instruction poisoning request */	5329	/* EUCLEAN is used to signal instruction poisoning request */
5234	res->poison = true;	5330	res->poison = true;
@@ -5268,6 +5364,28 @@ static bool is_ldimm64(struct bpf_insn *insn) · 33 unmodified lines
5268	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW);	5364	return insn->code == (BPF_LD \| BPF_IMM \| BPF_DW);
5269	}	5365	}
5270		5366
		5367	static int insn_bpf_size_to_bytes(struct bpf_insn *insn)
		5368	{
		5369	switch (BPF_SIZE(insn->code)) {
		5370	case BPF_DW: return 8;
		5371	case BPF_W: return 4;
		5372	case BPF_H: return 2;
		5373	case BPF_B: return 1;
		5374	default: return -1;
		5375	}
		5376	}
		5377
		5378	static int insn_bytes_to_bpf_size(__u32 sz)
		5379	{
		5380	switch (sz) {
		5381	case 8: return BPF_DW;
		5382	case 4: return BPF_W;
		5383	case 2: return BPF_H;
		5384	case 1: return BPF_B;
		5385	default: return -1;
		5386	}
		5387	}
		5388
5271	/*	5389	/*
5272	* Patch relocatable BPF instruction.	5390	* Patch relocatable BPF instruction.
5273	*	5391	*
@@ -5277,10 +5395,13 @@ static bool is_ldimm64(struct bpf_insn *insn) · 3 unmodified lines
5277	* spec, and is checked before patching instruction. If actual insn->imm value	5395	* spec, and is checked before patching instruction. If actual insn->imm value
5278	* is wrong, bail out with error.	5396	* is wrong, bail out with error.
5279	*	5397	*
5280	* Currently three kinds of BPF instructions are supported:	5398	* Currently supported classes of BPF instruction are:
5281	* 1. rX = <imm> (assignment with immediate operand);	5399	* 1. rX = <imm> (assignment with immediate operand);
5282	* 2. rX += <imm> (arithmetic operations with immediate operand);	5400	* 2. rX += <imm> (arithmetic operations with immediate operand);
5283	* 3. rX = <imm64> (load with 64-bit immediate value).	5401	* 3. rX = <imm64> (load with 64-bit immediate value);
		5402	* 4. rX = (T )(rY + <off>), where T is one of {u8, u16, u32, u64};
		5403	* 5. (T )(rX + <off>) = rY, where T is one of {u8, u16, u32, u64};
		5404	* 6. (T )(rX + <off>) = <imm>, where T is one of {u8, u16, u32, u64}.
5284	*/	5405	*/
5285	static int bpf_core_patch_insn(struct bpf_program *prog,	5406	static int bpf_core_patch_insn(struct bpf_program *prog,
5286	const struct bpf_core_relo *relo,	5407	const struct bpf_core_relo *relo,
@@ -5304,6 +5425,7 @@ static int bpf_core_patch_insn(struct bpf_program *prog, · 17 unmodified lines
5304	class = BPF_CLASS(insn->code);	5425	class = BPF_CLASS(insn->code);
5305		5426
5306	if (res->poison) {	5427	if (res->poison) {
		5428	poison:
5307	/* poison second part of ldimm64 to avoid confusing error from	5429	/* poison second part of ldimm64 to avoid confusing error from
5308	* verifier about "unknown opcode 00"	5430	* verifier about "unknown opcode 00"
5309	*/	5431	*/
@@ -5346,10 +5468,39 @@ static int bpf_core_patch_insn(struct bpf_program *prog, · 36 unmodified lines
5346	prog->name, relo_idx, insn_idx, new_val);	5468	prog->name, relo_idx, insn_idx, new_val);
5347	return -ERANGE;	5469	return -ERANGE;
5348	}	5470	}
		5471	if (res->fail_memsz_adjust) {
		5472	pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) accesses field incorrectly. "
		5473	"Make sure you are accessing pointers, unsigned integers, or fields of matching type and size.\n",
		5474	prog->name, relo_idx, insn_idx);
		5475	goto poison;
		5476	}
		5477
5349	orig_val = insn->off;	5478	orig_val = insn->off;
5350	insn->off = new_val;	5479	insn->off = new_val;
5351	pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",	5480	pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
5352	prog->name, relo_idx, insn_idx, orig_val, new_val);	5481	prog->name, relo_idx, insn_idx, orig_val, new_val);
		5482
		5483	if (res->new_sz != res->orig_sz) {
		5484	int insn_bytes_sz, insn_bpf_sz;
		5485
		5486	insn_bytes_sz = insn_bpf_size_to_bytes(insn);
		5487	if (insn_bytes_sz != res->orig_sz) {
		5488	pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) unexpected mem size: got %d, exp %u\n",
		5489	prog->name, relo_idx, insn_idx, insn_bytes_sz, res->orig_sz);
		5490	return -EINVAL;
		5491	}
		5492
		5493	insn_bpf_sz = insn_bytes_to_bpf_size(res->new_sz);
		5494	if (insn_bpf_sz < 0) {
		5495	pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) invalid new mem size: %u\n",
		5496	prog->name, relo_idx, insn_idx, res->new_sz);
		5497	return -EINVAL;
		5498	}
		5499
		5500	insn->code = BPF_MODE(insn->code) \| insn_bpf_sz \| BPF_CLASS(insn->code);
		5501	pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) mem_sz %u -> %u\n",
		5502	prog->name, relo_idx, insn_idx, res->orig_sz, res->new_sz);
		5503	}
5353	break;	5504	break;
5354	case BPF_LD: {	5505	case BPF_LD: {
5355	__u64 imm;	5506	__u64 imm;
@@ -5691,7 +5842,7 @@ bpf_object__relocate_core(struct bpf_object obj, const char targ_btf_path) · 335 unmodified lines
5691	return 0;	5842	return 0;
5692		5843
5693	if (targ_btf_path)	5844	if (targ_btf_path)
5694	targ_btf = btf__parse_elf(targ_btf_path, NULL);	5845	targ_btf = btf__parse(targ_btf_path, NULL);
5695	else	5846	else
5696	targ_btf = obj->btf_vmlinux;	5847	targ_btf = obj->btf_vmlinux;
5697	if (IS_ERR_OR_NULL(targ_btf)) {	5848	if (IS_ERR_OR_NULL(targ_btf)) {
@@ -5742,6 +5893,11 @@ bpf_object__relocate_core(struct bpf_object obj, const char targ_btf_path) · 44 unmodified lines
5742	err = -EINVAL;	5893	err = -EINVAL;
5743	goto out;	5894	goto out;
5744	}	5895	}
		5896	/* no need to apply CO-RE relocation if the program is
		5897	* not going to be loaded
		5898	*/
		5899	if (!prog->load)
		5900	continue;
5745		5901
5746	err = bpf_core_apply_relo(prog, rec, i, obj->btf,	5902	err = bpf_core_apply_relo(prog, rec, i, obj->btf,
5747	targ_btf, cand_cache);	5903	targ_btf, cand_cache);
@@ -5800,8 +5956,13 @@ bpf_object__relocate_data(struct bpf_object obj, struct bpf_program prog) · 52 unmodified lines
5800	insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;	5956	insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
5801	insn[1].imm = ext->kcfg.data_off;	5957	insn[1].imm = ext->kcfg.data_off;
5802	} else /* EXT_KSYM */ {	5958	} else /* EXT_KSYM */ {
5803	insn[0].imm = (__u32)ext->ksym.addr;	5959	if (ext->ksym.type_id) { /* typed ksyms */
5804	insn[1].imm = ext->ksym.addr >> 32;	5960	insn[0].src_reg = BPF_PSEUDO_BTF_ID;
		5961	insn[0].imm = ext->ksym.vmlinux_btf_id;
		5962	} else { /* typeless ksyms */
		5963	insn[0].imm = (__u32)ext->ksym.addr;
		5964	insn[1].imm = ext->ksym.addr >> 32;
		5965	}
5805	}	5966	}
5806	relo->processed = true;	5967	relo->processed = true;
5807	break;	5968	break;
@@ -6933,10 +7094,72 @@ out: · 1125 unmodified lines
6933	return err;	7094	return err;
6934	}	7095	}
6935		7096
		7097	static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
		7098	{
		7099	struct extern_desc *ext;
		7100	int i, id;
		7101
		7102	for (i = 0; i < obj->nr_extern; i++) {
		7103	const struct btf_type targ_var, targ_type;
		7104	__u32 targ_type_id, local_type_id;
		7105	const char *targ_var_name;
		7106	int ret;
		7107
		7108	ext = &obj->externs[i];
		7109	if (ext->type != EXT_KSYM \|\| !ext->ksym.type_id)
		7110	continue;
		7111
		7112	id = btf__find_by_name_kind(obj->btf_vmlinux, ext->name,
		7113	BTF_KIND_VAR);
		7114	if (id <= 0) {
		7115	pr_warn("extern (ksym) '%s': failed to find BTF ID in vmlinux BTF.\n",
		7116	ext->name);
		7117	return -ESRCH;
		7118	}
		7119
		7120	/* find local type_id */
		7121	local_type_id = ext->ksym.type_id;
		7122
		7123	/* find target type_id */
		7124	targ_var = btf__type_by_id(obj->btf_vmlinux, id);
		7125	targ_var_name = btf__name_by_offset(obj->btf_vmlinux,
		7126	targ_var->name_off);
		7127	targ_type = skip_mods_and_typedefs(obj->btf_vmlinux,
		7128	targ_var->type,
		7129	&targ_type_id);
		7130
		7131	ret = bpf_core_types_are_compat(obj->btf, local_type_id,
		7132	obj->btf_vmlinux, targ_type_id);
		7133	if (ret <= 0) {
		7134	const struct btf_type *local_type;
		7135	const char targ_name, local_name;
		7136
		7137	local_type = btf__type_by_id(obj->btf, local_type_id);
		7138	local_name = btf__name_by_offset(obj->btf,
		7139	local_type->name_off);
		7140	targ_name = btf__name_by_offset(obj->btf_vmlinux,
		7141	targ_type->name_off);
		7142
		7143	pr_warn("extern (ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n",
		7144	ext->name, local_type_id,
		7145	btf_kind_str(local_type), local_name, targ_type_id,
		7146	btf_kind_str(targ_type), targ_name);
		7147	return -EINVAL;
		7148	}
		7149
		7150	ext->is_set = true;
		7151	ext->ksym.vmlinux_btf_id = id;
		7152	pr_debug("extern (ksym) '%s': resolved to [%d] %s %s\n",
		7153	ext->name, id, btf_kind_str(targ_var), targ_var_name);
		7154	}
		7155	return 0;
		7156	}
		7157
6936	static int bpf_object__resolve_externs(struct bpf_object *obj,	7158	static int bpf_object__resolve_externs(struct bpf_object *obj,
6937	const char *extra_kconfig)	7159	const char *extra_kconfig)
6938	{	7160	{
6939	bool need_config = false, need_kallsyms = false;	7161	bool need_config = false, need_kallsyms = false;
		7162	bool need_vmlinux_btf = false;
6940	struct extern_desc *ext;	7163	struct extern_desc *ext;
6941	void *kcfg_data = NULL;	7164	void *kcfg_data = NULL;
6942	int err, i;	7165	int err, i;
@@ -6967,7 +7190,10 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, · 24 unmodified lines
6967	strncmp(ext->name, "CONFIG_", 7) == 0) {	7190	strncmp(ext->name, "CONFIG_", 7) == 0) {
6968	need_config = true;	7191	need_config = true;
6969	} else if (ext->type == EXT_KSYM) {	7192	} else if (ext->type == EXT_KSYM) {
6970	need_kallsyms = true;	7193	if (ext->ksym.type_id)
		7194	need_vmlinux_btf = true;
		7195	else
		7196	need_kallsyms = true;
6971	} else {	7197	} else {
6972	pr_warn("unrecognized extern '%s'\n", ext->name);	7198	pr_warn("unrecognized extern '%s'\n", ext->name);
6973	return -EINVAL;	7199	return -EINVAL;
@@ -6996,6 +7222,11 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, · 22 unmodified lines
6996	if (err)	7222	if (err)
6997	return -EINVAL;	7223	return -EINVAL;
6998	}	7224	}
		7225	if (need_vmlinux_btf) {
		7226	err = bpf_object__resolve_ksyms_btf_id(obj);
		7227	if (err)
		7228	return -EINVAL;
		7229	}
6999	for (i = 0; i < obj->nr_extern; i++) {	7230	for (i = 0; i < obj->nr_extern; i++) {
7000	ext = &obj->externs[i];	7231	ext = &obj->externs[i];
7001		7232
@@ -7028,10 +7259,10 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) · 26 unmodified lines
7028	}	7259	}
7029		7260
7030	err = bpf_object__probe_loading(obj);	7261	err = bpf_object__probe_loading(obj);
		7262	err = err ? : bpf_object__load_vmlinux_btf(obj);
7031	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);	7263	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7032	err = err ? : bpf_object__sanitize_and_load_btf(obj);	7264	err = err ? : bpf_object__sanitize_and_load_btf(obj);
7033	err = err ? : bpf_object__sanitize_maps(obj);	7265	err = err ? : bpf_object__sanitize_maps(obj);
7034	err = err ? : bpf_object__load_vmlinux_btf(obj);
7035	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);	7266	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7036	err = err ? : bpf_object__create_maps(obj);	7267	err = err ? : bpf_object__create_maps(obj);
7037	err = err ? : bpf_object__relocate(obj, attr->target_btf_path);	7268	err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
@@ -10353,9 +10584,8 @@ int bpf_program__set_attach_target(struct bpf_program *prog, · 3315 unmodified lines
10353	btf_id = libbpf_find_prog_btf_id(attach_func_name,	10584	btf_id = libbpf_find_prog_btf_id(attach_func_name,
10354	attach_prog_fd);	10585	attach_prog_fd);
10355	else	10586	else
10356	btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux,	10587	btf_id = libbpf_find_vmlinux_btf_id(attach_func_name,
10357	attach_func_name,	10588	prog->expected_attach_type);
10358	prog->expected_attach_type);
10359		10589
10360	if (btf_id < 0)	10590	if (btf_id < 0)
10361	return btf_id;	10591	return btf_id;

tools/lib/bpf/xsk.c+6−1

 	struct xsk_ctx *ctx;
 	int err, ifindex;
-	if (!umem || !xsk_ptr || !(rx || tx) || !fill || !comp)
+	if (!umem || !xsk_ptr || !(rx || tx))
 		return -EFAULT;
 	xsk = calloc(1, sizeof(*xsk));
 	ctx = xsk_get_ctx(umem, ifindex, queue_id);
 	if (!ctx) {
+		if (!fill || !comp) {
+			err = -EFAULT;
+			goto out_socket;
+		}
 		ctx = xsk_create_ctx(xsk, umem, ifindex, ifname, queue_id,
 				     fill, comp);
 		if (!ctx) {

tools/testing/selftests/bpf/README.rst+38−0

@@ -7,6 +7,44 @@ General instructions on running selftests can be found in · 6 unmodified lines
7	Additional information about selftest failures are	7	Additional information about selftest failures are
8	documented here.	8	documented here.
9		9
		10	profiler[23] test failures with clang/llvm <12.0.0
		11	==================================================
		12
		13	With clang/llvm <12.0.0, the profiler[23] test may fail.
		14	The symptom looks like
		15
		16	.. code-block:: c
		17
		18	// r9 is a pointer to map_value
		19	// r7 is a scalar
		20	17: bf 96 00 00 00 00 00 00 r6 = r9
		21	18: 0f 76 00 00 00 00 00 00 r6 += r7
		22	math between map_value pointer and register with unbounded min value is not allowed
		23
		24	// the instructions below will not be seen in the verifier log
		25	19: a5 07 01 00 01 01 00 00 if r7 < 257 goto +1
		26	20: bf 96 00 00 00 00 00 00 r6 = r9
		27	// r6 is used here
		28
		29	The verifier will reject such code with above error.
		30	At insn 18 the r7 is indeed unbounded. The later insn 19 checks the bounds and
		31	the insn 20 undoes map_value addition. It is currently impossible for the
		32	verifier to understand such speculative pointer arithmetic.
		33	Hence
		34	https://reviews.llvm.org/D85570
		35	addresses it on the compiler side. It was committed on llvm 12.
		36
		37	The corresponding C code
		38	.. code-block:: c
		39
		40	for (int i = 0; i < MAX_CGROUPS_PATH_DEPTH; i++) {
		41	filepart_length = bpf_probe_read_str(payload, ...);
		42	if (filepart_length <= MAX_PATH) {
		43	barrier_var(filepart_length); // workaround
		44	payload += filepart_length;
		45	}
		46	}
		47
10	bpf_iter test failures with clang/llvm 10.0.0	48	bpf_iter test failures with clang/llvm 10.0.0
11	=============================================	49	=============================================
12		50

tools/testing/selftests/bpf/prog_tests/align.c+8−8

tools/testing/selftests/bpf/prog_tests/btf_map_in_map.c+29−10

 static void test_lookup_update(void)
 {
-	int err, key = 0, val, i;
+	int map1_fd, map2_fd, map3_fd, map4_fd, map5_fd, map1_id, map2_id;
+	int outer_arr_fd, outer_hash_fd, outer_arr_dyn_fd;
 	struct test_btf_map_in_map *skel;
-	int outer_arr_fd, outer_hash_fd;
+	int err, key = 0, val, i, fd;
-	int fd, map1_fd, map2_fd, map1_id, map2_id;
 	skel = test_btf_map_in_map__open_and_load();
 	if (CHECK(!skel, "skel_open", "failed to open&load skeleton\n"))
 	map1_fd = bpf_map__fd(skel->maps.inner_map1);
 	map2_fd = bpf_map__fd(skel->maps.inner_map2);
+	map3_fd = bpf_map__fd(skel->maps.inner_map3);
+	map4_fd = bpf_map__fd(skel->maps.inner_map4);
+	map5_fd = bpf_map__fd(skel->maps.inner_map5);
+	outer_arr_dyn_fd = bpf_map__fd(skel->maps.outer_arr_dyn);
 	outer_arr_fd = bpf_map__fd(skel->maps.outer_arr);
 	outer_hash_fd = bpf_map__fd(skel->maps.outer_hash);
-	/* inner1 = input, inner2 = input + 1 */
+	/* inner1 = input, inner2 = input + 1, inner3 = input + 2 */
-	map1_fd = bpf_map__fd(skel->maps.inner_map1);
 	bpf_map_update_elem(outer_arr_fd, &key, &map1_fd, 0);
-	map2_fd = bpf_map__fd(skel->maps.inner_map2);
 	bpf_map_update_elem(outer_hash_fd, &key, &map2_fd, 0);
+	bpf_map_update_elem(outer_arr_dyn_fd, &key, &map3_fd, 0);
 	skel->bss->input = 1;
 	usleep(1);
 	bpf_map_lookup_elem(map1_fd, &key, &val);
 	CHECK(val != 1, "inner1", "got %d != exp %d\n", val, 1);
 	bpf_map_lookup_elem(map2_fd, &key, &val);
 	CHECK(val != 2, "inner2", "got %d != exp %d\n", val, 2);
+	bpf_map_lookup_elem(map3_fd, &key, &val);
+	CHECK(val != 3, "inner3", "got %d != exp %d\n", val, 3);
-	/* inner1 = input + 1, inner2 = input */
+	/* inner2 = input, inner1 = input + 1, inner4 = input + 2 */
 	bpf_map_update_elem(outer_arr_fd, &key, &map2_fd, 0);
 	bpf_map_update_elem(outer_hash_fd, &key, &map1_fd, 0);
+	bpf_map_update_elem(outer_arr_dyn_fd, &key, &map4_fd, 0);
 	skel->bss->input = 3;
 	usleep(1);
 	bpf_map_lookup_elem(map1_fd, &key, &val);
 	CHECK(val != 4, "inner1", "got %d != exp %d\n", val, 4);
 	bpf_map_lookup_elem(map2_fd, &key, &val);
 	CHECK(val != 3, "inner2", "got %d != exp %d\n", val, 3);
+	bpf_map_lookup_elem(map4_fd, &key, &val);
+	CHECK(val != 5, "inner4", "got %d != exp %d\n", val, 5);
+	/* inner5 = input + 2 */
+	bpf_map_update_elem(outer_arr_dyn_fd, &key, &map5_fd, 0);
+	skel->bss->input = 5;
+	usleep(1);
+	bpf_map_lookup_elem(map5_fd, &key, &val);
+	CHECK(val != 7, "inner5", "got %d != exp %d\n", val, 7);
 	for (i = 0; i < 5; i++) {
 		val = i % 2 ? map1_fd : map2_fd;
 		}
 		err = bpf_map_update_elem(outer_arr_fd, &key, &val, 0);
 		if (CHECK_FAIL(err)) {
-			printf("failed to update hash_of_maps on iter #%d\n", i);
+			printf("failed to update array_of_maps on iter #%d\n", i);
+			goto cleanup;
+		}
+		val = i % 2 ? map4_fd : map5_fd;
+		err = bpf_map_update_elem(outer_arr_dyn_fd, &key, &val, 0);
+		if (CHECK_FAIL(err)) {
+			printf("failed to update array_of_maps (dyn) on iter #%d\n", i);
 			goto cleanup;
 		}
 	}

tools/testing/selftests/bpf/prog_tests/core_autosize.c+225−0

@@ -0,0 +1,225 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3
	4	#include <test_progs.h>
	5	#include <bpf/btf.h>
	6
	7	/* real layout and sizes according to test's (32-bit) BTF
	8	* needs to be defined before skeleton is included */
	9	struct test_struct___real {
	10	unsigned int ptr; /* can't use `void `, it is always 8 byte in BPF target /
	11	unsigned int val2;
	12	unsigned long long val1;
	13	unsigned short val3;
	14	unsigned char val4;
	15	unsigned char _pad;
	16	};
	17
	18	#include "test_core_autosize.skel.h"
	19
	20	static int duration = 0;
	21
	22	static struct {
	23	unsigned long long ptr_samesized;
	24	unsigned long long val1_samesized;
	25	unsigned long long val2_samesized;
	26	unsigned long long val3_samesized;
	27	unsigned long long val4_samesized;
	28	struct test_struct___real output_samesized;
	29
	30	unsigned long long ptr_downsized;
	31	unsigned long long val1_downsized;
	32	unsigned long long val2_downsized;
	33	unsigned long long val3_downsized;
	34	unsigned long long val4_downsized;
	35	struct test_struct___real output_downsized;
	36
	37	unsigned long long ptr_probed;
	38	unsigned long long val1_probed;
	39	unsigned long long val2_probed;
	40	unsigned long long val3_probed;
	41	unsigned long long val4_probed;
	42
	43	unsigned long long ptr_signed;
	44	unsigned long long val1_signed;
	45	unsigned long long val2_signed;
	46	unsigned long long val3_signed;
	47	unsigned long long val4_signed;
	48	struct test_struct___real output_signed;
	49	} out;
	50
	51	void test_core_autosize(void)
	52	{
	53	char btf_file[] = "/tmp/core_autosize.btf.XXXXXX";
	54	int err, fd = -1, zero = 0;
	55	int char_id, short_id, int_id, long_long_id, void_ptr_id, id;
	56	struct test_core_autosize* skel = NULL;
	57	struct bpf_object_load_attr load_attr = {};
	58	struct bpf_program *prog;
	59	struct bpf_map *bss_map;
	60	struct btf *btf = NULL;
	61	size_t written;
	62	const void *raw_data;
	63	__u32 raw_sz;
	64	FILE *f = NULL;
	65
	66	btf = btf__new_empty();
	67	if (!ASSERT_OK_PTR(btf, "empty_btf"))
	68	return;
	69	/* Emit the following struct with 32-bit pointer size:
	70	*
	71	* struct test_struct {
	72	* void *ptr;
	73	* unsigned long val2;
	74	* unsigned long long val1;
	75	* unsigned short val3;
	76	* unsigned char val4;
	77	* char: 8;
	78	* };
	79	*
	80	* This struct is going to be used as the "kernel BTF" for this test.
	81	* It's equivalent memory-layout-wise to test_struct__real above.
	82	*/
	83
	84	/* force 32-bit pointer size */
	85	btf__set_pointer_size(btf, 4);
	86
	87	char_id = btf__add_int(btf, "unsigned char", 1, 0);
	88	ASSERT_EQ(char_id, 1, "char_id");
	89	short_id = btf__add_int(btf, "unsigned short", 2, 0);
	90	ASSERT_EQ(short_id, 2, "short_id");
	91	/* "long unsigned int" of 4 byte size tells BTF that sizeof(void ) == 4 /
	92	int_id = btf__add_int(btf, "long unsigned int", 4, 0);
	93	ASSERT_EQ(int_id, 3, "int_id");
	94	long_long_id = btf__add_int(btf, "unsigned long long", 8, 0);
	95	ASSERT_EQ(long_long_id, 4, "long_long_id");
	96	void_ptr_id = btf__add_ptr(btf, 0);
	97	ASSERT_EQ(void_ptr_id, 5, "void_ptr_id");
	98
	99	id = btf__add_struct(btf, "test_struct", 20 /* bytes */);
	100	ASSERT_EQ(id, 6, "struct_id");
	101	err = btf__add_field(btf, "ptr", void_ptr_id, 0, 0);
	102	err = err ?: btf__add_field(btf, "val2", int_id, 32, 0);
	103	err = err ?: btf__add_field(btf, "val1", long_long_id, 64, 0);
	104	err = err ?: btf__add_field(btf, "val3", short_id, 128, 0);
	105	err = err ?: btf__add_field(btf, "val4", char_id, 144, 0);
	106	ASSERT_OK(err, "struct_fields");
	107
	108	fd = mkstemp(btf_file);
	109	if (CHECK(fd < 0, "btf_tmp", "failed to create file: %d\n", fd))
	110	goto cleanup;
	111	f = fdopen(fd, "w");
	112	if (!ASSERT_OK_PTR(f, "btf_fdopen"))
	113	goto cleanup;
	114
	115	raw_data = btf__get_raw_data(btf, &raw_sz);
	116	if (!ASSERT_OK_PTR(raw_data, "raw_data"))
	117	goto cleanup;
	118	written = fwrite(raw_data, 1, raw_sz, f);
	119	if (CHECK(written != raw_sz, "btf_write", "written: %zu, errno: %d\n", written, errno))
	120	goto cleanup;
	121	fflush(f);
	122	fclose(f);
	123	f = NULL;
	124	close(fd);
	125	fd = -1;
	126
	127	/* open and load BPF program with custom BTF as the kernel BTF */
	128	skel = test_core_autosize__open();
	129	if (!ASSERT_OK_PTR(skel, "skel_open"))
	130	return;
	131
	132	/* disable handle_signed() for now */
	133	prog = bpf_object__find_program_by_name(skel->obj, "handle_signed");
	134	if (!ASSERT_OK_PTR(prog, "prog_find"))
	135	goto cleanup;
	136	bpf_program__set_autoload(prog, false);
	137
	138	load_attr.obj = skel->obj;
	139	load_attr.target_btf_path = btf_file;
	140	err = bpf_object__load_xattr(&load_attr);
	141	if (!ASSERT_OK(err, "prog_load"))
	142	goto cleanup;
	143
	144	prog = bpf_object__find_program_by_name(skel->obj, "handle_samesize");
	145	if (!ASSERT_OK_PTR(prog, "prog_find"))
	146	goto cleanup;
	147	skel->links.handle_samesize = bpf_program__attach(prog);
	148	if (!ASSERT_OK_PTR(skel->links.handle_samesize, "prog_attach"))
	149	goto cleanup;
	150
	151	prog = bpf_object__find_program_by_name(skel->obj, "handle_downsize");
	152	if (!ASSERT_OK_PTR(prog, "prog_find"))
	153	goto cleanup;
	154	skel->links.handle_downsize = bpf_program__attach(prog);
	155	if (!ASSERT_OK_PTR(skel->links.handle_downsize, "prog_attach"))
	156	goto cleanup;
	157
	158	prog = bpf_object__find_program_by_name(skel->obj, "handle_probed");
	159	if (!ASSERT_OK_PTR(prog, "prog_find"))
	160	goto cleanup;
	161	skel->links.handle_probed = bpf_program__attach(prog);
	162	if (!ASSERT_OK_PTR(skel->links.handle_probed, "prog_attach"))
	163	goto cleanup;
	164
	165	usleep(1);
	166
	167	bss_map = bpf_object__find_map_by_name(skel->obj, "test_cor.bss");
	168	if (!ASSERT_OK_PTR(bss_map, "bss_map_find"))
	169	goto cleanup;
	170
	171	err = bpf_map_lookup_elem(bpf_map__fd(bss_map), &zero, (void *)&out);
	172	if (!ASSERT_OK(err, "bss_lookup"))
	173	goto cleanup;
	174
	175	ASSERT_EQ(out.ptr_samesized, 0x01020304, "ptr_samesized");
	176	ASSERT_EQ(out.val1_samesized, 0x1020304050607080, "val1_samesized");
	177	ASSERT_EQ(out.val2_samesized, 0x0a0b0c0d, "val2_samesized");
	178	ASSERT_EQ(out.val3_samesized, 0xfeed, "val3_samesized");
	179	ASSERT_EQ(out.val4_samesized, 0xb9, "val4_samesized");
	180	ASSERT_EQ(out.output_samesized.ptr, 0x01020304, "ptr_samesized");
	181	ASSERT_EQ(out.output_samesized.val1, 0x1020304050607080, "val1_samesized");
	182	ASSERT_EQ(out.output_samesized.val2, 0x0a0b0c0d, "val2_samesized");
	183	ASSERT_EQ(out.output_samesized.val3, 0xfeed, "val3_samesized");
	184	ASSERT_EQ(out.output_samesized.val4, 0xb9, "val4_samesized");
	185
	186	ASSERT_EQ(out.ptr_downsized, 0x01020304, "ptr_downsized");
	187	ASSERT_EQ(out.val1_downsized, 0x1020304050607080, "val1_downsized");
	188	ASSERT_EQ(out.val2_downsized, 0x0a0b0c0d, "val2_downsized");
	189	ASSERT_EQ(out.val3_downsized, 0xfeed, "val3_downsized");
	190	ASSERT_EQ(out.val4_downsized, 0xb9, "val4_downsized");
	191	ASSERT_EQ(out.output_downsized.ptr, 0x01020304, "ptr_downsized");
	192	ASSERT_EQ(out.output_downsized.val1, 0x1020304050607080, "val1_downsized");
	193	ASSERT_EQ(out.output_downsized.val2, 0x0a0b0c0d, "val2_downsized");
	194	ASSERT_EQ(out.output_downsized.val3, 0xfeed, "val3_downsized");
	195	ASSERT_EQ(out.output_downsized.val4, 0xb9, "val4_downsized");
	196
	197	ASSERT_EQ(out.ptr_probed, 0x01020304, "ptr_probed");
	198	ASSERT_EQ(out.val1_probed, 0x1020304050607080, "val1_probed");
	199	ASSERT_EQ(out.val2_probed, 0x0a0b0c0d, "val2_probed");
	200	ASSERT_EQ(out.val3_probed, 0xfeed, "val3_probed");
	201	ASSERT_EQ(out.val4_probed, 0xb9, "val4_probed");
	202
	203	test_core_autosize__destroy(skel);
	204	skel = NULL;
	205
	206	/* now re-load with handle_signed() enabled, it should fail loading */
	207	skel = test_core_autosize__open();
	208	if (!ASSERT_OK_PTR(skel, "skel_open"))
	209	return;
	210
	211	load_attr.obj = skel->obj;
	212	load_attr.target_btf_path = btf_file;
	213	err = bpf_object__load_xattr(&load_attr);
	214	if (!ASSERT_ERR(err, "bad_prog_load"))
	215	goto cleanup;
	216
	217	cleanup:
	218	if (f)
	219	fclose(f);
	220	if (fd >= 0)
	221	close(fd);
	222	remove(btf_file);
	223	btf__free(btf);
	224	test_core_autosize__destroy(skel);
	225	}

tools/testing/selftests/bpf/prog_tests/ksyms.c+13−25

 static int duration;
-static __u64 kallsyms_find(const char *sym)
-{
-	char type, name[500];
-	__u64 addr, res = 0;
-	FILE *f;
-	f = fopen("/proc/kallsyms", "r");
-	if (CHECK(!f, "kallsyms_fopen", "failed to open: %d\n", errno))
-		return 0;
-	while (fscanf(f, "%llx %c %499s%*[^\n]\n", &addr, &type, name) > 0) {
-		if (strcmp(name, sym) == 0) {
-			res = addr;
-			goto out;
-		}
-	}
-	CHECK(false, "not_found", "symbol %s not found\n", sym);
-out:
-	fclose(f);
-	return res;
-}
 void test_ksyms(void)
 {
-	__u64 per_cpu_start_addr = kallsyms_find("__per_cpu_start");
-	__u64 link_fops_addr = kallsyms_find("bpf_link_fops");
 	const char *btf_path = "/sys/kernel/btf/vmlinux";
 	struct test_ksyms *skel;
 	struct test_ksyms__data *data;
+	__u64 link_fops_addr, per_cpu_start_addr;
 	struct stat st;
 	__u64 btf_size;
 	int err;
+	err = kallsyms_find("bpf_link_fops", &link_fops_addr);
+	if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
+		return;
+	if (CHECK(err == -ENOENT, "ksym_find", "symbol 'bpf_link_fops' not found\n"))
+		return;
+	err = kallsyms_find("__per_cpu_start", &per_cpu_start_addr);
+	if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
+		return;
+	if (CHECK(err == -ENOENT, "ksym_find", "symbol 'per_cpu_start' not found\n"))
+		return;
 	if (CHECK(stat(btf_path, &st), "stat_btf", "err %d\n", errno))
 		return;
 	btf_size = st.st_size;

tools/testing/selftests/bpf/prog_tests/ksyms_btf.c+88−0

@@ -0,0 +1,88 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Google */
	3
	4	#include <test_progs.h>
	5	#include <bpf/libbpf.h>
	6	#include <bpf/btf.h>
	7	#include "test_ksyms_btf.skel.h"
	8
	9	static int duration;
	10
	11	void test_ksyms_btf(void)
	12	{
	13	__u64 runqueues_addr, bpf_prog_active_addr;
	14	__u32 this_rq_cpu;
	15	int this_bpf_prog_active;
	16	struct test_ksyms_btf *skel = NULL;
	17	struct test_ksyms_btf__data *data;
	18	struct btf *btf;
	19	int percpu_datasec;
	20	int err;
	21
	22	err = kallsyms_find("runqueues", &runqueues_addr);
	23	if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
	24	return;
	25	if (CHECK(err == -ENOENT, "ksym_find", "symbol 'runqueues' not found\n"))
	26	return;
	27
	28	err = kallsyms_find("bpf_prog_active", &bpf_prog_active_addr);
	29	if (CHECK(err == -EINVAL, "kallsyms_fopen", "failed to open: %d\n", errno))
	30	return;
	31	if (CHECK(err == -ENOENT, "ksym_find", "symbol 'bpf_prog_active' not found\n"))
	32	return;
	33
	34	btf = libbpf_find_kernel_btf();
	35	if (CHECK(IS_ERR(btf), "btf_exists", "failed to load kernel BTF: %ld\n",
	36	PTR_ERR(btf)))
	37	return;
	38
	39	percpu_datasec = btf__find_by_name_kind(btf, ".data..percpu",
	40	BTF_KIND_DATASEC);
	41	if (percpu_datasec < 0) {
	42	printf("%s:SKIP:no PERCPU DATASEC in kernel btf\n",
	43	__func__);
	44	test__skip();
	45	goto cleanup;
	46	}
	47
	48	skel = test_ksyms_btf__open_and_load();
	49	if (CHECK(!skel, "skel_open", "failed to open and load skeleton\n"))
	50	goto cleanup;
	51
	52	err = test_ksyms_btf__attach(skel);
	53	if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
	54	goto cleanup;
	55
	56	/* trigger tracepoint */
	57	usleep(1);
	58
	59	data = skel->data;
	60	CHECK(data->out__runqueues_addr != runqueues_addr, "runqueues_addr",
	61	"got %llu, exp %llu\n",
	62	(unsigned long long)data->out__runqueues_addr,
	63	(unsigned long long)runqueues_addr);
	64	CHECK(data->out__bpf_prog_active_addr != bpf_prog_active_addr, "bpf_prog_active_addr",
	65	"got %llu, exp %llu\n",
	66	(unsigned long long)data->out__bpf_prog_active_addr,
	67	(unsigned long long)bpf_prog_active_addr);
	68
	69	CHECK(data->out__rq_cpu == -1, "rq_cpu",
	70	"got %u, exp != -1\n", data->out__rq_cpu);
	71	CHECK(data->out__bpf_prog_active < 0, "bpf_prog_active",
	72	"got %d, exp >= 0\n", data->out__bpf_prog_active);
	73	CHECK(data->out__cpu_0_rq_cpu != 0, "cpu_rq(0)->cpu",
	74	"got %u, exp 0\n", data->out__cpu_0_rq_cpu);
	75
	76	this_rq_cpu = data->out__this_rq_cpu;
	77	CHECK(this_rq_cpu != data->out__rq_cpu, "this_rq_cpu",
	78	"got %u, exp %u\n", this_rq_cpu, data->out__rq_cpu);
	79
	80	this_bpf_prog_active = data->out__this_bpf_prog_active;
	81	CHECK(this_bpf_prog_active != data->out__bpf_prog_active, "this_bpf_prog_active",
	82	"got %d, exp %d\n", this_bpf_prog_active,
	83	data->out__bpf_prog_active);
	84
	85	cleanup:
	86	btf__free(btf);
	87	test_ksyms_btf__destroy(skel);
	88	}

tools/testing/selftests/bpf/prog_tests/pinning.c+48−1

@@ -37,7 +37,7 @@ void test_pinning(void) · 36 unmodified lines
37	struct stat statbuf = {};	37	struct stat statbuf = {};
38	struct bpf_object *obj;	38	struct bpf_object *obj;
39	struct bpf_map *map;	39	struct bpf_map *map;
40	int err;	40	int err, map_fd;
41	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,	41	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
42	.pin_root_path = custpath,	42	.pin_root_path = custpath,
43	);	43	);
@@ -213,6 +213,53 @@ void test_pinning(void) · 169 unmodified lines
213	if (CHECK(err, "stat custpinpath", "err %d errno %d\n", err, errno))	213	if (CHECK(err, "stat custpinpath", "err %d errno %d\n", err, errno))
214	goto out;	214	goto out;
215		215
		216	/* remove the custom pin path to re-test it with reuse fd below */
		217	err = unlink(custpinpath);
		218	if (CHECK(err, "unlink custpinpath", "err %d errno %d\n", err, errno))
		219	goto out;
		220
		221	err = rmdir(custpath);
		222	if (CHECK(err, "rmdir custpindir", "err %d errno %d\n", err, errno))
		223	goto out;
		224
		225	bpf_object__close(obj);
		226
		227	/* test pinning at custom path with reuse fd */
		228	obj = bpf_object__open_file(file, NULL);
		229	err = libbpf_get_error(obj);
		230	if (CHECK(err, "default open", "err %d errno %d\n", err, errno)) {
		231	obj = NULL;
		232	goto out;
		233	}
		234
		235	map_fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(__u32),
		236	sizeof(__u64), 1, 0);
		237	if (CHECK(map_fd < 0, "create pinmap manually", "fd %d\n", map_fd))
		238	goto out;
		239
		240	map = bpf_object__find_map_by_name(obj, "pinmap");
		241	if (CHECK(!map, "find map", "NULL map"))
		242	goto close_map_fd;
		243
		244	err = bpf_map__reuse_fd(map, map_fd);
		245	if (CHECK(err, "reuse pinmap fd", "err %d errno %d\n", err, errno))
		246	goto close_map_fd;
		247
		248	err = bpf_map__set_pin_path(map, custpinpath);
		249	if (CHECK(err, "set pin path", "err %d errno %d\n", err, errno))
		250	goto close_map_fd;
		251
		252	err = bpf_object__load(obj);
		253	if (CHECK(err, "custom load", "err %d errno %d\n", err, errno))
		254	goto close_map_fd;
		255
		256	/* check that pinmap was pinned at the custom path */
		257	err = stat(custpinpath, &statbuf);
		258	if (CHECK(err, "stat custpinpath", "err %d errno %d\n", err, errno))
		259	goto close_map_fd;
		260
		261	close_map_fd:
		262	close(map_fd);
216	out:	263	out:
217	unlink(pinpath);	264	unlink(pinpath);
218	unlink(nopinpath);	265	unlink(nopinpath);

tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+1−1

@@ -198,7 +198,7 @@ static void test_sockmap_copy(enum bpf_map_type map_type) · 197 unmodified lines
198	{	198	{
199	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);	199	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
200	int err, len, src_fd, iter_fd, duration = 0;	200	int err, len, src_fd, iter_fd, duration = 0;
201	union bpf_iter_link_info linfo = {0};	201	union bpf_iter_link_info linfo = {};
202	__u32 i, num_sockets, num_elems;	202	__u32 i, num_sockets, num_elems;
203	struct bpf_iter_sockmap *skel;	203	struct bpf_iter_sockmap *skel;
204	__s64 *sock_fd = NULL;	204	__s64 *sock_fd = NULL;

tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c+12−0

 static void check_hdr_and_close_fds(struct sk_fds *sk_fds)
 {
+	const __u32 expected_inherit_cb_flags =
+		BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG |
+		BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG |
+		BPF_SOCK_OPS_STATE_CB_FLAG;
 	if (sk_fds_shutdown(sk_fds))
 		goto check_linum;
+	if (CHECK(expected_inherit_cb_flags != skel->bss->inherit_cb_flags,
+		  "Unexpected inherit_cb_flags", "0x%x != 0x%x\n",
+		  skel->bss->inherit_cb_flags, expected_inherit_cb_flags))
+		goto check_linum;
 	if (check_hdr_stg(&exp_passive_hdr_stg, sk_fds->passive_fd,
 			  "passive_hdr_stg"))
 		goto check_linum;
 	memset(&skel->bss->active_estab_in, 0, optsize);
 	memset(&skel->bss->active_fin_in, 0, optsize);
+	skel->bss->inherit_cb_flags = 0;
 	skel->data->test_kind = TCPOPT_EXP;
 	skel->data->test_magic = 0xeB9F;

tools/testing/selftests/bpf/prog_tests/test_profiler.c+72−0

@@ -0,0 +1,72 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#include <test_progs.h>
	4	#include "progs/profiler.h"
	5	#include "profiler1.skel.h"
	6	#include "profiler2.skel.h"
	7	#include "profiler3.skel.h"
	8
	9	static int sanity_run(struct bpf_program *prog)
	10	{
	11	struct bpf_prog_test_run_attr test_attr = {};
	12	__u64 args[] = {1, 2, 3};
	13	__u32 duration = 0;
	14	int err, prog_fd;
	15
	16	prog_fd = bpf_program__fd(prog);
	17	test_attr.prog_fd = prog_fd;
	18	test_attr.ctx_in = args;
	19	test_attr.ctx_size_in = sizeof(args);
	20	err = bpf_prog_test_run_xattr(&test_attr);
	21	if (CHECK(err \|\| test_attr.retval, "test_run",
	22	"err %d errno %d retval %d duration %d\n",
	23	err, errno, test_attr.retval, duration))
	24	return -1;
	25	return 0;
	26	}
	27
	28	void test_test_profiler(void)
	29	{
	30	struct profiler1 *profiler1_skel = NULL;
	31	struct profiler2 *profiler2_skel = NULL;
	32	struct profiler3 *profiler3_skel = NULL;
	33	__u32 duration = 0;
	34	int err;
	35
	36	profiler1_skel = profiler1__open_and_load();
	37	if (CHECK(!profiler1_skel, "profiler1_skel_load", "profiler1 skeleton failed\n"))
	38	goto cleanup;
	39
	40	err = profiler1__attach(profiler1_skel);
	41	if (CHECK(err, "profiler1_attach", "profiler1 attach failed: %d\n", err))
	42	goto cleanup;
	43
	44	if (sanity_run(profiler1_skel->progs.raw_tracepoint__sched_process_exec))
	45	goto cleanup;
	46
	47	profiler2_skel = profiler2__open_and_load();
	48	if (CHECK(!profiler2_skel, "profiler2_skel_load", "profiler2 skeleton failed\n"))
	49	goto cleanup;
	50
	51	err = profiler2__attach(profiler2_skel);
	52	if (CHECK(err, "profiler2_attach", "profiler2 attach failed: %d\n", err))
	53	goto cleanup;
	54
	55	if (sanity_run(profiler2_skel->progs.raw_tracepoint__sched_process_exec))
	56	goto cleanup;
	57
	58	profiler3_skel = profiler3__open_and_load();
	59	if (CHECK(!profiler3_skel, "profiler3_skel_load", "profiler3 skeleton failed\n"))
	60	goto cleanup;
	61
	62	err = profiler3__attach(profiler3_skel);
	63	if (CHECK(err, "profiler3_attach", "profiler3 attach failed: %d\n", err))
	64	goto cleanup;
	65
	66	if (sanity_run(profiler3_skel->progs.raw_tracepoint__sched_process_exec))
	67	goto cleanup;
	68	cleanup:
	69	profiler1__destroy(profiler1_skel);
	70	profiler2__destroy(profiler2_skel);
	71	profiler3__destroy(profiler3_skel);
	72	}

tools/testing/selftests/bpf/prog_tests/xdp_noinline.c+1−1

@@ -25,7 +25,7 @@ void test_xdp_noinline(void) · 24 unmodified lines
25	__u8 flags;	25	__u8 flags;
26	} real_def = {.dst = MAGIC_VAL};	26	} real_def = {.dst = MAGIC_VAL};
27	__u32 ch_key = 11, real_num = 3;	27	__u32 ch_key = 11, real_num = 3;
28	__u32 duration, retval, size;	28	__u32 duration = 0, retval, size;
29	int err, i;	29	int err, i;
30	__u64 bytes = 0, pkts = 0;	30	__u64 bytes = 0, pkts = 0;
31	char buf[128];	31	char buf[128];

tools/testing/selftests/bpf/progs/connect4_prog.c+19−0

@@ -23,6 +23,10 @@ · 22 unmodified lines
23	#define TCP_CA_NAME_MAX 16	23	#define TCP_CA_NAME_MAX 16
24	#endif	24	#endif
25		25
		26	#ifndef TCP_NOTSENT_LOWAT
		27	#define TCP_NOTSENT_LOWAT 25
		28	#endif
		29
26	#ifndef IFNAMSIZ	30	#ifndef IFNAMSIZ
27	#define IFNAMSIZ 16	31	#define IFNAMSIZ 16
28	#endif	32	#endif
@@ -128,6 +132,18 @@ static __inline int set_keepalive(struct bpf_sock_addr *ctx) · 99 unmodified lines
128	return 0;	132	return 0;
129	}	133	}
130		134
		135	static __inline int set_notsent_lowat(struct bpf_sock_addr *ctx)
		136	{
		137	int lowat = 65535;
		138
		139	if (ctx->type == SOCK_STREAM) {
		140	if (bpf_setsockopt(ctx, SOL_TCP, TCP_NOTSENT_LOWAT, &lowat, sizeof(lowat)))
		141	return 1;
		142	}
		143
		144	return 0;
		145	}
		146
131	SEC("cgroup/connect4")	147	SEC("cgroup/connect4")
132	int connect_v4_prog(struct bpf_sock_addr *ctx)	148	int connect_v4_prog(struct bpf_sock_addr *ctx)
133	{	149	{
@@ -148,6 +164,9 @@ int connect_v4_prog(struct bpf_sock_addr *ctx) · 14 unmodified lines
148	if (set_keepalive(ctx))	164	if (set_keepalive(ctx))
149	return 0;	165	return 0;
150		166
		167	if (set_notsent_lowat(ctx))
		168	return 0;
		169
151	if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)	170	if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM)
152	return 0;	171	return 0;
153	else if (ctx->type == SOCK_STREAM)	172	else if (ctx->type == SOCK_STREAM)

tools/testing/selftests/bpf/progs/profiler.h+177−0

@@ -0,0 +1,177 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#pragma once
	4
	5	#define TASK_COMM_LEN 16
	6	#define MAX_ANCESTORS 4
	7	#define MAX_PATH 256
	8	#define KILL_TARGET_LEN 64
	9	#define CTL_MAXNAME 10
	10	#define MAX_ARGS_LEN 4096
	11	#define MAX_FILENAME_LEN 512
	12	#define MAX_ENVIRON_LEN 8192
	13	#define MAX_PATH_DEPTH 32
	14	#define MAX_FILEPATH_LENGTH (MAX_PATH_DEPTH * MAX_PATH)
	15	#define MAX_CGROUPS_PATH_DEPTH 8
	16
	17	#define MAX_METADATA_PAYLOAD_LEN TASK_COMM_LEN
	18
	19	#define MAX_CGROUP_PAYLOAD_LEN \
	20	(MAX_PATH * 2 + (MAX_PATH * MAX_CGROUPS_PATH_DEPTH))
	21
	22	#define MAX_CAP_PAYLOAD_LEN (MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN)
	23
	24	#define MAX_SYSCTL_PAYLOAD_LEN \
	25	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + CTL_MAXNAME + MAX_PATH)
	26
	27	#define MAX_KILL_PAYLOAD_LEN \
	28	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + TASK_COMM_LEN + \
	29	KILL_TARGET_LEN)
	30
	31	#define MAX_EXEC_PAYLOAD_LEN \
	32	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + MAX_FILENAME_LEN + \
	33	MAX_ARGS_LEN + MAX_ENVIRON_LEN)
	34
	35	#define MAX_FILEMOD_PAYLOAD_LEN \
	36	(MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN + MAX_FILEPATH_LENGTH + \
	37	MAX_FILEPATH_LENGTH)
	38
	39	enum data_type {
	40	INVALID_EVENT,
	41	EXEC_EVENT,
	42	FORK_EVENT,
	43	KILL_EVENT,
	44	SYSCTL_EVENT,
	45	FILEMOD_EVENT,
	46	MAX_DATA_TYPE_EVENT
	47	};
	48
	49	enum filemod_type {
	50	FMOD_OPEN,
	51	FMOD_LINK,
	52	FMOD_SYMLINK,
	53	};
	54
	55	struct ancestors_data_t {
	56	pid_t ancestor_pids[MAX_ANCESTORS];
	57	uint32_t ancestor_exec_ids[MAX_ANCESTORS];
	58	uint64_t ancestor_start_times[MAX_ANCESTORS];
	59	uint32_t num_ancestors;
	60	};
	61
	62	struct var_metadata_t {
	63	enum data_type type;
	64	pid_t pid;
	65	uint32_t exec_id;
	66	uid_t uid;
	67	gid_t gid;
	68	uint64_t start_time;
	69	uint32_t cpu_id;
	70	uint64_t bpf_stats_num_perf_events;
	71	uint64_t bpf_stats_start_ktime_ns;
	72	uint8_t comm_length;
	73	};
	74
	75	struct cgroup_data_t {
	76	ino_t cgroup_root_inode;
	77	ino_t cgroup_proc_inode;
	78	uint64_t cgroup_root_mtime;
	79	uint64_t cgroup_proc_mtime;
	80	uint16_t cgroup_root_length;
	81	uint16_t cgroup_proc_length;
	82	uint16_t cgroup_full_length;
	83	int cgroup_full_path_root_pos;
	84	};
	85
	86	struct var_sysctl_data_t {
	87	struct var_metadata_t meta;
	88	struct cgroup_data_t cgroup_data;
	89	struct ancestors_data_t ancestors_info;
	90	uint8_t sysctl_val_length;
	91	uint16_t sysctl_path_length;
	92	char payload[MAX_SYSCTL_PAYLOAD_LEN];
	93	};
	94
	95	struct var_kill_data_t {
	96	struct var_metadata_t meta;
	97	struct cgroup_data_t cgroup_data;
	98	struct ancestors_data_t ancestors_info;
	99	pid_t kill_target_pid;
	100	int kill_sig;
	101	uint32_t kill_count;
	102	uint64_t last_kill_time;
	103	uint8_t kill_target_name_length;
	104	uint8_t kill_target_cgroup_proc_length;
	105	char payload[MAX_KILL_PAYLOAD_LEN];
	106	size_t payload_length;
	107	};
	108
	109	struct var_exec_data_t {
	110	struct var_metadata_t meta;
	111	struct cgroup_data_t cgroup_data;
	112	pid_t parent_pid;
	113	uint32_t parent_exec_id;
	114	uid_t parent_uid;
	115	uint64_t parent_start_time;
	116	uint16_t bin_path_length;
	117	uint16_t cmdline_length;
	118	uint16_t environment_length;
	119	char payload[MAX_EXEC_PAYLOAD_LEN];
	120	};
	121
	122	struct var_fork_data_t {
	123	struct var_metadata_t meta;
	124	pid_t parent_pid;
	125	uint32_t parent_exec_id;
	126	uint64_t parent_start_time;
	127	char payload[MAX_METADATA_PAYLOAD_LEN];
	128	};
	129
	130	struct var_filemod_data_t {
	131	struct var_metadata_t meta;
	132	struct cgroup_data_t cgroup_data;
	133	enum filemod_type fmod_type;
	134	unsigned int dst_flags;
	135	uint32_t src_device_id;
	136	uint32_t dst_device_id;
	137	ino_t src_inode;
	138	ino_t dst_inode;
	139	uint16_t src_filepath_length;
	140	uint16_t dst_filepath_length;
	141	char payload[MAX_FILEMOD_PAYLOAD_LEN];
	142	};
	143
	144	struct profiler_config_struct {
	145	bool fetch_cgroups_from_bpf;
	146	ino_t cgroup_fs_inode;
	147	ino_t cgroup_login_session_inode;
	148	uint64_t kill_signals_mask;
	149	ino_t inode_filter;
	150	uint32_t stale_info_secs;
	151	bool use_variable_buffers;
	152	bool read_environ_from_exec;
	153	bool enable_cgroup_v1_resolver;
	154	};
	155
	156	struct bpf_func_stats_data {
	157	uint64_t time_elapsed_ns;
	158	uint64_t num_executions;
	159	uint64_t num_perf_events;
	160	};
	161
	162	struct bpf_func_stats_ctx {
	163	uint64_t start_time_ns;
	164	struct bpf_func_stats_data* bpf_func_stats_data_val;
	165	};
	166
	167	enum bpf_function_id {
	168	profiler_bpf_proc_sys_write,
	169	profiler_bpf_sched_process_exec,
	170	profiler_bpf_sched_process_exit,
	171	profiler_bpf_sys_enter_kill,
	172	profiler_bpf_do_filp_open_ret,
	173	profiler_bpf_sched_process_fork,
	174	profiler_bpf_vfs_link,
	175	profiler_bpf_vfs_symlink,
	176	profiler_bpf_max_function_id
	177	};

tools/testing/selftests/bpf/progs/profiler.inc.h+969−0

@@ -0,0 +1,969 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#include <vmlinux.h>
	4	#include <bpf/bpf_core_read.h>
	5	#include <bpf/bpf_helpers.h>
	6	#include <bpf/bpf_tracing.h>
	7
	8	#include "profiler.h"
	9
	10	#ifndef NULL
	11	#define NULL 0
	12	#endif
	13
	14	#define O_WRONLY 00000001
	15	#define O_RDWR 00000002
	16	#define O_DIRECTORY 00200000
	17	#define __O_TMPFILE 020000000
	18	#define O_TMPFILE (__O_TMPFILE \| O_DIRECTORY)
	19	#define MAX_ERRNO 4095
	20	#define S_IFMT 00170000
	21	#define S_IFSOCK 0140000
	22	#define S_IFLNK 0120000
	23	#define S_IFREG 0100000
	24	#define S_IFBLK 0060000
	25	#define S_IFDIR 0040000
	26	#define S_IFCHR 0020000
	27	#define S_IFIFO 0010000
	28	#define S_ISUID 0004000
	29	#define S_ISGID 0002000
	30	#define S_ISVTX 0001000
	31	#define S_ISLNK(m) (((m)&S_IFMT) == S_IFLNK)
	32	#define S_ISDIR(m) (((m)&S_IFMT) == S_IFDIR)
	33	#define S_ISCHR(m) (((m)&S_IFMT) == S_IFCHR)
	34	#define S_ISBLK(m) (((m)&S_IFMT) == S_IFBLK)
	35	#define S_ISFIFO(m) (((m)&S_IFMT) == S_IFIFO)
	36	#define S_ISSOCK(m) (((m)&S_IFMT) == S_IFSOCK)
	37	#define IS_ERR_VALUE(x) (unsigned long)(void*)(x) >= (unsigned long)-MAX_ERRNO
	38
	39	#define KILL_DATA_ARRAY_SIZE 8
	40
	41	struct var_kill_data_arr_t {
	42	struct var_kill_data_t array[KILL_DATA_ARRAY_SIZE];
	43	};
	44
	45	union any_profiler_data_t {
	46	struct var_exec_data_t var_exec;
	47	struct var_kill_data_t var_kill;
	48	struct var_sysctl_data_t var_sysctl;
	49	struct var_filemod_data_t var_filemod;
	50	struct var_fork_data_t var_fork;
	51	struct var_kill_data_arr_t var_kill_data_arr;
	52	};
	53
	54	volatile struct profiler_config_struct bpf_config = {};
	55
	56	#define FETCH_CGROUPS_FROM_BPF (bpf_config.fetch_cgroups_from_bpf)
	57	#define CGROUP_FS_INODE (bpf_config.cgroup_fs_inode)
	58	#define CGROUP_LOGIN_SESSION_INODE \
	59	(bpf_config.cgroup_login_session_inode)
	60	#define KILL_SIGNALS (bpf_config.kill_signals_mask)
	61	#define STALE_INFO (bpf_config.stale_info_secs)
	62	#define INODE_FILTER (bpf_config.inode_filter)
	63	#define READ_ENVIRON_FROM_EXEC (bpf_config.read_environ_from_exec)
	64	#define ENABLE_CGROUP_V1_RESOLVER (bpf_config.enable_cgroup_v1_resolver)
	65
	66	struct kernfs_iattrs___52 {
	67	struct iattr ia_iattr;
	68	};
	69
	70	struct kernfs_node___52 {
	71	union /* kernfs_node_id */ {
	72	struct {
	73	u32 ino;
	74	u32 generation;
	75	};
	76	u64 id;
	77	} id;
	78	};
	79
	80	struct {
	81	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	82	__uint(max_entries, 1);
	83	__type(key, u32);
	84	__type(value, union any_profiler_data_t);
	85	} data_heap SEC(".maps");
	86
	87	struct {
	88	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
	89	__uint(key_size, sizeof(int));
	90	__uint(value_size, sizeof(int));
	91	} events SEC(".maps");
	92
	93	struct {
	94	__uint(type, BPF_MAP_TYPE_HASH);
	95	__uint(max_entries, KILL_DATA_ARRAY_SIZE);
	96	__type(key, u32);
	97	__type(value, struct var_kill_data_arr_t);
	98	} var_tpid_to_data SEC(".maps");
	99
	100	struct {
	101	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	102	__uint(max_entries, profiler_bpf_max_function_id);
	103	__type(key, u32);
	104	__type(value, struct bpf_func_stats_data);
	105	} bpf_func_stats SEC(".maps");
	106
	107	struct {
	108	__uint(type, BPF_MAP_TYPE_HASH);
	109	__type(key, u32);
	110	__type(value, bool);
	111	__uint(max_entries, 16);
	112	} allowed_devices SEC(".maps");
	113
	114	struct {
	115	__uint(type, BPF_MAP_TYPE_HASH);
	116	__type(key, u64);
	117	__type(value, bool);
	118	__uint(max_entries, 1024);
	119	} allowed_file_inodes SEC(".maps");
	120
	121	struct {
	122	__uint(type, BPF_MAP_TYPE_HASH);
	123	__type(key, u64);
	124	__type(value, bool);
	125	__uint(max_entries, 1024);
	126	} allowed_directory_inodes SEC(".maps");
	127
	128	struct {
	129	__uint(type, BPF_MAP_TYPE_HASH);
	130	__type(key, u32);
	131	__type(value, bool);
	132	__uint(max_entries, 16);
	133	} disallowed_exec_inodes SEC(".maps");
	134
	135	#ifndef ARRAY_SIZE
	136	#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
	137	#endif
	138
	139	static INLINE bool IS_ERR(const void* ptr)
	140	{
	141	return IS_ERR_VALUE((unsigned long)ptr);
	142	}
	143
	144	static INLINE u32 get_userspace_pid()
	145	{
	146	return bpf_get_current_pid_tgid() >> 32;
	147	}
	148
	149	static INLINE bool is_init_process(u32 tgid)
	150	{
	151	return tgid == 1 \|\| tgid == 0;
	152	}
	153
	154	static INLINE unsigned long
	155	probe_read_lim(void* dst, void* src, unsigned long len, unsigned long max)
	156	{
	157	len = len < max ? len : max;
	158	if (len > 1) {
	159	if (bpf_probe_read(dst, len, src))
	160	return 0;
	161	} else if (len == 1) {
	162	if (bpf_probe_read(dst, 1, src))
	163	return 0;
	164	}
	165	return len;
	166	}
	167
	168	static INLINE int get_var_spid_index(struct var_kill_data_arr_t* arr_struct,
	169	int spid)
	170	{
	171	#ifdef UNROLL
	172	#pragma unroll
	173	#endif
	174	for (int i = 0; i < ARRAY_SIZE(arr_struct->array); i++)
	175	if (arr_struct->array[i].meta.pid == spid)
	176	return i;
	177	return -1;
	178	}
	179
	180	static INLINE void populate_ancestors(struct task_struct* task,
	181	struct ancestors_data_t* ancestors_data)
	182	{
	183	struct task_struct* parent = task;
	184	u32 num_ancestors, ppid;
	185
	186	ancestors_data->num_ancestors = 0;
	187	#ifdef UNROLL
	188	#pragma unroll
	189	#endif
	190	for (num_ancestors = 0; num_ancestors < MAX_ANCESTORS; num_ancestors++) {
	191	parent = BPF_CORE_READ(parent, real_parent);
	192	if (parent == NULL)
	193	break;
	194	ppid = BPF_CORE_READ(parent, tgid);
	195	if (is_init_process(ppid))
	196	break;
	197	ancestors_data->ancestor_pids[num_ancestors] = ppid;
	198	ancestors_data->ancestor_exec_ids[num_ancestors] =
	199	BPF_CORE_READ(parent, self_exec_id);
	200	ancestors_data->ancestor_start_times[num_ancestors] =
	201	BPF_CORE_READ(parent, start_time);
	202	ancestors_data->num_ancestors = num_ancestors;
	203	}
	204	}
	205
	206	static INLINE void* read_full_cgroup_path(struct kernfs_node* cgroup_node,
	207	struct kernfs_node* cgroup_root_node,
	208	void* payload,
	209	int* root_pos)
	210	{
	211	void* payload_start = payload;
	212	size_t filepart_length;
	213
	214	#ifdef UNROLL
	215	#pragma unroll
	216	#endif
	217	for (int i = 0; i < MAX_CGROUPS_PATH_DEPTH; i++) {
	218	filepart_length =
	219	bpf_probe_read_str(payload, MAX_PATH, BPF_CORE_READ(cgroup_node, name));
	220	if (!cgroup_node)
	221	return payload;
	222	if (cgroup_node == cgroup_root_node)
	223	*root_pos = payload - payload_start;
	224	if (filepart_length <= MAX_PATH) {
	225	barrier_var(filepart_length);
	226	payload += filepart_length;
	227	}
	228	cgroup_node = BPF_CORE_READ(cgroup_node, parent);
	229	}
	230	return payload;
	231	}
	232
	233	static ino_t get_inode_from_kernfs(struct kernfs_node* node)
	234	{
	235	struct kernfs_node___52* node52 = (void*)node;
	236
	237	if (bpf_core_field_exists(node52->id.ino)) {
	238	barrier_var(node52);
	239	return BPF_CORE_READ(node52, id.ino);
	240	} else {
	241	barrier_var(node);
	242	return (u64)BPF_CORE_READ(node, id);
	243	}
	244	}
	245
	246	int pids_cgrp_id = 1;
	247
	248	static INLINE void* populate_cgroup_info(struct cgroup_data_t* cgroup_data,
	249	struct task_struct* task,
	250	void* payload)
	251	{
	252	struct kernfs_node* root_kernfs =
	253	BPF_CORE_READ(task, nsproxy, cgroup_ns, root_cset, dfl_cgrp, kn);
	254	struct kernfs_node* proc_kernfs = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn);
	255
	256	if (ENABLE_CGROUP_V1_RESOLVER) {
	257	#ifdef UNROLL
	258	#pragma unroll
	259	#endif
	260	for (int i = 0; i < CGROUP_SUBSYS_COUNT; i++) {
	261	struct cgroup_subsys_state* subsys =
	262	BPF_CORE_READ(task, cgroups, subsys[i]);
	263	if (subsys != NULL) {
	264	int subsys_id = BPF_CORE_READ(subsys, ss, id);
	265	if (subsys_id == pids_cgrp_id) {
	266	proc_kernfs = BPF_CORE_READ(subsys, cgroup, kn);
	267	root_kernfs = BPF_CORE_READ(subsys, ss, root, kf_root, kn);
	268	break;
	269	}
	270	}
	271	}
	272	}
	273
	274	cgroup_data->cgroup_root_inode = get_inode_from_kernfs(root_kernfs);
	275	cgroup_data->cgroup_proc_inode = get_inode_from_kernfs(proc_kernfs);
	276
	277	if (bpf_core_field_exists(root_kernfs->iattr->ia_mtime)) {
	278	cgroup_data->cgroup_root_mtime =
	279	BPF_CORE_READ(root_kernfs, iattr, ia_mtime.tv_nsec);
	280	cgroup_data->cgroup_proc_mtime =
	281	BPF_CORE_READ(proc_kernfs, iattr, ia_mtime.tv_nsec);
	282	} else {
	283	struct kernfs_iattrs___52* root_iattr =
	284	(struct kernfs_iattrs___52*)BPF_CORE_READ(root_kernfs, iattr);
	285	cgroup_data->cgroup_root_mtime =
	286	BPF_CORE_READ(root_iattr, ia_iattr.ia_mtime.tv_nsec);
	287
	288	struct kernfs_iattrs___52* proc_iattr =
	289	(struct kernfs_iattrs___52*)BPF_CORE_READ(proc_kernfs, iattr);
	290	cgroup_data->cgroup_proc_mtime =
	291	BPF_CORE_READ(proc_iattr, ia_iattr.ia_mtime.tv_nsec);
	292	}
	293
	294	cgroup_data->cgroup_root_length = 0;
	295	cgroup_data->cgroup_proc_length = 0;
	296	cgroup_data->cgroup_full_length = 0;
	297
	298	size_t cgroup_root_length =
	299	bpf_probe_read_str(payload, MAX_PATH, BPF_CORE_READ(root_kernfs, name));
	300	barrier_var(cgroup_root_length);
	301	if (cgroup_root_length <= MAX_PATH) {
	302	barrier_var(cgroup_root_length);
	303	cgroup_data->cgroup_root_length = cgroup_root_length;
	304	payload += cgroup_root_length;
	305	}
	306
	307	size_t cgroup_proc_length =
	308	bpf_probe_read_str(payload, MAX_PATH, BPF_CORE_READ(proc_kernfs, name));
	309	barrier_var(cgroup_proc_length);
	310	if (cgroup_proc_length <= MAX_PATH) {
	311	barrier_var(cgroup_proc_length);
	312	cgroup_data->cgroup_proc_length = cgroup_proc_length;
	313	payload += cgroup_proc_length;
	314	}
	315
	316	if (FETCH_CGROUPS_FROM_BPF) {
	317	cgroup_data->cgroup_full_path_root_pos = -1;
	318	void* payload_end_pos = read_full_cgroup_path(proc_kernfs, root_kernfs, payload,
	319	&cgroup_data->cgroup_full_path_root_pos);
	320	cgroup_data->cgroup_full_length = payload_end_pos - payload;
	321	payload = payload_end_pos;
	322	}
	323
	324	return (void*)payload;
	325	}
	326
	327	static INLINE void* populate_var_metadata(struct var_metadata_t* metadata,
	328	struct task_struct* task,
	329	u32 pid, void* payload)
	330	{
	331	u64 uid_gid = bpf_get_current_uid_gid();
	332
	333	metadata->uid = (u32)uid_gid;
	334	metadata->gid = uid_gid >> 32;
	335	metadata->pid = pid;
	336	metadata->exec_id = BPF_CORE_READ(task, self_exec_id);
	337	metadata->start_time = BPF_CORE_READ(task, start_time);
	338	metadata->comm_length = 0;
	339
	340	size_t comm_length = bpf_core_read_str(payload, TASK_COMM_LEN, &task->comm);
	341	barrier_var(comm_length);
	342	if (comm_length <= TASK_COMM_LEN) {
	343	barrier_var(comm_length);
	344	metadata->comm_length = comm_length;
	345	payload += comm_length;
	346	}
	347
	348	return (void*)payload;
	349	}
	350
	351	static INLINE struct var_kill_data_t*
	352	get_var_kill_data(struct pt_regs* ctx, int spid, int tpid, int sig)
	353	{
	354	int zero = 0;
	355	struct var_kill_data_t* kill_data = bpf_map_lookup_elem(&data_heap, &zero);
	356
	357	if (kill_data == NULL)
	358	return NULL;
	359	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	360
	361	void* payload = populate_var_metadata(&kill_data->meta, task, spid, kill_data->payload);
	362	payload = populate_cgroup_info(&kill_data->cgroup_data, task, payload);
	363	size_t payload_length = payload - (void*)kill_data->payload;
	364	kill_data->payload_length = payload_length;
	365	populate_ancestors(task, &kill_data->ancestors_info);
	366	kill_data->meta.type = KILL_EVENT;
	367	kill_data->kill_target_pid = tpid;
	368	kill_data->kill_sig = sig;
	369	kill_data->kill_count = 1;
	370	kill_data->last_kill_time = bpf_ktime_get_ns();
	371	return kill_data;
	372	}
	373
	374	static INLINE int trace_var_sys_kill(void* ctx, int tpid, int sig)
	375	{
	376	if ((KILL_SIGNALS & (1ULL << sig)) == 0)
	377	return 0;
	378
	379	u32 spid = get_userspace_pid();
	380	struct var_kill_data_arr_t* arr_struct = bpf_map_lookup_elem(&var_tpid_to_data, &tpid);
	381
	382	if (arr_struct == NULL) {
	383	struct var_kill_data_t* kill_data = get_var_kill_data(ctx, spid, tpid, sig);
	384	int zero = 0;
	385
	386	if (kill_data == NULL)
	387	return 0;
	388	arr_struct = bpf_map_lookup_elem(&data_heap, &zero);
	389	if (arr_struct == NULL)
	390	return 0;
	391	bpf_probe_read(&arr_struct->array[0], sizeof(arr_struct->array[0]), kill_data);
	392	} else {
	393	int index = get_var_spid_index(arr_struct, spid);
	394
	395	if (index == -1) {
	396	struct var_kill_data_t* kill_data =
	397	get_var_kill_data(ctx, spid, tpid, sig);
	398	if (kill_data == NULL)
	399	return 0;
	400	#ifdef UNROLL
	401	#pragma unroll
	402	#endif
	403	for (int i = 0; i < ARRAY_SIZE(arr_struct->array); i++)
	404	if (arr_struct->array[i].meta.pid == 0) {
	405	bpf_probe_read(&arr_struct->array[i],
	406	sizeof(arr_struct->array[i]), kill_data);
	407	bpf_map_update_elem(&var_tpid_to_data, &tpid,
	408	arr_struct, 0);
	409
	410	return 0;
	411	}
	412	return 0;
	413	}
	414
	415	struct var_kill_data_t* kill_data = &arr_struct->array[index];
	416
	417	u64 delta_sec =
	418	(bpf_ktime_get_ns() - kill_data->last_kill_time) / 1000000000;
	419
	420	if (delta_sec < STALE_INFO) {
	421	kill_data->kill_count++;
	422	kill_data->last_kill_time = bpf_ktime_get_ns();
	423	bpf_probe_read(&arr_struct->array[index],
	424	sizeof(arr_struct->array[index]),
	425	kill_data);
	426	} else {
	427	struct var_kill_data_t* kill_data =
	428	get_var_kill_data(ctx, spid, tpid, sig);
	429	if (kill_data == NULL)
	430	return 0;
	431	bpf_probe_read(&arr_struct->array[index],
	432	sizeof(arr_struct->array[index]),
	433	kill_data);
	434	}
	435	}
	436	bpf_map_update_elem(&var_tpid_to_data, &tpid, arr_struct, 0);
	437	return 0;
	438	}
	439
	440	static INLINE void bpf_stats_enter(struct bpf_func_stats_ctx* bpf_stat_ctx,
	441	enum bpf_function_id func_id)
	442	{
	443	int func_id_key = func_id;
	444
	445	bpf_stat_ctx->start_time_ns = bpf_ktime_get_ns();
	446	bpf_stat_ctx->bpf_func_stats_data_val =
	447	bpf_map_lookup_elem(&bpf_func_stats, &func_id_key);
	448	if (bpf_stat_ctx->bpf_func_stats_data_val)
	449	bpf_stat_ctx->bpf_func_stats_data_val->num_executions++;
	450	}
	451
	452	static INLINE void bpf_stats_exit(struct bpf_func_stats_ctx* bpf_stat_ctx)
	453	{
	454	if (bpf_stat_ctx->bpf_func_stats_data_val)
	455	bpf_stat_ctx->bpf_func_stats_data_val->time_elapsed_ns +=
	456	bpf_ktime_get_ns() - bpf_stat_ctx->start_time_ns;
	457	}
	458
	459	static INLINE void
	460	bpf_stats_pre_submit_var_perf_event(struct bpf_func_stats_ctx* bpf_stat_ctx,
	461	struct var_metadata_t* meta)
	462	{
	463	if (bpf_stat_ctx->bpf_func_stats_data_val) {
	464	bpf_stat_ctx->bpf_func_stats_data_val->num_perf_events++;
	465	meta->bpf_stats_num_perf_events =
	466	bpf_stat_ctx->bpf_func_stats_data_val->num_perf_events;
	467	}
	468	meta->bpf_stats_start_ktime_ns = bpf_stat_ctx->start_time_ns;
	469	meta->cpu_id = bpf_get_smp_processor_id();
	470	}
	471
	472	static INLINE size_t
	473	read_absolute_file_path_from_dentry(struct dentry* filp_dentry, void* payload)
	474	{
	475	size_t length = 0;
	476	size_t filepart_length;
	477	struct dentry* parent_dentry;
	478
	479	#ifdef UNROLL
	480	#pragma unroll
	481	#endif
	482	for (int i = 0; i < MAX_PATH_DEPTH; i++) {
	483	filepart_length = bpf_probe_read_str(payload, MAX_PATH,
	484	BPF_CORE_READ(filp_dentry, d_name.name));
	485	barrier_var(filepart_length);
	486	if (filepart_length > MAX_PATH)
	487	break;
	488	barrier_var(filepart_length);
	489	payload += filepart_length;
	490	length += filepart_length;
	491
	492	parent_dentry = BPF_CORE_READ(filp_dentry, d_parent);
	493	if (filp_dentry == parent_dentry)
	494	break;
	495	filp_dentry = parent_dentry;
	496	}
	497
	498	return length;
	499	}
	500
	501	static INLINE bool
	502	is_ancestor_in_allowed_inodes(struct dentry* filp_dentry)
	503	{
	504	struct dentry* parent_dentry;
	505	#ifdef UNROLL
	506	#pragma unroll
	507	#endif
	508	for (int i = 0; i < MAX_PATH_DEPTH; i++) {
	509	u64 dir_ino = BPF_CORE_READ(filp_dentry, d_inode, i_ino);
	510	bool* allowed_dir = bpf_map_lookup_elem(&allowed_directory_inodes, &dir_ino);
	511
	512	if (allowed_dir != NULL)
	513	return true;
	514	parent_dentry = BPF_CORE_READ(filp_dentry, d_parent);
	515	if (filp_dentry == parent_dentry)
	516	break;
	517	filp_dentry = parent_dentry;
	518	}
	519	return false;
	520	}
	521
	522	static INLINE bool is_dentry_allowed_for_filemod(struct dentry* file_dentry,
	523	u32* device_id,
	524	u64* file_ino)
	525	{
	526	u32 dev_id = BPF_CORE_READ(file_dentry, d_sb, s_dev);
	527	*device_id = dev_id;
	528	bool* allowed_device = bpf_map_lookup_elem(&allowed_devices, &dev_id);
	529
	530	if (allowed_device == NULL)
	531	return false;
	532
	533	u64 ino = BPF_CORE_READ(file_dentry, d_inode, i_ino);
	534	*file_ino = ino;
	535	bool* allowed_file = bpf_map_lookup_elem(&allowed_file_inodes, &ino);
	536
	537	if (allowed_file == NULL)
	538	if (!is_ancestor_in_allowed_inodes(BPF_CORE_READ(file_dentry, d_parent)))
	539	return false;
	540	return true;
	541	}
	542
	543	SEC("kprobe/proc_sys_write")
	544	ssize_t BPF_KPROBE(kprobe__proc_sys_write,
	545	struct file* filp, const char* buf,
	546	size_t count, loff_t* ppos)
	547	{
	548	struct bpf_func_stats_ctx stats_ctx;
	549	bpf_stats_enter(&stats_ctx, profiler_bpf_proc_sys_write);
	550
	551	u32 pid = get_userspace_pid();
	552	int zero = 0;
	553	struct var_sysctl_data_t* sysctl_data =
	554	bpf_map_lookup_elem(&data_heap, &zero);
	555	if (!sysctl_data)
	556	goto out;
	557
	558	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	559	sysctl_data->meta.type = SYSCTL_EVENT;
	560	void* payload = populate_var_metadata(&sysctl_data->meta, task, pid, sysctl_data->payload);
	561	payload = populate_cgroup_info(&sysctl_data->cgroup_data, task, payload);
	562
	563	populate_ancestors(task, &sysctl_data->ancestors_info);
	564
	565	sysctl_data->sysctl_val_length = 0;
	566	sysctl_data->sysctl_path_length = 0;
	567
	568	size_t sysctl_val_length = bpf_probe_read_str(payload, CTL_MAXNAME, buf);
	569	barrier_var(sysctl_val_length);
	570	if (sysctl_val_length <= CTL_MAXNAME) {
	571	barrier_var(sysctl_val_length);
	572	sysctl_data->sysctl_val_length = sysctl_val_length;
	573	payload += sysctl_val_length;
	574	}
	575
	576	size_t sysctl_path_length = bpf_probe_read_str(payload, MAX_PATH,
	577	BPF_CORE_READ(filp, f_path.dentry, d_name.name));
	578	barrier_var(sysctl_path_length);
	579	if (sysctl_path_length <= MAX_PATH) {
	580	barrier_var(sysctl_path_length);
	581	sysctl_data->sysctl_path_length = sysctl_path_length;
	582	payload += sysctl_path_length;
	583	}
	584
	585	bpf_stats_pre_submit_var_perf_event(&stats_ctx, &sysctl_data->meta);
	586	unsigned long data_len = payload - (void*)sysctl_data;
	587	data_len = data_len > sizeof(struct var_sysctl_data_t)
	588	? sizeof(struct var_sysctl_data_t)
	589	: data_len;
	590	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, sysctl_data, data_len);
	591	out:
	592	bpf_stats_exit(&stats_ctx);
	593	return 0;
	594	}
	595
	596	SEC("tracepoint/syscalls/sys_enter_kill")
	597	int tracepoint__syscalls__sys_enter_kill(struct trace_event_raw_sys_enter* ctx)
	598	{
	599	struct bpf_func_stats_ctx stats_ctx;
	600
	601	bpf_stats_enter(&stats_ctx, profiler_bpf_sys_enter_kill);
	602	int pid = ctx->args[0];
	603	int sig = ctx->args[1];
	604	int ret = trace_var_sys_kill(ctx, pid, sig);
	605	bpf_stats_exit(&stats_ctx);
	606	return ret;
	607	};
	608
	609	SEC("raw_tracepoint/sched_process_exit")
	610	int raw_tracepoint__sched_process_exit(void* ctx)
	611	{
	612	int zero = 0;
	613	struct bpf_func_stats_ctx stats_ctx;
	614	bpf_stats_enter(&stats_ctx, profiler_bpf_sched_process_exit);
	615
	616	u32 tpid = get_userspace_pid();
	617
	618	struct var_kill_data_arr_t* arr_struct = bpf_map_lookup_elem(&var_tpid_to_data, &tpid);
	619	struct var_kill_data_t* kill_data = bpf_map_lookup_elem(&data_heap, &zero);
	620
	621	if (arr_struct == NULL \|\| kill_data == NULL)
	622	goto out;
	623
	624	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	625	struct kernfs_node* proc_kernfs = BPF_CORE_READ(task, cgroups, dfl_cgrp, kn);
	626
	627	#ifdef UNROLL
	628	#pragma unroll
	629	#endif
	630	for (int i = 0; i < ARRAY_SIZE(arr_struct->array); i++) {
	631	struct var_kill_data_t* past_kill_data = &arr_struct->array[i];
	632
	633	if (past_kill_data != NULL && past_kill_data->kill_target_pid == tpid) {
	634	bpf_probe_read(kill_data, sizeof(*past_kill_data), past_kill_data);
	635	void* payload = kill_data->payload;
	636	size_t offset = kill_data->payload_length;
	637	if (offset >= MAX_METADATA_PAYLOAD_LEN + MAX_CGROUP_PAYLOAD_LEN)
	638	return 0;
	639	payload += offset;
	640
	641	kill_data->kill_target_name_length = 0;
	642	kill_data->kill_target_cgroup_proc_length = 0;
	643
	644	size_t comm_length = bpf_core_read_str(payload, TASK_COMM_LEN, &task->comm);
	645	barrier_var(comm_length);
	646	if (comm_length <= TASK_COMM_LEN) {
	647	barrier_var(comm_length);
	648	kill_data->kill_target_name_length = comm_length;
	649	payload += comm_length;
	650	}
	651
	652	size_t cgroup_proc_length = bpf_probe_read_str(payload, KILL_TARGET_LEN,
	653	BPF_CORE_READ(proc_kernfs, name));
	654	barrier_var(cgroup_proc_length);
	655	if (cgroup_proc_length <= KILL_TARGET_LEN) {
	656	barrier_var(cgroup_proc_length);
	657	kill_data->kill_target_cgroup_proc_length = cgroup_proc_length;
	658	payload += cgroup_proc_length;
	659	}
	660
	661	bpf_stats_pre_submit_var_perf_event(&stats_ctx, &kill_data->meta);
	662	unsigned long data_len = (void)payload - (void)kill_data;
	663	data_len = data_len > sizeof(struct var_kill_data_t)
	664	? sizeof(struct var_kill_data_t)
	665	: data_len;
	666	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, kill_data, data_len);
	667	}
	668	}
	669	bpf_map_delete_elem(&var_tpid_to_data, &tpid);
	670	out:
	671	bpf_stats_exit(&stats_ctx);
	672	return 0;
	673	}
	674
	675	SEC("raw_tracepoint/sched_process_exec")
	676	int raw_tracepoint__sched_process_exec(struct bpf_raw_tracepoint_args* ctx)
	677	{
	678	struct bpf_func_stats_ctx stats_ctx;
	679	bpf_stats_enter(&stats_ctx, profiler_bpf_sched_process_exec);
	680
	681	struct linux_binprm* bprm = (struct linux_binprm*)ctx->args[2];
	682	u64 inode = BPF_CORE_READ(bprm, file, f_inode, i_ino);
	683
	684	bool* should_filter_binprm = bpf_map_lookup_elem(&disallowed_exec_inodes, &inode);
	685	if (should_filter_binprm != NULL)
	686	goto out;
	687
	688	int zero = 0;
	689	struct var_exec_data_t* proc_exec_data = bpf_map_lookup_elem(&data_heap, &zero);
	690	if (!proc_exec_data)
	691	goto out;
	692
	693	if (INODE_FILTER && inode != INODE_FILTER)
	694	return 0;
	695
	696	u32 pid = get_userspace_pid();
	697	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	698
	699	proc_exec_data->meta.type = EXEC_EVENT;
	700	proc_exec_data->bin_path_length = 0;
	701	proc_exec_data->cmdline_length = 0;
	702	proc_exec_data->environment_length = 0;
	703	void* payload = populate_var_metadata(&proc_exec_data->meta, task, pid,
	704	proc_exec_data->payload);
	705	payload = populate_cgroup_info(&proc_exec_data->cgroup_data, task, payload);
	706
	707	struct task_struct* parent_task = BPF_CORE_READ(task, real_parent);
	708	proc_exec_data->parent_pid = BPF_CORE_READ(parent_task, tgid);
	709	proc_exec_data->parent_uid = BPF_CORE_READ(parent_task, real_cred, uid.val);
	710	proc_exec_data->parent_exec_id = BPF_CORE_READ(parent_task, self_exec_id);
	711	proc_exec_data->parent_start_time = BPF_CORE_READ(parent_task, start_time);
	712
	713	const char* filename = BPF_CORE_READ(bprm, filename);
	714	size_t bin_path_length = bpf_probe_read_str(payload, MAX_FILENAME_LEN, filename);
	715	barrier_var(bin_path_length);
	716	if (bin_path_length <= MAX_FILENAME_LEN) {
	717	barrier_var(bin_path_length);
	718	proc_exec_data->bin_path_length = bin_path_length;
	719	payload += bin_path_length;
	720	}
	721
	722	void* arg_start = (void*)BPF_CORE_READ(task, mm, arg_start);
	723	void* arg_end = (void*)BPF_CORE_READ(task, mm, arg_end);
	724	unsigned int cmdline_length = probe_read_lim(payload, arg_start,
	725	arg_end - arg_start, MAX_ARGS_LEN);
	726
	727	if (cmdline_length <= MAX_ARGS_LEN) {
	728	barrier_var(cmdline_length);
	729	proc_exec_data->cmdline_length = cmdline_length;
	730	payload += cmdline_length;
	731	}
	732
	733	if (READ_ENVIRON_FROM_EXEC) {
	734	void* env_start = (void*)BPF_CORE_READ(task, mm, env_start);
	735	void* env_end = (void*)BPF_CORE_READ(task, mm, env_end);
	736	unsigned long env_len = probe_read_lim(payload, env_start,
	737	env_end - env_start, MAX_ENVIRON_LEN);
	738	if (cmdline_length <= MAX_ENVIRON_LEN) {
	739	proc_exec_data->environment_length = env_len;
	740	payload += env_len;
	741	}
	742	}
	743
	744	bpf_stats_pre_submit_var_perf_event(&stats_ctx, &proc_exec_data->meta);
	745	unsigned long data_len = payload - (void*)proc_exec_data;
	746	data_len = data_len > sizeof(struct var_exec_data_t)
	747	? sizeof(struct var_exec_data_t)
	748	: data_len;
	749	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, proc_exec_data, data_len);
	750	out:
	751	bpf_stats_exit(&stats_ctx);
	752	return 0;
	753	}
	754
	755	SEC("kretprobe/do_filp_open")
	756	int kprobe_ret__do_filp_open(struct pt_regs* ctx)
	757	{
	758	struct bpf_func_stats_ctx stats_ctx;
	759	bpf_stats_enter(&stats_ctx, profiler_bpf_do_filp_open_ret);
	760
	761	struct file* filp = (struct file*)PT_REGS_RC_CORE(ctx);
	762
	763	if (filp == NULL \|\| IS_ERR(filp))
	764	goto out;
	765	unsigned int flags = BPF_CORE_READ(filp, f_flags);
	766	if ((flags & (O_RDWR \| O_WRONLY)) == 0)
	767	goto out;
	768	if ((flags & O_TMPFILE) > 0)
	769	goto out;
	770	struct inode* file_inode = BPF_CORE_READ(filp, f_inode);
	771	umode_t mode = BPF_CORE_READ(file_inode, i_mode);
	772	if (S_ISDIR(mode) \|\| S_ISCHR(mode) \|\| S_ISBLK(mode) \|\| S_ISFIFO(mode) \|\|
	773	S_ISSOCK(mode))
	774	goto out;
	775
	776	struct dentry* filp_dentry = BPF_CORE_READ(filp, f_path.dentry);
	777	u32 device_id = 0;
	778	u64 file_ino = 0;
	779	if (!is_dentry_allowed_for_filemod(filp_dentry, &device_id, &file_ino))
	780	goto out;
	781
	782	int zero = 0;
	783	struct var_filemod_data_t* filemod_data = bpf_map_lookup_elem(&data_heap, &zero);
	784	if (!filemod_data)
	785	goto out;
	786
	787	u32 pid = get_userspace_pid();
	788	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	789
	790	filemod_data->meta.type = FILEMOD_EVENT;
	791	filemod_data->fmod_type = FMOD_OPEN;
	792	filemod_data->dst_flags = flags;
	793	filemod_data->src_inode = 0;
	794	filemod_data->dst_inode = file_ino;
	795	filemod_data->src_device_id = 0;
	796	filemod_data->dst_device_id = device_id;
	797	filemod_data->src_filepath_length = 0;
	798	filemod_data->dst_filepath_length = 0;
	799
	800	void* payload = populate_var_metadata(&filemod_data->meta, task, pid,
	801	filemod_data->payload);
	802	payload = populate_cgroup_info(&filemod_data->cgroup_data, task, payload);
	803
	804	size_t len = read_absolute_file_path_from_dentry(filp_dentry, payload);
	805	barrier_var(len);
	806	if (len <= MAX_FILEPATH_LENGTH) {
	807	barrier_var(len);
	808	payload += len;
	809	filemod_data->dst_filepath_length = len;
	810	}
	811	bpf_stats_pre_submit_var_perf_event(&stats_ctx, &filemod_data->meta);
	812	unsigned long data_len = payload - (void*)filemod_data;
	813	data_len = data_len > sizeof(filemod_data) ? sizeof(filemod_data) : data_len;
	814	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, filemod_data, data_len);
	815	out:
	816	bpf_stats_exit(&stats_ctx);
	817	return 0;
	818	}
	819
	820	SEC("kprobe/vfs_link")
	821	int BPF_KPROBE(kprobe__vfs_link,
	822	struct dentry* old_dentry, struct inode* dir,
	823	struct dentry* new_dentry, struct inode** delegated_inode)
	824	{
	825	struct bpf_func_stats_ctx stats_ctx;
	826	bpf_stats_enter(&stats_ctx, profiler_bpf_vfs_link);
	827
	828	u32 src_device_id = 0;
	829	u64 src_file_ino = 0;
	830	u32 dst_device_id = 0;
	831	u64 dst_file_ino = 0;
	832	if (!is_dentry_allowed_for_filemod(old_dentry, &src_device_id, &src_file_ino) &&
	833	!is_dentry_allowed_for_filemod(new_dentry, &dst_device_id, &dst_file_ino))
	834	goto out;
	835
	836	int zero = 0;
	837	struct var_filemod_data_t* filemod_data = bpf_map_lookup_elem(&data_heap, &zero);
	838	if (!filemod_data)
	839	goto out;
	840
	841	u32 pid = get_userspace_pid();
	842	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	843
	844	filemod_data->meta.type = FILEMOD_EVENT;
	845	filemod_data->fmod_type = FMOD_LINK;
	846	filemod_data->dst_flags = 0;
	847	filemod_data->src_inode = src_file_ino;
	848	filemod_data->dst_inode = dst_file_ino;
	849	filemod_data->src_device_id = src_device_id;
	850	filemod_data->dst_device_id = dst_device_id;
	851	filemod_data->src_filepath_length = 0;
	852	filemod_data->dst_filepath_length = 0;
	853
	854	void* payload = populate_var_metadata(&filemod_data->meta, task, pid,
	855	filemod_data->payload);
	856	payload = populate_cgroup_info(&filemod_data->cgroup_data, task, payload);
	857
	858	size_t len = read_absolute_file_path_from_dentry(old_dentry, payload);
	859	barrier_var(len);
	860	if (len <= MAX_FILEPATH_LENGTH) {
	861	barrier_var(len);
	862	payload += len;
	863	filemod_data->src_filepath_length = len;
	864	}
	865
	866	len = read_absolute_file_path_from_dentry(new_dentry, payload);
	867	barrier_var(len);
	868	if (len <= MAX_FILEPATH_LENGTH) {
	869	barrier_var(len);
	870	payload += len;
	871	filemod_data->dst_filepath_length = len;
	872	}
	873
	874	bpf_stats_pre_submit_var_perf_event(&stats_ctx, &filemod_data->meta);
	875	unsigned long data_len = payload - (void*)filemod_data;
	876	data_len = data_len > sizeof(filemod_data) ? sizeof(filemod_data) : data_len;
	877	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, filemod_data, data_len);
	878	out:
	879	bpf_stats_exit(&stats_ctx);
	880	return 0;
	881	}
	882
	883	SEC("kprobe/vfs_symlink")
	884	int BPF_KPROBE(kprobe__vfs_symlink, struct inode* dir, struct dentry* dentry,
	885	const char* oldname)
	886	{
	887	struct bpf_func_stats_ctx stats_ctx;
	888	bpf_stats_enter(&stats_ctx, profiler_bpf_vfs_symlink);
	889
	890	u32 dst_device_id = 0;
	891	u64 dst_file_ino = 0;
	892	if (!is_dentry_allowed_for_filemod(dentry, &dst_device_id, &dst_file_ino))
	893	goto out;
	894
	895	int zero = 0;
	896	struct var_filemod_data_t* filemod_data = bpf_map_lookup_elem(&data_heap, &zero);
	897	if (!filemod_data)
	898	goto out;
	899
	900	u32 pid = get_userspace_pid();
	901	struct task_struct* task = (struct task_struct*)bpf_get_current_task();
	902
	903	filemod_data->meta.type = FILEMOD_EVENT;
	904	filemod_data->fmod_type = FMOD_SYMLINK;
	905	filemod_data->dst_flags = 0;
	906	filemod_data->src_inode = 0;
	907	filemod_data->dst_inode = dst_file_ino;
	908	filemod_data->src_device_id = 0;
	909	filemod_data->dst_device_id = dst_device_id;
	910	filemod_data->src_filepath_length = 0;
	911	filemod_data->dst_filepath_length = 0;
	912
	913	void* payload = populate_var_metadata(&filemod_data->meta, task, pid,
	914	filemod_data->payload);
	915	payload = populate_cgroup_info(&filemod_data->cgroup_data, task, payload);
	916
	917	size_t len = bpf_probe_read_str(payload, MAX_FILEPATH_LENGTH, oldname);
	918	barrier_var(len);
	919	if (len <= MAX_FILEPATH_LENGTH) {
	920	barrier_var(len);
	921	payload += len;
	922	filemod_data->src_filepath_length = len;
	923	}
	924	len = read_absolute_file_path_from_dentry(dentry, payload);
	925	barrier_var(len);
	926	if (len <= MAX_FILEPATH_LENGTH) {
	927	barrier_var(len);
	928	payload += len;
	929	filemod_data->dst_filepath_length = len;
	930	}
	931	bpf_stats_pre_submit_var_perf_event(&stats_ctx, &filemod_data->meta);
	932	unsigned long data_len = payload - (void*)filemod_data;
	933	data_len = data_len > sizeof(filemod_data) ? sizeof(filemod_data) : data_len;
	934	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, filemod_data, data_len);
	935	out:
	936	bpf_stats_exit(&stats_ctx);
	937	return 0;
	938	}
	939
	940	SEC("raw_tracepoint/sched_process_fork")
	941	int raw_tracepoint__sched_process_fork(struct bpf_raw_tracepoint_args* ctx)
	942	{
	943	struct bpf_func_stats_ctx stats_ctx;
	944	bpf_stats_enter(&stats_ctx, profiler_bpf_sched_process_fork);
	945
	946	int zero = 0;
	947	struct var_fork_data_t* fork_data = bpf_map_lookup_elem(&data_heap, &zero);
	948	if (!fork_data)
	949	goto out;
	950
	951	struct task_struct* parent = (struct task_struct*)ctx->args[0];
	952	struct task_struct* child = (struct task_struct*)ctx->args[1];
	953	fork_data->meta.type = FORK_EVENT;
	954
	955	void* payload = populate_var_metadata(&fork_data->meta, child,
	956	BPF_CORE_READ(child, pid), fork_data->payload);
	957	fork_data->parent_pid = BPF_CORE_READ(parent, pid);
	958	fork_data->parent_exec_id = BPF_CORE_READ(parent, self_exec_id);
	959	fork_data->parent_start_time = BPF_CORE_READ(parent, start_time);
	960	bpf_stats_pre_submit_var_perf_event(&stats_ctx, &fork_data->meta);
	961
	962	unsigned long data_len = payload - (void*)fork_data;
	963	data_len = data_len > sizeof(fork_data) ? sizeof(fork_data) : data_len;
	964	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU, fork_data, data_len);
	965	out:
	966	bpf_stats_exit(&stats_ctx);
	967	return 0;
	968	}
	969	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/profiler1.c+6−0

@@ -0,0 +1,6 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#define barrier_var(var) asm volatile("" : "=r"(var) : "0"(var))
	4	#define UNROLL
	5	#define INLINE __always_inline
	6	#include "profiler.inc.h"

tools/testing/selftests/bpf/progs/profiler2.c+6−0

@@ -0,0 +1,6 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#define barrier_var(var) /**/
	4	/* undef #define UNROLL */
	5	#define INLINE /**/
	6	#include "profiler.inc.h"

tools/testing/selftests/bpf/progs/profiler3.c+6−0

@@ -0,0 +1,6 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#define barrier_var(var) /**/
	4	#define UNROLL
	5	#define INLINE __noinline
	6	#include "profiler.inc.h"

tools/testing/selftests/bpf/progs/test_btf_map_in_map.c+43−0

@@ -41,6 +41,43 @@ struct outer_arr { · 40 unmodified lines
41	.values = { (void )&inner_map1, 0, (void )&inner_map2 },	41	.values = { (void )&inner_map1, 0, (void )&inner_map2 },
42	};	42	};
43		43
		44	struct inner_map_sz3 {
		45	__uint(type, BPF_MAP_TYPE_ARRAY);
		46	__uint(map_flags, BPF_F_INNER_MAP);
		47	__uint(max_entries, 3);
		48	__type(key, int);
		49	__type(value, int);
		50	} inner_map3 SEC(".maps"),
		51	inner_map4 SEC(".maps");
		52
		53	struct inner_map_sz4 {
		54	__uint(type, BPF_MAP_TYPE_ARRAY);
		55	__uint(map_flags, BPF_F_INNER_MAP);
		56	__uint(max_entries, 5);
		57	__type(key, int);
		58	__type(value, int);
		59	} inner_map5 SEC(".maps");
		60
		61	struct outer_arr_dyn {
		62	__uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
		63	__uint(max_entries, 3);
		64	__uint(key_size, sizeof(int));
		65	__uint(value_size, sizeof(int));
		66	__array(values, struct {
		67	__uint(type, BPF_MAP_TYPE_ARRAY);
		68	__uint(map_flags, BPF_F_INNER_MAP);
		69	__uint(max_entries, 1);
		70	__type(key, int);
		71	__type(value, int);
		72	});
		73	} outer_arr_dyn SEC(".maps") = {
		74	.values = {
		75	[0] = (void *)&inner_map3,
		76	[1] = (void *)&inner_map4,
		77	[2] = (void *)&inner_map5,
		78	},
		79	};
		80
44	struct outer_hash {	81	struct outer_hash {
45	__uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);	82	__uint(type, BPF_MAP_TYPE_HASH_OF_MAPS);
46	__uint(max_entries, 5);	83	__uint(max_entries, 5);
@@ -101,6 +138,12 @@ int handle__sys_enter(void *ctx) · 54 unmodified lines
101	val = input + 1;	138	val = input + 1;
102	bpf_map_update_elem(inner_map, &key, &val, 0);	139	bpf_map_update_elem(inner_map, &key, &val, 0);
103		140
		141	inner_map = bpf_map_lookup_elem(&outer_arr_dyn, &key);
		142	if (!inner_map)
		143	return 1;
		144	val = input + 2;
		145	bpf_map_update_elem(inner_map, &key, &val, 0);
		146
104	return 0;	147	return 0;
105	}	148	}
106		149

tools/testing/selftests/bpf/progs/test_core_autosize.c+172−0

@@ -0,0 +1,172 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3
	4	#include <linux/bpf.h>
	5	#include <stdint.h>
	6	#include <bpf/bpf_helpers.h>
	7	#include <bpf/bpf_core_read.h>
	8
	9	char _license[] SEC("license") = "GPL";
	10
	11	/* fields of exactly the same size */
	12	struct test_struct___samesize {
	13	void *ptr;
	14	unsigned long long val1;
	15	unsigned int val2;
	16	unsigned short val3;
	17	unsigned char val4;
	18	} __attribute((preserve_access_index));
	19
	20	/* unsigned fields that have to be downsized by libbpf */
	21	struct test_struct___downsize {
	22	void *ptr;
	23	unsigned long val1;
	24	unsigned long val2;
	25	unsigned long val3;
	26	unsigned long val4;
	27	/* total sz: 40 */
	28	} __attribute__((preserve_access_index));
	29
	30	/* fields with signed integers of wrong size, should be rejected */
	31	struct test_struct___signed {
	32	void *ptr;
	33	long val1;
	34	long val2;
	35	long val3;
	36	long val4;
	37	} __attribute((preserve_access_index));
	38
	39	/* real layout and sizes according to test's (32-bit) BTF */
	40	struct test_struct___real {
	41	unsigned int ptr; /* can't use `void `, it is always 8 byte in BPF target /
	42	unsigned int val2;
	43	unsigned long long val1;
	44	unsigned short val3;
	45	unsigned char val4;
	46	unsigned char _pad;
	47	/* total sz: 20 */
	48	};
	49
	50	struct test_struct___real input = {
	51	.ptr = 0x01020304,
	52	.val1 = 0x1020304050607080,
	53	.val2 = 0x0a0b0c0d,
	54	.val3 = 0xfeed,
	55	.val4 = 0xb9,
	56	._pad = 0xff, /* make sure no accidental zeros are present */
	57	};
	58
	59	unsigned long long ptr_samesized = 0;
	60	unsigned long long val1_samesized = 0;
	61	unsigned long long val2_samesized = 0;
	62	unsigned long long val3_samesized = 0;
	63	unsigned long long val4_samesized = 0;
	64	struct test_struct___real output_samesized = {};
	65
	66	unsigned long long ptr_downsized = 0;
	67	unsigned long long val1_downsized = 0;
	68	unsigned long long val2_downsized = 0;
	69	unsigned long long val3_downsized = 0;
	70	unsigned long long val4_downsized = 0;
	71	struct test_struct___real output_downsized = {};
	72
	73	unsigned long long ptr_probed = 0;
	74	unsigned long long val1_probed = 0;
	75	unsigned long long val2_probed = 0;
	76	unsigned long long val3_probed = 0;
	77	unsigned long long val4_probed = 0;
	78
	79	unsigned long long ptr_signed = 0;
	80	unsigned long long val1_signed = 0;
	81	unsigned long long val2_signed = 0;
	82	unsigned long long val3_signed = 0;
	83	unsigned long long val4_signed = 0;
	84	struct test_struct___real output_signed = {};
	85
	86	SEC("raw_tp/sys_exit")
	87	int handle_samesize(void *ctx)
	88	{
	89	struct test_struct___samesize in = (void )&input;
	90	struct test_struct___samesize out = (void )&output_samesized;
	91
	92	ptr_samesized = (unsigned long long)in->ptr;
	93	val1_samesized = in->val1;
	94	val2_samesized = in->val2;
	95	val3_samesized = in->val3;
	96	val4_samesized = in->val4;
	97
	98	out->ptr = in->ptr;
	99	out->val1 = in->val1;
	100	out->val2 = in->val2;
	101	out->val3 = in->val3;
	102	out->val4 = in->val4;
	103
	104	return 0;
	105	}
	106
	107	SEC("raw_tp/sys_exit")
	108	int handle_downsize(void *ctx)
	109	{
	110	struct test_struct___downsize in = (void )&input;
	111	struct test_struct___downsize out = (void )&output_downsized;
	112
	113	ptr_downsized = (unsigned long long)in->ptr;
	114	val1_downsized = in->val1;
	115	val2_downsized = in->val2;
	116	val3_downsized = in->val3;
	117	val4_downsized = in->val4;
	118
	119	out->ptr = in->ptr;
	120	out->val1 = in->val1;
	121	out->val2 = in->val2;
	122	out->val3 = in->val3;
	123	out->val4 = in->val4;
	124
	125	return 0;
	126	}
	127
	128	SEC("raw_tp/sys_enter")
	129	int handle_probed(void *ctx)
	130	{
	131	struct test_struct___downsize in = (void )&input;
	132	__u64 tmp;
	133
	134	tmp = 0;
	135	bpf_core_read(&tmp, bpf_core_field_size(in->ptr), &in->ptr);
	136	ptr_probed = tmp;
	137
	138	tmp = 0;
	139	bpf_core_read(&tmp, bpf_core_field_size(in->val1), &in->val1);
	140	val1_probed = tmp;
	141
	142	tmp = 0;
	143	bpf_core_read(&tmp, bpf_core_field_size(in->val2), &in->val2);
	144	val2_probed = tmp;
	145
	146	tmp = 0;
	147	bpf_core_read(&tmp, bpf_core_field_size(in->val3), &in->val3);
	148	val3_probed = tmp;
	149
	150	tmp = 0;
	151	bpf_core_read(&tmp, bpf_core_field_size(in->val4), &in->val4);
	152	val4_probed = tmp;
	153
	154	return 0;
	155	}
	156
	157	SEC("raw_tp/sys_enter")
	158	int handle_signed(void *ctx)
	159	{
	160	struct test_struct___signed in = (void )&input;
	161	struct test_struct___signed out = (void )&output_signed;
	162
	163	val2_signed = in->val2;
	164	val3_signed = in->val3;
	165	val4_signed = in->val4;
	166
	167	out->val2= in->val2;
	168	out->val3= in->val3;
	169	out->val4= in->val4;
	170
	171	return 0;
	172	}

tools/testing/selftests/bpf/progs/test_ksyms_btf.c+55−0

@@ -0,0 +1,55 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Google */
	3
	4	#include "vmlinux.h"
	5
	6	#include <bpf/bpf_helpers.h>
	7
	8	__u64 out__runqueues_addr = -1;
	9	__u64 out__bpf_prog_active_addr = -1;
	10
	11	__u32 out__rq_cpu = -1; /* percpu struct fields */
	12	int out__bpf_prog_active = -1; /* percpu int */
	13
	14	__u32 out__this_rq_cpu = -1;
	15	int out__this_bpf_prog_active = -1;
	16
	17	__u32 out__cpu_0_rq_cpu = -1; /* cpu_rq(0)->cpu */
	18
	19	extern const struct rq runqueues __ksym; /* struct type global var. */
	20	extern const int bpf_prog_active __ksym; /* int type global var. */
	21
	22	SEC("raw_tp/sys_enter")
	23	int handler(const void *ctx)
	24	{
	25	struct rq *rq;
	26	int *active;
	27	__u32 cpu;
	28
	29	out__runqueues_addr = (__u64)&runqueues;
	30	out__bpf_prog_active_addr = (__u64)&bpf_prog_active;
	31
	32	cpu = bpf_get_smp_processor_id();
	33
	34	/* test bpf_per_cpu_ptr() */
	35	rq = (struct rq *)bpf_per_cpu_ptr(&runqueues, cpu);
	36	if (rq)
	37	out__rq_cpu = rq->cpu;
	38	active = (int *)bpf_per_cpu_ptr(&bpf_prog_active, cpu);
	39	if (active)
	40	out__bpf_prog_active = *active;
	41
	42	rq = (struct rq *)bpf_per_cpu_ptr(&runqueues, 0);
	43	if (rq) /* should always be valid, but we can't spare the check. */
	44	out__cpu_0_rq_cpu = rq->cpu;
	45
	46	/* test bpf_this_cpu_ptr */
	47	rq = (struct rq *)bpf_this_cpu_ptr(&runqueues);
	48	out__this_rq_cpu = rq->cpu;
	49	active = (int *)bpf_this_cpu_ptr(&bpf_prog_active);
	50	out__this_bpf_prog_active = *active;
	51
	52	return 0;
	53	}
	54
	55	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_misc_tcp_hdr_options.c+2−2

@@ -304,10 +304,10 @@ int misc_estab(struct bpf_sock_ops *skops) · 303 unmodified lines
304	passive_lport_n = __bpf_htons(passive_lport_h);	304	passive_lport_n = __bpf_htons(passive_lport_h);
305	bpf_setsockopt(skops, SOL_TCP, TCP_SAVE_SYN,	305	bpf_setsockopt(skops, SOL_TCP, TCP_SAVE_SYN,
306	&true_val, sizeof(true_val));	306	&true_val, sizeof(true_val));
307	set_hdr_cb_flags(skops);	307	set_hdr_cb_flags(skops, 0);
308	break;	308	break;
309	case BPF_SOCK_OPS_TCP_CONNECT_CB:	309	case BPF_SOCK_OPS_TCP_CONNECT_CB:
310	set_hdr_cb_flags(skops);	310	set_hdr_cb_flags(skops, 0);
311	break;	311	break;
312	case BPF_SOCK_OPS_PARSE_HDR_OPT_CB:	312	case BPF_SOCK_OPS_PARSE_HDR_OPT_CB:
313	return handle_parse_hdr(skops);	313	return handle_parse_hdr(skops);

tools/testing/selftests/bpf/progs/test_sockmap_kern.h+25−9

 }
-SEC("sk_skb3")
+static inline void bpf_write_pass(struct __sk_buff *skb, int offset)
-int bpf_prog3(struct __sk_buff *skb)
 {
-	const int one = 1;
+	int err = bpf_skb_pull_data(skb, 6 + offset);
-	int err, *f, ret = SK_PASS;
 	void *data_end;
 	char *c;
-	err = bpf_skb_pull_data(skb, 19);
 	if (err)
-		goto tls_out;
+		return;
 	c = (char *)(long)skb->data;
 	data_end = (void *)(long)skb->data_end;
-	if (c + 18 < data_end)
+	if (c + 5 + offset < data_end)
-		memcpy(&c[13], "PASS", 4);
+		memcpy(c + offset, "PASS", 4);
+}
+SEC("sk_skb3")
+int bpf_prog3(struct __sk_buff *skb)
+{
+	int err, *f, ret = SK_PASS;
+	const int one = 1;
 	f = bpf_map_lookup_elem(&sock_skb_opts, &one);
 	if (f && *f) {
 		__u64 flags = 0;
 		ret = 0;
 		flags = *f;
+		err = bpf_skb_adjust_room(skb, -13, 0, 0);
+		if (err)
+			return SK_DROP;
+		err = bpf_skb_adjust_room(skb, 4, 0, 0);
+		if (err)
+			return SK_DROP;
+		bpf_write_pass(skb, 0);
 #ifdef SOCKMAP
 		return bpf_sk_redirect_map(skb, &tls_sock_map, ret, flags);
 #else
 		return bpf_sk_redirect_hash(skb, &tls_sock_map, &ret, flags);
 #endif
 	}
 	f = bpf_map_lookup_elem(&sock_skb_opts, &one);
 	if (f && *f)
 		ret = SK_DROP;
+	err = bpf_skb_adjust_room(skb, 4, 0, 0);
+	if (err)
+		return SK_DROP;
+	bpf_write_pass(skb, 13);
 tls_out:
 	return ret;
 }

tools/testing/selftests/bpf/progs/test_tc_neigh.c+22−18

 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_endian.h>
-#ifndef barrier_data
-# define barrier_data(ptr)	asm volatile("": :"r"(ptr) :"memory")
-#endif
 #ifndef ctx_ptr
 # define ctx_ptr(field)		(void *)(long)(field)
 #endif
-#define dst_to_src_tmp		0xeeddddeeU
-#define src_to_dst_tmp		0xeeffffeeU
 #define ip4_src			0xac100164 /* 172.16.1.100 */
 #define ip4_dst			0xac100264 /* 172.16.2.100 */
 				 a.s6_addr32[3] == b.s6_addr32[3])
 #endif
+enum {
+	dev_src,
+	dev_dst,
+};
+struct bpf_map_def SEC("maps") ifindex_map = {
+	.type		= BPF_MAP_TYPE_ARRAY,
+	.key_size	= sizeof(int),
+	.value_size	= sizeof(int),
+	.max_entries	= 2,
+};
 static __always_inline bool is_remote_ep_v4(struct __sk_buff *skb,
 					    __be32 addr)
 {
 	return v6_equal(ip6h->daddr, addr);
 }
-SEC("chk_neigh") int tc_chk(struct __sk_buff *skb)
+static __always_inline int get_dev_ifindex(int which)
+{
+	int *ifindex = bpf_map_lookup_elem(&ifindex_map, &which);
+	return ifindex ? *ifindex : 0;
+}
+SEC("chk_egress") int tc_chk(struct __sk_buff *skb)
 {
 	void *data_end = ctx_ptr(skb->data_end);
 	void *data = ctx_ptr(skb->data);
 SEC("dst_ingress") int tc_dst(struct __sk_buff *skb)
 {
-	int idx = dst_to_src_tmp;
 	__u8 zero[ETH_ALEN * 2];
 	bool redirect = false;
 	if (!redirect)
 		return TC_ACT_OK;
-	barrier_data(&idx);
-	idx = bpf_ntohl(idx);
 	__builtin_memset(&zero, 0, sizeof(zero));
 	if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0)
 		return TC_ACT_SHOT;
-	return bpf_redirect_neigh(idx, 0);
+	return bpf_redirect_neigh(get_dev_ifindex(dev_src), 0);
 }
 SEC("src_ingress") int tc_src(struct __sk_buff *skb)
 {
-	int idx = src_to_dst_tmp;
 	__u8 zero[ETH_ALEN * 2];
 	bool redirect = false;
 	if (!redirect)
 		return TC_ACT_OK;
-	barrier_data(&idx);
-	idx = bpf_ntohl(idx);
 	__builtin_memset(&zero, 0, sizeof(zero));
 	if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0)
 		return TC_ACT_SHOT;
-	return bpf_redirect_neigh(idx, 0);
+	return bpf_redirect_neigh(get_dev_ifindex(dev_dst), 0);
 }
 char __license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_tc_peer.c+45−0

@@ -0,0 +1,45 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <stdint.h>
	3	#include <stdbool.h>
	4
	5	#include <linux/bpf.h>
	6	#include <linux/stddef.h>
	7	#include <linux/pkt_cls.h>
	8
	9	#include <bpf/bpf_helpers.h>
	10
	11	enum {
	12	dev_src,
	13	dev_dst,
	14	};
	15
	16	struct bpf_map_def SEC("maps") ifindex_map = {
	17	.type = BPF_MAP_TYPE_ARRAY,
	18	.key_size = sizeof(int),
	19	.value_size = sizeof(int),
	20	.max_entries = 2,
	21	};
	22
	23	static __always_inline int get_dev_ifindex(int which)
	24	{
	25	int *ifindex = bpf_map_lookup_elem(&ifindex_map, &which);
	26
	27	return ifindex ? *ifindex : 0;
	28	}
	29
	30	SEC("chk_egress") int tc_chk(struct __sk_buff *skb)
	31	{
	32	return TC_ACT_SHOT;
	33	}
	34
	35	SEC("dst_ingress") int tc_dst(struct __sk_buff *skb)
	36	{
	37	return bpf_redirect_peer(get_dev_ifindex(dev_src), 0);
	38	}
	39
	40	SEC("src_ingress") int tc_src(struct __sk_buff *skb)
	41	{
	42	return bpf_redirect_peer(get_dev_ifindex(dev_dst), 0);
	43	}
	44
	45	char __license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_tcp_hdr_options.c+5−2

 __u8 test_kind = TCPOPT_EXP;
 __u16 test_magic = 0xeB9F;
+__u32 inherit_cb_flags = 0;
 struct bpf_test_option passive_synack_out = {};
 struct bpf_test_option passive_fin_out	= {};
 	struct tcphdr *th;
 	int err;
+	inherit_cb_flags = skops->bpf_sock_ops_cb_flags;
 	err = load_option(skops, &passive_estab_in, true);
 	if (err == -ENOENT) {
 		/* saved_syn is not found. It was in syncookie mode.
 	case BPF_SOCK_OPS_TCP_LISTEN_CB:
 		bpf_setsockopt(skops, SOL_TCP, TCP_SAVE_SYN,
 			       &true_val, sizeof(true_val));
-		set_hdr_cb_flags(skops);
+		set_hdr_cb_flags(skops, BPF_SOCK_OPS_STATE_CB_FLAG);
 		break;
 	case BPF_SOCK_OPS_TCP_CONNECT_CB:
-		set_hdr_cb_flags(skops);
+		set_hdr_cb_flags(skops, 0);
 		break;
 	case BPF_SOCK_OPS_PARSE_HDR_OPT_CB:
 		return handle_parse_hdr(skops);

tools/testing/selftests/bpf/test_sockmap.c+45−36

 int ktls;
 int peek_flag;
 int skb_use_parser;
+int txmsg_omit_skb_parser;
 static const struct option long_options[] = {
 	{"help",	no_argument,		NULL, 'h' },
 	{"txmsg_redir_skb", no_argument,	&txmsg_redir_skb, 1 },
 	{"ktls", no_argument,			&ktls, 1 },
 	{"peek", no_argument,			&peek_flag, 1 },
+	{"txmsg_omit_skb_parser", no_argument,      &txmsg_omit_skb_parser, 1},
 	{"whitelist", required_argument,	NULL, 'n' },
 	{"blacklist", required_argument,	NULL, 'b' },
 	{0, 0, NULL, 0 }
 	txmsg_apply = txmsg_cork = 0;
 	txmsg_ingress = txmsg_redir_skb = 0;
 	txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
+	txmsg_omit_skb_parser = 0;
 	skb_use_parser = 0;
 }
 		if (i == 0 && txmsg_ktls_skb) {
 			if (msg->msg_iov[i].iov_len < 4)
 				return -EIO;
-			if (txmsg_ktls_skb_redir) {
+			if (memcmp(d, "PASS", 4) != 0) {
-				if (memcmp(&d[13], "PASS", 4) != 0) {
+				fprintf(stderr,
-					fprintf(stderr,
+					"detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
-						"detected redirect ktls_skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", i, 0, d[13], d[14], d[15], d[16]);
+					i, 0, d[0], d[1], d[2], d[3]);
-					return -EIO;
+				return -EIO;
-				}
-				d[13] = 0;
-				d[14] = 1;
-				d[15] = 2;
-				d[16] = 3;
-				j = 13;
-			} else if (txmsg_ktls_skb) {
-				if (memcmp(d, "PASS", 4) != 0) {
-					fprintf(stderr,
-						"detected ktls_skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n", i, 0, d[0], d[1], d[2], d[3]);
-					return -EIO;
-				}
-				d[0] = 0;
-				d[1] = 1;
-				d[2] = 2;
-				d[3] = 3;
 			}
+			j = 4; /* advance index past PASS header */
 		}
 		for (; j < msg->msg_iov[i].iov_len && size; j++) {
 		goto run;
 	/* Attach programs to sockmap */
-	err = bpf_prog_attach(prog_fd[0], map_fd[0],
+	if (!txmsg_omit_skb_parser) {
-				BPF_SK_SKB_STREAM_PARSER, 0);
+		err = bpf_prog_attach(prog_fd[0], map_fd[0],
-	if (err) {
+				      BPF_SK_SKB_STREAM_PARSER, 0);
-		fprintf(stderr,
+		if (err) {
-			"ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
+			fprintf(stderr,
-			prog_fd[0], map_fd[0], err, strerror(errno));
+				"ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
-		return err;
+				prog_fd[0], map_fd[0], err, strerror(errno));
+			return err;
+		}
 	}
 	err = bpf_prog_attach(prog_fd[1], map_fd[0],
 	/* Attach programs to TLS sockmap */
 	if (txmsg_ktls_skb) {
-		err = bpf_prog_attach(prog_fd[0], map_fd[8],
+		if (!txmsg_omit_skb_parser) {
-					BPF_SK_SKB_STREAM_PARSER, 0);
+			err = bpf_prog_attach(prog_fd[0], map_fd[8],
-		if (err) {
+					      BPF_SK_SKB_STREAM_PARSER, 0);
-			fprintf(stderr,
+			if (err) {
-				"ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
+				fprintf(stderr,
-				prog_fd[0], map_fd[8], err, strerror(errno));
+					"ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
-			return err;
+					prog_fd[0], map_fd[8], err, strerror(errno));
+				return err;
+			}
 		}
 		err = bpf_prog_attach(prog_fd[2], map_fd[8],
 	txmsg_ktls_skb_drop = 0;
 	txmsg_ktls_skb_redir = 1;
 	test_exec(cgrp, opt);
+	txmsg_ktls_skb_redir = 0;
+	/* Tests that omit skb_parser */
+	txmsg_omit_skb_parser = 1;
+	ktls = 0;
+	txmsg_ktls_skb = 0;
+	test_exec(cgrp, opt);
+	txmsg_ktls_skb_drop = 1;
+	test_exec(cgrp, opt);
+	txmsg_ktls_skb_drop = 0;
+	txmsg_ktls_skb_redir = 1;
+	test_exec(cgrp, opt);
+	ktls = 1;
+	test_exec(cgrp, opt);
+	txmsg_omit_skb_parser = 0;
 	opt->data_test = data;
 	ktls = k;
 }
 /* Test cork with hung data. This tests poor usage patterns where
  * cork can leave data on the ring if user program is buggy and
  * doesn't flush them somehow. They do take some time however

tools/testing/selftests/bpf/test_tc_neigh.sh+0−168

@@ -1,168 +0,0 @@
1	#!/bin/bash
2	# SPDX-License-Identifier: GPL-2.0
3	#
4	# This test sets up 3 netns (src <-> fwd <-> dst). There is no direct veth link
5	# between src and dst. The netns fwd has veth links to each src and dst. The
6	# client is in src and server in dst. The test installs a TC BPF program to each
7	# host facing veth in fwd which calls into bpf_redirect_peer() to perform the
8	# neigh addr population and redirect; it also installs a dropper prog on the
9	# egress side to drop skbs if neigh addrs were not populated.
10
11	if [[ $EUID -ne 0 ]]; then
12	echo "This script must be run as root"
13	echo "FAIL"
14	exit 1
15	fi
16
17	# check that nc, dd, ping, ping6 and timeout are present
18	command -v nc >/dev/null 2>&1 \|\| \
19	{ echo >&2 "nc is not available"; exit 1; }
20	command -v dd >/dev/null 2>&1 \|\| \
21	{ echo >&2 "dd is not available"; exit 1; }
22	command -v timeout >/dev/null 2>&1 \|\| \
23	{ echo >&2 "timeout is not available"; exit 1; }
24	command -v ping >/dev/null 2>&1 \|\| \
25	{ echo >&2 "ping is not available"; exit 1; }
26	command -v ping6 >/dev/null 2>&1 \|\| \
27	{ echo >&2 "ping6 is not available"; exit 1; }
28
29	readonly GREEN='\033[0;92m'
30	readonly RED='\033[0;31m'
31	readonly NC='\033[0m' # No Color
32
33	readonly PING_ARG="-c 3 -w 10 -q"
34
35	readonly TIMEOUT=10
36
37	readonly NS_SRC="ns-src-$(mktemp -u XXXXXX)"
38	readonly NS_FWD="ns-fwd-$(mktemp -u XXXXXX)"
39	readonly NS_DST="ns-dst-$(mktemp -u XXXXXX)"
40
41	readonly IP4_SRC="172.16.1.100"
42	readonly IP4_DST="172.16.2.100"
43
44	readonly IP6_SRC="::1:dead:beef:cafe"
45	readonly IP6_DST="::2:dead:beef:cafe"
46
47	readonly IP4_SLL="169.254.0.1"
48	readonly IP4_DLL="169.254.0.2"
49	readonly IP4_NET="169.254.0.0"
50
51	cleanup()
52	{
53	ip netns del ${NS_SRC}
54	ip netns del ${NS_FWD}
55	ip netns del ${NS_DST}
56	}
57
58	trap cleanup EXIT
59
60	set -e
61
62	ip netns add "${NS_SRC}"
63	ip netns add "${NS_FWD}"
64	ip netns add "${NS_DST}"
65
66	ip link add veth_src type veth peer name veth_src_fwd
67	ip link add veth_dst type veth peer name veth_dst_fwd
68
69	ip link set veth_src netns ${NS_SRC}
70	ip link set veth_src_fwd netns ${NS_FWD}
71
72	ip link set veth_dst netns ${NS_DST}
73	ip link set veth_dst_fwd netns ${NS_FWD}
74
75	ip -netns ${NS_SRC} addr add ${IP4_SRC}/32 dev veth_src
76	ip -netns ${NS_DST} addr add ${IP4_DST}/32 dev veth_dst
77
78	# The fwd netns automatically get a v6 LL address / routes, but also needs v4
79	# one in order to start ARP probing. IP4_NET route is added to the endpoints
80	# so that the ARP processing will reply.
81
82	ip -netns ${NS_FWD} addr add ${IP4_SLL}/32 dev veth_src_fwd
83	ip -netns ${NS_FWD} addr add ${IP4_DLL}/32 dev veth_dst_fwd
84
85	ip -netns ${NS_SRC} addr add ${IP6_SRC}/128 dev veth_src nodad
86	ip -netns ${NS_DST} addr add ${IP6_DST}/128 dev veth_dst nodad
87
88	ip -netns ${NS_SRC} link set dev veth_src up
89	ip -netns ${NS_FWD} link set dev veth_src_fwd up
90
91	ip -netns ${NS_DST} link set dev veth_dst up
92	ip -netns ${NS_FWD} link set dev veth_dst_fwd up
93
94	ip -netns ${NS_SRC} route add ${IP4_DST}/32 dev veth_src scope global
95	ip -netns ${NS_SRC} route add ${IP4_NET}/16 dev veth_src scope global
96	ip -netns ${NS_FWD} route add ${IP4_SRC}/32 dev veth_src_fwd scope global
97
98	ip -netns ${NS_SRC} route add ${IP6_DST}/128 dev veth_src scope global
99	ip -netns ${NS_FWD} route add ${IP6_SRC}/128 dev veth_src_fwd scope global
100
101	ip -netns ${NS_DST} route add ${IP4_SRC}/32 dev veth_dst scope global
102	ip -netns ${NS_DST} route add ${IP4_NET}/16 dev veth_dst scope global
103	ip -netns ${NS_FWD} route add ${IP4_DST}/32 dev veth_dst_fwd scope global
104
105	ip -netns ${NS_DST} route add ${IP6_SRC}/128 dev veth_dst scope global
106	ip -netns ${NS_FWD} route add ${IP6_DST}/128 dev veth_dst_fwd scope global
107
108	fmac_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/address)
109	fmac_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/address)
110
111	ip -netns ${NS_SRC} neigh add ${IP4_DST} dev veth_src lladdr $fmac_src
112	ip -netns ${NS_DST} neigh add ${IP4_SRC} dev veth_dst lladdr $fmac_dst
113
114	ip -netns ${NS_SRC} neigh add ${IP6_DST} dev veth_src lladdr $fmac_src
115	ip -netns ${NS_DST} neigh add ${IP6_SRC} dev veth_dst lladdr $fmac_dst
116
117	veth_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/ifindex \| awk '{printf "%08x\n", $1}')
118	veth_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/ifindex \| awk '{printf "%08x\n", $1}')
119
120	xxd -p < test_tc_neigh.o \| sed "s/eeddddee/$veth_src/g" \| xxd -r -p > test_tc_neigh.x.o
121	xxd -p < test_tc_neigh.x.o \| sed "s/eeffffee/$veth_dst/g" \| xxd -r -p > test_tc_neigh.y.o
122
123	ip netns exec ${NS_FWD} tc qdisc add dev veth_src_fwd clsact
124	ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd ingress bpf da obj test_tc_neigh.y.o sec src_ingress
125	ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh
126
127	ip netns exec ${NS_FWD} tc qdisc add dev veth_dst_fwd clsact
128	ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd ingress bpf da obj test_tc_neigh.y.o sec dst_ingress
129	ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh
130
131	rm -f test_tc_neigh.x.o test_tc_neigh.y.o
132
133	ip netns exec ${NS_DST} bash -c "nc -4 -l -p 9004 &"
134	ip netns exec ${NS_DST} bash -c "nc -6 -l -p 9006 &"
135
136	set +e
137
138	TEST="TCPv4 connectivity test"
139	ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP4_DST}/9004"
140	if [ $? -ne 0 ]; then
141	echo -e "${TEST}: ${RED}FAIL${NC}"
142	exit 1
143	fi
144	echo -e "${TEST}: ${GREEN}PASS${NC}"
145
146	TEST="TCPv6 connectivity test"
147	ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP6_DST}/9006"
148	if [ $? -ne 0 ]; then
149	echo -e "${TEST}: ${RED}FAIL${NC}"
150	exit 1
151	fi
152	echo -e "${TEST}: ${GREEN}PASS${NC}"
153
154	TEST="ICMPv4 connectivity test"
155	ip netns exec ${NS_SRC} ping $PING_ARG ${IP4_DST}
156	if [ $? -ne 0 ]; then
157	echo -e "${TEST}: ${RED}FAIL${NC}"
158	exit 1
159	fi
160	echo -e "${TEST}: ${GREEN}PASS${NC}"
161
162	TEST="ICMPv6 connectivity test"
163	ip netns exec ${NS_SRC} ping6 $PING_ARG ${IP6_DST}
164	if [ $? -ne 0 ]; then
165	echo -e "${TEST}: ${RED}FAIL${NC}"
166	exit 1
167	fi
168	echo -e "${TEST}: ${GREEN}PASS${NC}"

tools/testing/selftests/bpf/test_tc_redirect.sh+204−0

@@ -0,0 +1,204 @@
	1	#!/bin/bash
	2	# SPDX-License-Identifier: GPL-2.0
	3	#
	4	# This test sets up 3 netns (src <-> fwd <-> dst). There is no direct veth link
	5	# between src and dst. The netns fwd has veth links to each src and dst. The
	6	# client is in src and server in dst. The test installs a TC BPF program to each
	7	# host facing veth in fwd which calls into i) bpf_redirect_neigh() to perform the
	8	# neigh addr population and redirect or ii) bpf_redirect_peer() for namespace
	9	# switch from ingress side; it also installs a checker prog on the egress side
	10	# to drop unexpected traffic.
	11
	12	if [[ $EUID -ne 0 ]]; then
	13	echo "This script must be run as root"
	14	echo "FAIL"
	15	exit 1
	16	fi
	17
	18	# check that needed tools are present
	19	command -v nc >/dev/null 2>&1 \|\| \
	20	{ echo >&2 "nc is not available"; exit 1; }
	21	command -v dd >/dev/null 2>&1 \|\| \
	22	{ echo >&2 "dd is not available"; exit 1; }
	23	command -v timeout >/dev/null 2>&1 \|\| \
	24	{ echo >&2 "timeout is not available"; exit 1; }
	25	command -v ping >/dev/null 2>&1 \|\| \
	26	{ echo >&2 "ping is not available"; exit 1; }
	27	command -v ping6 >/dev/null 2>&1 \|\| \
	28	{ echo >&2 "ping6 is not available"; exit 1; }
	29	command -v perl >/dev/null 2>&1 \|\| \
	30	{ echo >&2 "perl is not available"; exit 1; }
	31	command -v jq >/dev/null 2>&1 \|\| \
	32	{ echo >&2 "jq is not available"; exit 1; }
	33	command -v bpftool >/dev/null 2>&1 \|\| \
	34	{ echo >&2 "bpftool is not available"; exit 1; }
	35
	36	readonly GREEN='\033[0;92m'
	37	readonly RED='\033[0;31m'
	38	readonly NC='\033[0m' # No Color
	39
	40	readonly PING_ARG="-c 3 -w 10 -q"
	41
	42	readonly TIMEOUT=10
	43
	44	readonly NS_SRC="ns-src-$(mktemp -u XXXXXX)"
	45	readonly NS_FWD="ns-fwd-$(mktemp -u XXXXXX)"
	46	readonly NS_DST="ns-dst-$(mktemp -u XXXXXX)"
	47
	48	readonly IP4_SRC="172.16.1.100"
	49	readonly IP4_DST="172.16.2.100"
	50
	51	readonly IP6_SRC="::1:dead:beef:cafe"
	52	readonly IP6_DST="::2:dead:beef:cafe"
	53
	54	readonly IP4_SLL="169.254.0.1"
	55	readonly IP4_DLL="169.254.0.2"
	56	readonly IP4_NET="169.254.0.0"
	57
	58	netns_cleanup()
	59	{
	60	ip netns del ${NS_SRC}
	61	ip netns del ${NS_FWD}
	62	ip netns del ${NS_DST}
	63	}
	64
	65	netns_setup()
	66	{
	67	ip netns add "${NS_SRC}"
	68	ip netns add "${NS_FWD}"
	69	ip netns add "${NS_DST}"
	70
	71	ip link add veth_src type veth peer name veth_src_fwd
	72	ip link add veth_dst type veth peer name veth_dst_fwd
	73
	74	ip link set veth_src netns ${NS_SRC}
	75	ip link set veth_src_fwd netns ${NS_FWD}
	76
	77	ip link set veth_dst netns ${NS_DST}
	78	ip link set veth_dst_fwd netns ${NS_FWD}
	79
	80	ip -netns ${NS_SRC} addr add ${IP4_SRC}/32 dev veth_src
	81	ip -netns ${NS_DST} addr add ${IP4_DST}/32 dev veth_dst
	82
	83	# The fwd netns automatically get a v6 LL address / routes, but also
	84	# needs v4 one in order to start ARP probing. IP4_NET route is added
	85	# to the endpoints so that the ARP processing will reply.
	86
	87	ip -netns ${NS_FWD} addr add ${IP4_SLL}/32 dev veth_src_fwd
	88	ip -netns ${NS_FWD} addr add ${IP4_DLL}/32 dev veth_dst_fwd
	89
	90	ip -netns ${NS_SRC} addr add ${IP6_SRC}/128 dev veth_src nodad
	91	ip -netns ${NS_DST} addr add ${IP6_DST}/128 dev veth_dst nodad
	92
	93	ip -netns ${NS_SRC} link set dev veth_src up
	94	ip -netns ${NS_FWD} link set dev veth_src_fwd up
	95
	96	ip -netns ${NS_DST} link set dev veth_dst up
	97	ip -netns ${NS_FWD} link set dev veth_dst_fwd up
	98
	99	ip -netns ${NS_SRC} route add ${IP4_DST}/32 dev veth_src scope global
	100	ip -netns ${NS_SRC} route add ${IP4_NET}/16 dev veth_src scope global
	101	ip -netns ${NS_FWD} route add ${IP4_SRC}/32 dev veth_src_fwd scope global
	102
	103	ip -netns ${NS_SRC} route add ${IP6_DST}/128 dev veth_src scope global
	104	ip -netns ${NS_FWD} route add ${IP6_SRC}/128 dev veth_src_fwd scope global
	105
	106	ip -netns ${NS_DST} route add ${IP4_SRC}/32 dev veth_dst scope global
	107	ip -netns ${NS_DST} route add ${IP4_NET}/16 dev veth_dst scope global
	108	ip -netns ${NS_FWD} route add ${IP4_DST}/32 dev veth_dst_fwd scope global
	109
	110	ip -netns ${NS_DST} route add ${IP6_SRC}/128 dev veth_dst scope global
	111	ip -netns ${NS_FWD} route add ${IP6_DST}/128 dev veth_dst_fwd scope global
	112
	113	fmac_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/address)
	114	fmac_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/address)
	115
	116	ip -netns ${NS_SRC} neigh add ${IP4_DST} dev veth_src lladdr $fmac_src
	117	ip -netns ${NS_DST} neigh add ${IP4_SRC} dev veth_dst lladdr $fmac_dst
	118
	119	ip -netns ${NS_SRC} neigh add ${IP6_DST} dev veth_src lladdr $fmac_src
	120	ip -netns ${NS_DST} neigh add ${IP6_SRC} dev veth_dst lladdr $fmac_dst
	121	}
	122
	123	netns_test_connectivity()
	124	{
	125	set +e
	126
	127	ip netns exec ${NS_DST} bash -c "nc -4 -l -p 9004 &"
	128	ip netns exec ${NS_DST} bash -c "nc -6 -l -p 9006 &"
	129
	130	TEST="TCPv4 connectivity test"
	131	ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP4_DST}/9004"
	132	if [ $? -ne 0 ]; then
	133	echo -e "${TEST}: ${RED}FAIL${NC}"
	134	exit 1
	135	fi
	136	echo -e "${TEST}: ${GREEN}PASS${NC}"
	137
	138	TEST="TCPv6 connectivity test"
	139	ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP6_DST}/9006"
	140	if [ $? -ne 0 ]; then
	141	echo -e "${TEST}: ${RED}FAIL${NC}"
	142	exit 1
	143	fi
	144	echo -e "${TEST}: ${GREEN}PASS${NC}"
	145
	146	TEST="ICMPv4 connectivity test"
	147	ip netns exec ${NS_SRC} ping $PING_ARG ${IP4_DST}
	148	if [ $? -ne 0 ]; then
	149	echo -e "${TEST}: ${RED}FAIL${NC}"
	150	exit 1
	151	fi
	152	echo -e "${TEST}: ${GREEN}PASS${NC}"
	153
	154	TEST="ICMPv6 connectivity test"
	155	ip netns exec ${NS_SRC} ping6 $PING_ARG ${IP6_DST}
	156	if [ $? -ne 0 ]; then
	157	echo -e "${TEST}: ${RED}FAIL${NC}"
	158	exit 1
	159	fi
	160	echo -e "${TEST}: ${GREEN}PASS${NC}"
	161
	162	set -e
	163	}
	164
	165	hex_mem_str()
	166	{
	167	perl -e 'print join(" ", unpack("(H2)8", pack("L", @ARGV)))' $1
	168	}
	169
	170	netns_setup_bpf()
	171	{
	172	local obj=$1
	173
	174	ip netns exec ${NS_FWD} tc qdisc add dev veth_src_fwd clsact
	175	ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd ingress bpf da obj $obj sec src_ingress
	176	ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd egress bpf da obj $obj sec chk_egress
	177
	178	ip netns exec ${NS_FWD} tc qdisc add dev veth_dst_fwd clsact
	179	ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd ingress bpf da obj $obj sec dst_ingress
	180	ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd egress bpf da obj $obj sec chk_egress
	181
	182	veth_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/ifindex)
	183	veth_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/ifindex)
	184
	185	progs=$(ip netns exec ${NS_FWD} bpftool net --json \| jq -r '.[] \| .tc \| map(.id) \| .[]')
	186	for prog in $progs; do
	187	map=$(bpftool prog show id $prog --json \| jq -r '.map_ids \| .? \| .[]')
	188	if [ ! -z "$map" ]; then
	189	bpftool map update id $map key hex $(hex_mem_str 0) value hex $(hex_mem_str $veth_src)
	190	bpftool map update id $map key hex $(hex_mem_str 1) value hex $(hex_mem_str $veth_dst)
	191	fi
	192	done
	193	}
	194
	195	trap netns_cleanup EXIT
	196	set -e
	197
	198	netns_setup
	199	netns_setup_bpf test_tc_neigh.o
	200	netns_test_connectivity
	201	netns_cleanup
	202	netns_setup
	203	netns_setup_bpf test_tc_peer.o
	204	netns_test_connectivity

tools/testing/selftests/bpf/test_tcp_hdr_options.h+3−2

 				    BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG));
 }
-static inline void set_hdr_cb_flags(struct bpf_sock_ops *skops)
+static inline void set_hdr_cb_flags(struct bpf_sock_ops *skops, __u32 extra)
 {
 	bpf_sock_ops_cb_flags_set(skops,
 				  skops->bpf_sock_ops_cb_flags |
 				  BPF_SOCK_OPS_PARSE_UNKNOWN_HDR_OPT_CB_FLAG |
-				  BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG);
+				  BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG |
+				  extra);
 }
 static inline void
 clear_parse_all_hdr_cb_flags(struct bpf_sock_ops *skops)

tools/testing/selftests/bpf/trace_helpers.c+27−0

@@ -90,6 +90,33 @@ long ksym_get_addr(const char *name) · 89 unmodified lines
90	return 0;	90	return 0;
91	}	91	}
92		92
		93	/* open kallsyms and read symbol addresses on the fly. Without caching all symbols,
		94	* this is faster than load + find.
		95	*/
		96	int kallsyms_find(const char sym, unsigned long long addr)
		97	{
		98	char type, name[500];
		99	unsigned long long value;
		100	int err = 0;
		101	FILE *f;
		102
		103	f = fopen("/proc/kallsyms", "r");
		104	if (!f)
		105	return -EINVAL;
		106
		107	while (fscanf(f, "%llx %c %499s%*[^\n]\n", &value, &type, name) > 0) {
		108	if (strcmp(name, sym) == 0) {
		109	*addr = value;
		110	goto out;
		111	}
		112	}
		113	err = -ENOENT;
		114
		115	out:
		116	fclose(f);
		117	return err;
		118	}
		119
93	void read_trace_pipe(void)	120	void read_trace_pipe(void)
94	{	121	{
95	int trace_fd;	122	int trace_fd;

tools/testing/selftests/bpf/trace_helpers.h+4−0

@@ -12,6 +12,10 @@ struct ksym { · 11 unmodified lines
12	int load_kallsyms(void);	12	int load_kallsyms(void);
13	struct ksym *ksym_search(long key);	13	struct ksym *ksym_search(long key);
14	long ksym_get_addr(const char *name);	14	long ksym_get_addr(const char *name);
		15
		16	/* open kallsyms and find addresses on the fly, faster than load + search. */
		17	int kallsyms_find(const char sym, unsigned long long addr);
		18
15	void read_trace_pipe(void);	19	void read_trace_pipe(void);
16		20
17	#endif	21	#endif

tools/testing/selftests/bpf/verifier/basic.c+1−1

@@ -2,7 +2,7 @@ · 1 unmodified line
2	"empty prog",	2	"empty prog",
3	.insns = {	3	.insns = {
4	},	4	},
5	.errstr = "unknown opcode 00",	5	.errstr = "last insn is not an exit or jmp",
6	.result = REJECT,	6	.result = REJECT,
7	},	7	},
8	{	8	{

tools/testing/selftests/bpf/verifier/direct_packet_access.c+1−1

@@ -529,7 +529,7 @@ · 528 unmodified lines
529	},	529	},
530	.prog_type = BPF_PROG_TYPE_SCHED_CLS,	530	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
531	.result = REJECT,	531	.result = REJECT,
532	.errstr = "invalid access to packet, off=0 size=8, R5(id=1,off=0,r=0)",	532	.errstr = "invalid access to packet, off=0 size=8, R5(id=2,off=0,r=0)",
533	.flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,	533	.flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
534	},	534	},
535	{	535	{

tools/testing/selftests/bpf/verifier/ld_imm64.c+0−8

@@ -50,14 +50,6 @@ · 49 unmodified lines
50	.errstr = "invalid bpf_ld_imm64 insn",	50	.errstr = "invalid bpf_ld_imm64 insn",
51	.result = REJECT,	51	.result = REJECT,
52	},	52	},
53	{
54	"test5 ld_imm64",
55	.insns = {
56	BPF_RAW_INSN(BPF_LD \| BPF_IMM \| BPF_DW, 0, 0, 0, 0),
57	},
58	.errstr = "invalid bpf_ld_imm64 insn",
59	.result = REJECT,
60	},
61	{	53	{
62	"test6 ld_imm64",	54	"test6 ld_imm64",
63	.insns = {	55	.insns = {

tools/testing/selftests/bpf/verifier/regalloc.c+243−0

@@ -0,0 +1,243 @@
	1	{
	2	"regalloc basic",
	3	.insns = {
	4	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	5	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	6	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	7	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	8	BPF_LD_MAP_FD(BPF_REG_1, 0),
	9	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	10	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 8),
	11	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	12	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	13	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	14	BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 20, 4),
	15	BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 3),
	16	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	17	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	18	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	19	BPF_EXIT_INSN(),
	20	},
	21	.fixup_map_hash_48b = { 4 },
	22	.result = ACCEPT,
	23	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	24	},
	25	{
	26	"regalloc negative",
	27	.insns = {
	28	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	29	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	30	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	31	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	32	BPF_LD_MAP_FD(BPF_REG_1, 0),
	33	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	34	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 8),
	35	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	36	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	37	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	38	BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 24, 4),
	39	BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 3),
	40	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	41	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	42	BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_7, 0),
	43	BPF_EXIT_INSN(),
	44	},
	45	.fixup_map_hash_48b = { 4 },
	46	.result = REJECT,
	47	.errstr = "invalid access to map value, value_size=48 off=48 size=1",
	48	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	49	},
	50	{
	51	"regalloc src_reg mark",
	52	.insns = {
	53	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	54	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	55	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	56	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	57	BPF_LD_MAP_FD(BPF_REG_1, 0),
	58	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	59	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 9),
	60	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	61	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	62	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	63	BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 20, 5),
	64	BPF_MOV64_IMM(BPF_REG_3, 0),
	65	BPF_JMP_REG(BPF_JSGE, BPF_REG_3, BPF_REG_2, 3),
	66	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	67	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	68	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	69	BPF_EXIT_INSN(),
	70	},
	71	.fixup_map_hash_48b = { 4 },
	72	.result = ACCEPT,
	73	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	74	},
	75	{
	76	"regalloc src_reg negative",
	77	.insns = {
	78	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	79	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	80	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	81	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	82	BPF_LD_MAP_FD(BPF_REG_1, 0),
	83	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	84	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 9),
	85	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	86	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	87	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	88	BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 22, 5),
	89	BPF_MOV64_IMM(BPF_REG_3, 0),
	90	BPF_JMP_REG(BPF_JSGE, BPF_REG_3, BPF_REG_2, 3),
	91	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	92	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	93	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	94	BPF_EXIT_INSN(),
	95	},
	96	.fixup_map_hash_48b = { 4 },
	97	.result = REJECT,
	98	.errstr = "invalid access to map value, value_size=48 off=44 size=8",
	99	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	100	},
	101	{
	102	"regalloc and spill",
	103	.insns = {
	104	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	105	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	106	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	107	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	108	BPF_LD_MAP_FD(BPF_REG_1, 0),
	109	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	110	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 11),
	111	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	112	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	113	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	114	BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 20, 7),
	115	/* r0 has upper bound that should propagate into r2 */
	116	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -8), /* spill r2 */
	117	BPF_MOV64_IMM(BPF_REG_0, 0),
	118	BPF_MOV64_IMM(BPF_REG_2, 0), /* clear r0 and r2 */
	119	BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_10, -8), /* fill r3 */
	120	BPF_JMP_REG(BPF_JSGE, BPF_REG_0, BPF_REG_3, 2),
	121	/* r3 has lower and upper bounds */
	122	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_3),
	123	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	124	BPF_EXIT_INSN(),
	125	},
	126	.fixup_map_hash_48b = { 4 },
	127	.result = ACCEPT,
	128	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	129	},
	130	{
	131	"regalloc and spill negative",
	132	.insns = {
	133	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	134	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	135	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	136	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	137	BPF_LD_MAP_FD(BPF_REG_1, 0),
	138	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	139	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 11),
	140	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	141	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	142	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	143	BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 48, 7),
	144	/* r0 has upper bound that should propagate into r2 */
	145	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -8), /* spill r2 */
	146	BPF_MOV64_IMM(BPF_REG_0, 0),
	147	BPF_MOV64_IMM(BPF_REG_2, 0), /* clear r0 and r2 */
	148	BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_10, -8), /* fill r3 */
	149	BPF_JMP_REG(BPF_JSGE, BPF_REG_0, BPF_REG_3, 2),
	150	/* r3 has lower and upper bounds */
	151	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_3),
	152	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	153	BPF_EXIT_INSN(),
	154	},
	155	.fixup_map_hash_48b = { 4 },
	156	.result = REJECT,
	157	.errstr = "invalid access to map value, value_size=48 off=48 size=8",
	158	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	159	},
	160	{
	161	"regalloc three regs",
	162	.insns = {
	163	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	164	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	165	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	166	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	167	BPF_LD_MAP_FD(BPF_REG_1, 0),
	168	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	169	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10),
	170	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	171	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	172	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	173	BPF_MOV64_REG(BPF_REG_4, BPF_REG_2),
	174	BPF_JMP_IMM(BPF_JSGT, BPF_REG_0, 12, 5),
	175	BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 4),
	176	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_0),
	177	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_2),
	178	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_4),
	179	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	180	BPF_EXIT_INSN(),
	181	},
	182	.fixup_map_hash_48b = { 4 },
	183	.result = ACCEPT,
	184	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	185	},
	186	{
	187	"regalloc after call",
	188	.insns = {
	189	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	190	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	191	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	192	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	193	BPF_LD_MAP_FD(BPF_REG_1, 0),
	194	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	195	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10),
	196	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	197	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	198	BPF_MOV64_REG(BPF_REG_8, BPF_REG_0),
	199	BPF_MOV64_REG(BPF_REG_9, BPF_REG_0),
	200	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, 6),
	201	BPF_JMP_IMM(BPF_JSGT, BPF_REG_8, 20, 4),
	202	BPF_JMP_IMM(BPF_JSLT, BPF_REG_9, 0, 3),
	203	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_8),
	204	BPF_ALU64_REG(BPF_ADD, BPF_REG_7, BPF_REG_9),
	205	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_7, 0),
	206	BPF_EXIT_INSN(),
	207	BPF_MOV64_IMM(BPF_REG_0, 0),
	208	BPF_EXIT_INSN(),
	209	},
	210	.fixup_map_hash_48b = { 4 },
	211	.result = ACCEPT,
	212	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	213	},
	214	{
	215	"regalloc in callee",
	216	.insns = {
	217	BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	218	BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
	219	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	220	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	221	BPF_LD_MAP_FD(BPF_REG_1, 0),
	222	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	223	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 6),
	224	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
	225	BPF_EMIT_CALL(BPF_FUNC_get_prandom_u32),
	226	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
	227	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
	228	BPF_MOV64_REG(BPF_REG_3, BPF_REG_7),
	229	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, 1),
	230	BPF_EXIT_INSN(),
	231	BPF_JMP_IMM(BPF_JSGT, BPF_REG_1, 20, 5),
	232	BPF_JMP_IMM(BPF_JSLT, BPF_REG_2, 0, 4),
	233	BPF_ALU64_REG(BPF_ADD, BPF_REG_3, BPF_REG_1),
	234	BPF_ALU64_REG(BPF_ADD, BPF_REG_3, BPF_REG_2),
	235	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_3, 0),
	236	BPF_EXIT_INSN(),
	237	BPF_MOV64_IMM(BPF_REG_0, 0),
	238	BPF_EXIT_INSN(),
	239	},
	240	.fixup_map_hash_48b = { 4 },
	241	.result = ACCEPT,
	242	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
	243	},

@@ -195,13 +195,13 @@ static struct bpf_align_test tests[] = { · 194 unmodified lines
195	.prog_type = BPF_PROG_TYPE_SCHED_CLS,	195	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
196	.matches = {	196	.matches = {
197	{7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},	197	{7, "R3_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
198	{8, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},	198	{8, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
199	{9, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},	199	{9, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},
200	{10, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},	200	{10, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
201	{11, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},	201	{11, "R4_w=inv(id=0,umax_value=510,var_off=(0x0; 0x1fe))"},
202	{12, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},	202	{12, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
203	{13, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},	203	{13, "R4_w=inv(id=0,umax_value=1020,var_off=(0x0; 0x3fc))"},
204	{14, "R4_w=inv(id=0,umax_value=255,var_off=(0x0; 0xff))"},	204	{14, "R4_w=inv(id=1,umax_value=255,var_off=(0x0; 0xff))"},
205	{15, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},	205	{15, "R4_w=inv(id=0,umax_value=2040,var_off=(0x0; 0x7f8))"},
206	{16, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},	206	{16, "R4_w=inv(id=0,umax_value=4080,var_off=(0x0; 0xff0))"},
207	},	207	},
@@ -518,7 +518,7 @@ static struct bpf_align_test tests[] = { · 310 unmodified lines
518	* the total offset is 4-byte aligned and meets the	518	* the total offset is 4-byte aligned and meets the
519	* load's requirements.	519	* load's requirements.
520	*/	520	*/
521	{20, "R5=pkt(id=1,off=0,r=4,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc)"},	521	{20, "R5=pkt(id=2,off=0,r=4,umin_value=2,umax_value=1034,var_off=(0x2; 0x7fc)"},
522		522
523	},	523	},
524	},	524	},
@@ -561,18 +561,18 @@ static struct bpf_align_test tests[] = { · 36 unmodified lines
561	/* Adding 14 makes R6 be (4n+2) */	561	/* Adding 14 makes R6 be (4n+2) */
562	{11, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"},	562	{11, "R6_w=inv(id=0,umin_value=14,umax_value=74,var_off=(0x2; 0x7c))"},
563	/* Subtracting from packet pointer overflows ubounds */	563	/* Subtracting from packet pointer overflows ubounds */
564	{13, "R5_w=pkt(id=1,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"},	564	{13, "R5_w=pkt(id=2,off=0,r=8,umin_value=18446744073709551542,umax_value=18446744073709551602,var_off=(0xffffffffffffff82; 0x7c)"},
565	/* New unknown value in R7 is (4n), >= 76 */	565	/* New unknown value in R7 is (4n), >= 76 */
566	{15, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"},	566	{15, "R7_w=inv(id=0,umin_value=76,umax_value=1096,var_off=(0x0; 0x7fc))"},
567	/* Adding it to packet pointer gives nice bounds again */	567	/* Adding it to packet pointer gives nice bounds again */
568	{16, "R5_w=pkt(id=2,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},	568	{16, "R5_w=pkt(id=3,off=0,r=0,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},
569	/* At the time the word size load is performed from R5,	569	/* At the time the word size load is performed from R5,
570	* its total fixed offset is NET_IP_ALIGN + reg->off (0)	570	* its total fixed offset is NET_IP_ALIGN + reg->off (0)
571	* which is 2. Then the variable offset is (4n+2), so	571	* which is 2. Then the variable offset is (4n+2), so
572	* the total offset is 4-byte aligned and meets the	572	* the total offset is 4-byte aligned and meets the
573	* load's requirements.	573	* load's requirements.
574	*/	574	*/
575	{20, "R5=pkt(id=2,off=0,r=4,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},	575	{20, "R5=pkt(id=3,off=0,r=4,umin_value=2,umax_value=1082,var_off=(0x2; 0xfffffffc)"},
576	},	576	},
577	},	577	},
578	};	578	};