lael-labs/linux

arch/mips/net/bpf_jit_comp32.c+9−1

 	const u8 *fp = bpf2mips32[BPF_REG_FP];
 	int stack, saved, locals, reserved;
+	/*
+	 * In the unlikely event that the TCC limit is raised to more
+	 * than 16 bits, it is clamped to the maximum value allowed for
+	 * the generated code (0xffff). It is better fail to compile
+	 * instead of degrading gracefully.
+	 */
+	BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff);
 	/*
 	 * The first two instructions initialize TCC in the reserved (for us)
 	 * 16-byte area in the parent's stack frame. On a tail call, the
 	 * calling function jumps into the prologue after these instructions.
 	 */
-	emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff));
+	emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
 	emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP);
 	/*

arch/mips/net/bpf_jit_comp64.c+9−1

 	u8 zx = bpf2mips64[JIT_REG_ZX];
 	int stack, saved, locals, reserved;
+	/*
+	 * In the unlikely event that the TCC limit is raised to more
+	 * than 16 bits, it is clamped to the maximum value allowed for
+	 * the generated code (0xffff). It is better fail to compile
+	 * instead of degrading gracefully.
+	 */
+	BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff);
 	/*
 	 * The first instruction initializes the tail call count register.
 	 * On a tail call, the calling function jumps into the prologue
 	 * after this instruction.
 	 */
-	emit(ctx, ori, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff));
+	emit(ctx, ori, tc, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
 	/* === Entry-point for tail calls === */

include/linux/bpf-cgroup.h+17−0

@@ -414,6 +414,11 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr, · 413 unmodified lines
414	int cgroup_bpf_link_attach(const union bpf_attr attr, struct bpf_prog prog);	414	int cgroup_bpf_link_attach(const union bpf_attr attr, struct bpf_prog prog);
415	int cgroup_bpf_prog_query(const union bpf_attr *attr,	415	int cgroup_bpf_prog_query(const union bpf_attr *attr,
416	union bpf_attr __user *uattr);	416	union bpf_attr __user *uattr);
		417
		418	const struct bpf_func_proto *
		419	cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
		420	const struct bpf_func_proto *
		421	cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
417	#else	422	#else
418		423
419	static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }	424	static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
@@ -444,6 +449,18 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, · 24 unmodified lines
444	return -EINVAL;	449	return -EINVAL;
445	}	450	}
446		451
		452	static inline const struct bpf_func_proto *
		453	cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
		454	{
		455	return NULL;
		456	}
		457
		458	static inline const struct bpf_func_proto *
		459	cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
		460	{
		461	return NULL;
		462	}
		463
447	static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,	464	static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
448	struct bpf_map *map) { return 0; }	465	struct bpf_map *map) { return 0; }
449	static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(	466	static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(

include/linux/bpf.h+22−0

 struct module;
 struct bpf_func_state;
 struct ftrace_ops;
+struct cgroup;
 extern struct idr btf_idr;
 extern spinlock_t btf_idr_lock;
 	int __init bpf_iter_ ## target(args) { return 0; }
 struct bpf_iter_aux_info {
+	/* for map_elem iter */
 	struct bpf_map *map;
+	/* for cgroup iter */
+	struct {
+		struct cgroup *start; /* starting cgroup */
+		enum bpf_cgroup_iter_order order;
+	} cgroup;
 };
 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
 	return !sysctl_unprivileged_bpf_disabled;
 }
+/* Not all bpf prog type has the bpf_ctx.
+ * For the bpf prog type that has initialized the bpf_ctx,
+ * this function can be used to decide if a kernel function
+ * is called by a bpf program.
+ */
+static inline bool has_current_bpf_ctx(void)
+{
+	return !!current->bpf_ctx;
+}
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
 	return false;
 }
+static inline bool has_current_bpf_ctx(void)
+{
+	return false;
+}
 #endif /* CONFIG_BPF_SYSCALL */
 void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
+extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
 extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;

include/linux/bpf_mem_alloc.h+28−0

@@ -0,0 +1,28 @@
	1	/* SPDX-License-Identifier: GPL-2.0-only */
	2	/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
	3	#ifndef _BPF_MEM_ALLOC_H
	4	#define _BPF_MEM_ALLOC_H
	5	#include <linux/compiler_types.h>
	6	#include <linux/workqueue.h>
	7
	8	struct bpf_mem_cache;
	9	struct bpf_mem_caches;
	10
	11	struct bpf_mem_alloc {
	12	struct bpf_mem_caches __percpu *caches;
	13	struct bpf_mem_cache __percpu *cache;
	14	struct work_struct work;
	15	};
	16
	17	int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu);
	18	void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma);
	19
	20	/* kmalloc/kfree equivalent: */
	21	void bpf_mem_alloc(struct bpf_mem_alloc ma, size_t size);
	22	void bpf_mem_free(struct bpf_mem_alloc ma, void ptr);
	23
	24	/* kmem_cache_alloc/free equivalent: */
	25	void bpf_mem_cache_alloc(struct bpf_mem_alloc ma);
	26	void bpf_mem_cache_free(struct bpf_mem_alloc ma, void ptr);
	27
	28	#endif /* _BPF_MEM_ALLOC_H */

include/linux/bpf_verifier.h+11−0

@@ -212,6 +212,17 @@ struct bpf_reference_state { · 211 unmodified lines
212	* is used purely to inform the user of a reference leak.	212	* is used purely to inform the user of a reference leak.
213	*/	213	*/
214	int insn_idx;	214	int insn_idx;
		215	/* There can be a case like:
		216	* main (frame 0)
		217	* cb (frame 1)
		218	* func (frame 3)
		219	* cb (frame 4)
		220	* Hence for frame 4, if callback_ref just stored boolean, it would be
		221	* impossible to distinguish nested callback refs. Hence store the
		222	* frameno and compare that to callback_ref in check_reference_leak when
		223	* exiting a callback function.
		224	*/
		225	int callback_ref;
215	};	226	};
216		227
217	/* state of the program:	228	/* state of the program:

include/linux/filter.h+1−2

@@ -900,8 +900,7 @@ int sk_reuseport_attach_filter(struct sock_fprog fprog, struct sock sk); · 899 unmodified lines
900	int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk);	900	int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk);
901	void sk_reuseport_prog_free(struct bpf_prog *prog);	901	void sk_reuseport_prog_free(struct bpf_prog *prog);
902	int sk_detach_filter(struct sock *sk);	902	int sk_detach_filter(struct sock *sk);
903	int sk_get_filter(struct sock sk, struct sock_filter __user filter,	903	int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len);
904	unsigned int len);
905		904
906	bool sk_filter_charge(struct sock sk, struct sk_filter fp);	905	bool sk_filter_charge(struct sock sk, struct sk_filter fp);
907	void sk_filter_uncharge(struct sock sk, struct sk_filter fp);	906	void sk_filter_uncharge(struct sock sk, struct sk_filter fp);

include/linux/igmp.h+2−2

@@ -118,9 +118,9 @@ extern int ip_mc_source(int add, int omode, struct sock *sk, · 117 unmodified lines
118	struct ip_mreq_source *mreqs, int ifindex);	118	struct ip_mreq_source *mreqs, int ifindex);
119	extern int ip_mc_msfilter(struct sock sk, struct ip_msfilter msf,int ifindex);	119	extern int ip_mc_msfilter(struct sock sk, struct ip_msfilter msf,int ifindex);
120	extern int ip_mc_msfget(struct sock sk, struct ip_msfilter msf,	120	extern int ip_mc_msfget(struct sock sk, struct ip_msfilter msf,
121	struct ip_msfilter __user optval, int __user optlen);	121	sockptr_t optval, sockptr_t optlen);
122	extern int ip_mc_gsfget(struct sock sk, struct group_filter gsf,	122	extern int ip_mc_gsfget(struct sock sk, struct group_filter gsf,
123	struct sockaddr_storage __user *p);	123	sockptr_t optval, size_t offset);
124	extern int ip_mc_sf_allow(struct sock *sk, __be32 local, __be32 rmt,	124	extern int ip_mc_sf_allow(struct sock *sk, __be32 local, __be32 rmt,
125	int dif, int sdif);	125	int dif, int sdif);
126	extern void ip_mc_init_dev(struct in_device *);	126	extern void ip_mc_init_dev(struct in_device *);

include/linux/mroute.h+3−3

@@ -17,7 +17,7 @@ static inline int ip_mroute_opt(int opt) · 16 unmodified lines
17	}	17	}
18		18
19	int ip_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int);	19	int ip_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int);
20	int ip_mroute_getsockopt(struct sock , int, char __user , int __user *);	20	int ip_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t);
21	int ipmr_ioctl(struct sock sk, int cmd, void __user arg);	21	int ipmr_ioctl(struct sock sk, int cmd, void __user arg);
22	int ipmr_compat_ioctl(struct sock sk, unsigned int cmd, void __user arg);	22	int ipmr_compat_ioctl(struct sock sk, unsigned int cmd, void __user arg);
23	int ip_mr_init(void);	23	int ip_mr_init(void);
@@ -29,8 +29,8 @@ static inline int ip_mroute_setsockopt(struct sock *sock, int optname, · 5 unmodified lines
29	return -ENOPROTOOPT;	29	return -ENOPROTOOPT;
30	}	30	}
31		31
32	static inline int ip_mroute_getsockopt(struct sock *sock, int optname,	32	static inline int ip_mroute_getsockopt(struct sock *sk, int optname,
33	char __user optval, int __user optlen)	33	sockptr_t optval, sockptr_t optlen)
34	{	34	{
35	return -ENOPROTOOPT;	35	return -ENOPROTOOPT;
36	}	36	}

include/linux/mroute6.h+2−2

@@ -27,7 +27,7 @@ struct sock; · 26 unmodified lines
27		27
28	#ifdef CONFIG_IPV6_MROUTE	28	#ifdef CONFIG_IPV6_MROUTE
29	extern int ip6_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int);	29	extern int ip6_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int);
30	extern int ip6_mroute_getsockopt(struct sock , int, char __user , int __user *);	30	extern int ip6_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t);
31	extern int ip6_mr_input(struct sk_buff *skb);	31	extern int ip6_mr_input(struct sk_buff *skb);
32	extern int ip6mr_ioctl(struct sock sk, int cmd, void __user arg);	32	extern int ip6mr_ioctl(struct sock sk, int cmd, void __user arg);
33	extern int ip6mr_compat_ioctl(struct sock sk, unsigned int cmd, void __user arg);	33	extern int ip6mr_compat_ioctl(struct sock sk, unsigned int cmd, void __user arg);
@@ -42,7 +42,7 @@ static inline int ip6_mroute_setsockopt(struct sock *sock, int optname, · 8 unmodified lines
42		42
43	static inline	43	static inline
44	int ip6_mroute_getsockopt(struct sock *sock,	44	int ip6_mroute_getsockopt(struct sock *sock,
45	int optname, char __user optval, int __user optlen)	45	int optname, sockptr_t optval, sockptr_t optlen)
46	{	46	{
47	return -ENOPROTOOPT;	47	return -ENOPROTOOPT;
48	}	48	}

include/linux/skbuff.h+2−2

@@ -1461,8 +1461,8 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector, · 1460 unmodified lines
1461	unsigned int key_count);	1461	unsigned int key_count);
1462		1462
1463	struct bpf_flow_dissector;	1463	struct bpf_flow_dissector;
1464	bool bpf_flow_dissect(struct bpf_prog prog, struct bpf_flow_dissector ctx,	1464	u32 bpf_flow_dissect(struct bpf_prog prog, struct bpf_flow_dissector ctx,
1465	__be16 proto, int nhoff, int hlen, unsigned int flags);	1465	__be16 proto, int nhoff, int hlen, unsigned int flags);
1466		1466
1467	bool __skb_flow_dissect(const struct net *net,	1467	bool __skb_flow_dissect(const struct net *net,
1468	const struct sk_buff *skb,	1468	const struct sk_buff *skb,

include/linux/sockptr.h+5−0

 	return 0;
 }
+static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size)
+{
+	return copy_to_sockptr_offset(dst, 0, src, size);
+}
 static inline void *memdup_sockptr(sockptr_t src, size_t len)
 {
 	void *p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN);

include/linux/tnum.h+18−2

@@ -21,7 +21,12 @@ struct tnum { · 20 unmodified lines
21	struct tnum tnum_const(u64 value);	21	struct tnum tnum_const(u64 value);
22	/* A completely unknown value */	22	/* A completely unknown value */
23	extern const struct tnum tnum_unknown;	23	extern const struct tnum tnum_unknown;
24	/* A value that's unknown except that @min <= value <= @max */	24	/* An unknown value that is a superset of @min <= value <= @max.
		25	*
		26	* Could include values outside the range of [@min, @max].
		27	* For example tnum_range(0, 2) is represented by {0, 1, 2, 3},
		28	* rather than the intended set of {0, 1, 2}.
		29	*/
25	struct tnum tnum_range(u64 min, u64 max);	30	struct tnum tnum_range(u64 min, u64 max);
26		31
27	/* Arithmetic and logical ops */	32	/* Arithmetic and logical ops */
@@ -73,7 +78,18 @@ static inline bool tnum_is_unknown(struct tnum a) · 45 unmodified lines
73	*/	78	*/
74	bool tnum_is_aligned(struct tnum a, u64 size);	79	bool tnum_is_aligned(struct tnum a, u64 size);
75		80
76	/* Returns true if @b represents a subset of @a. */	81	/* Returns true if @b represents a subset of @a.
		82	*
		83	* Note that using tnum_range() as @a requires extra cautions as tnum_in() may
		84	* return true unexpectedly due to tnum limited ability to represent tight
		85	* range, e.g.
		86	*
		87	* tnum_in(tnum_range(0, 2), tnum_const(3)) == true
		88	*
		89	* As a rule of thumb, if @a is explicitly coded rather than coming from
		90	* reg->var_off, it should be in form of tnum_const(), tnum_range(0, 2**n - 1),
		91	* or tnum_range(2n, 2(n+1) - 1).
		92	*/
77	bool tnum_in(struct tnum a, struct tnum b);	93	bool tnum_in(struct tnum a, struct tnum b);
78		94
79	/* Formatting functions. These have snprintf-like semantics: they will write	95	/* Formatting functions. These have snprintf-like semantics: they will write

include/net/ip.h+4−0

 int ip_cmsg_send(struct sock *sk, struct msghdr *msg,
 		 struct ipcm_cookie *ipc, bool allow_ipv6);
 DECLARE_STATIC_KEY_FALSE(ip4_min_ttl);
+int do_ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
+		     unsigned int optlen);
 int ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
 		  unsigned int optlen);
+int do_ip_getsockopt(struct sock *sk, int level, int optname,
+		     sockptr_t optval, sockptr_t optlen);
 int ip_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
 		  int __user *optlen);
 int ip_ra_control(struct sock *sk, unsigned char on,

include/net/ipv6.h+5−1

  */
 DECLARE_STATIC_KEY_FALSE(ip6_min_hopcount);
+int do_ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
+		       unsigned int optlen);
 int ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
 		    unsigned int optlen);
+int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
+		       sockptr_t optval, sockptr_t optlen);
 int ipv6_getsockopt(struct sock *sk, int level, int optname,
 		    char __user *optval, int __user *optlen);
 int ip6_mc_msfilter(struct sock *sk, struct group_filter *gsf,
 		  struct sockaddr_storage *list);
 int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf,
-		  struct sockaddr_storage __user *p);
+		  sockptr_t optval, size_t ss_offset);
 #ifdef CONFIG_PROC_FS
 int ac6_proc_init(struct net *net);

include/net/ipv6_stubs.h+4−0

 				     const struct in6_addr *daddr, __be16 dport,
 				     int dif, int sdif, struct udp_table *tbl,
 				     struct sk_buff *skb);
+	int (*ipv6_setsockopt)(struct sock *sk, int level, int optname,
+			       sockptr_t optval, unsigned int optlen);
+	int (*ipv6_getsockopt)(struct sock *sk, int level, int optname,
+			       sockptr_t optval, sockptr_t optlen);
 };
 extern const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly;

include/net/sock.h+9−0

 	}
 }
+void sockopt_lock_sock(struct sock *sk);
+void sockopt_release_sock(struct sock *sk);
+bool sockopt_ns_capable(struct user_namespace *ns, int cap);
+bool sockopt_capable(int cap);
 /* Used by processes to "lock" a socket state, so that
  * interrupts and bottom half handlers won't change it
  * from under us. It essentially blocks any incoming
 #define sock_edemux sock_efree
 #endif
+int sk_setsockopt(struct sock *sk, int level, int optname,
+		  sockptr_t optval, unsigned int optlen);
 int sock_setsockopt(struct socket *sock, int level, int op,
 		    sockptr_t optval, unsigned int optlen);
+int sk_getsockopt(struct sock *sk, int level, int optname,
+		  sockptr_t optval, sockptr_t optlen);
 int sock_getsockopt(struct socket *sock, int level, int op,
 		    char __user *optval, int __user *optlen);
 int sock_gettstamp(struct socket *sock, void __user *userstamp,

include/net/tcp.h+4−0

 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
 __poll_t tcp_poll(struct file *file, struct socket *sock,
 		      struct poll_table_struct *wait);
+int do_tcp_getsockopt(struct sock *sk, int level,
+		      int optname, sockptr_t optval, sockptr_t optlen);
 int tcp_getsockopt(struct sock *sk, int level, int optname,
 		   char __user *optval, int __user *optlen);
 bool tcp_bpf_bypass_getsockopt(int level, int optname);
+int do_tcp_setsockopt(struct sock *sk, int level, int optname,
+		      sockptr_t optval, unsigned int optlen);
 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
 		   unsigned int optlen);
 void tcp_set_keepalive(struct sock *sk, int val);

include/uapi/linux/bpf.h+69−14

 	__u32	attach_type;		/* program attach type (enum bpf_attach_type) */
 };
+enum bpf_cgroup_iter_order {
+	BPF_CGROUP_ITER_ORDER_UNSPEC = 0,
+	BPF_CGROUP_ITER_SELF_ONLY,		/* process only a single object. */
+	BPF_CGROUP_ITER_DESCENDANTS_PRE,	/* walk descendants in pre-order. */
+	BPF_CGROUP_ITER_DESCENDANTS_POST,	/* walk descendants in post-order. */
+	BPF_CGROUP_ITER_ANCESTORS_UP,		/* walk ancestors upward. */
+};
 union bpf_iter_link_info {
 	struct {
 		__u32	map_fd;
 	} map;
+	struct {
+		enum bpf_cgroup_iter_order order;
+		/* At most one of cgroup_fd and cgroup_id can be non-zero. If
+		 * both are zero, the walk starts from the default cgroup v2
+		 * root. For walking v1 hierarchy, one should always explicitly
+		 * specify cgroup_fd.
+		 */
+		__u32	cgroup_fd;
+		__u64	cgroup_id;
+	} cgroup;
 };
 /* BPF syscall commands, see bpf(2) man-page for more details. */
  *
  *		**-EEXIST** if the option already exists.
  *
- *		**-EFAULT** on failrue to parse the existing header options.
+ *		**-EFAULT** on failure to parse the existing header options.
  *
  *		**-EPERM** if the helper cannot be used under the current
  *		*skops*\ **->op**.
  *		a *map* with *task* as the **key**.  From this
  *		perspective,  the usage is not much different from
  *		**bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this
- *		helper enforces the key must be an task_struct and the map must also
+ *		helper enforces the key must be a task_struct and the map must also
  *		be a **BPF_MAP_TYPE_TASK_STORAGE**.
  *
  *		Underneath, the value is stored locally at *task* instead of
  *
  * long bpf_ima_inode_hash(struct inode *inode, void *dst, u32 size)
  *	Description
- *		Returns the stored IMA hash of the *inode* (if it's avaialable).
+ *		Returns the stored IMA hash of the *inode* (if it's available).
  *		If the hash is larger than *size*, then only *size*
  *		bytes will be copied to *dst*
  *	Return
  *
  *		The argument *len_diff* can be used for querying with a planned
  *		size change. This allows to check MTU prior to changing packet
- *		ctx. Providing an *len_diff* adjustment that is larger than the
+ *		ctx. Providing a *len_diff* adjustment that is larger than the
  *		actual packet size (resulting in negative packet size) will in
- *		principle not exceed the MTU, why it is not considered a
+ *		principle not exceed the MTU, which is why it is not considered
- *		failure.  Other BPF-helpers are needed for performing the
+ *		a failure.  Other BPF helpers are needed for performing the
- *		planned size change, why the responsability for catch a negative
+ *		planned size change; therefore the responsibility for catching
- *		packet size belong in those helpers.
+ *		a negative packet size belongs in those helpers.
  *
  *		Specifying *ifindex* zero means the MTU check is performed
  *		against the current net device.  This is practical if this isn't
  *
  * int bpf_get_retval(void)
  *	Description
- *		Get the syscall's return value that will be returned to userspace.
+ *		Get the BPF program's return value that will be returned to the upper layers.
  *
- *		This helper is currently supported by cgroup programs only.
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
  *	Return
- *		The syscall's return value.
+ *		The BPF program's return value.
  *
  * int bpf_set_retval(int retval)
  *	Description
- *		Set the syscall's return value that will be returned to userspace.
+ *		Set the BPF program's return value that will be returned to the upper layers.
+ *
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
+ *
+ *		Note that there is the following corner case where the program exports an error
+ *		via bpf_set_retval but signals success via 'return 1':
+ *
+ *			bpf_set_retval(-EPERM);
+ *			return 1;
+ *
+ *		In this case, the BPF program's return value will use helper's -EPERM. This
+ *		still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case.
  *
- *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
  *
 	BPF_F_SEQ_NUMBER		= (1ULL << 3),
 };
+/* BPF_FUNC_skb_get_tunnel_key flags. */
+enum {
+	BPF_F_TUNINFO_FLAGS		= (1ULL << 4),
+};
 /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and
  * BPF_FUNC_perf_event_read_value flags.
  */
 	};
 	__u8 tunnel_tos;
 	__u8 tunnel_ttl;
-	__u16 tunnel_ext;	/* Padding, future use. */
+	union {
+		__u16 tunnel_ext;	/* compat */
+		__be16 tunnel_flags;
+	};
 	__u32 tunnel_label;
 	union {
 		__u32 local_ipv4;
 	 *    represented by BPF_REDIRECT above).
 	 */
 	BPF_LWT_REROUTE = 128,
+	/* BPF_FLOW_DISSECTOR_CONTINUE: used by BPF_PROG_TYPE_FLOW_DISSECTOR
+	 *   to indicate that no custom dissection was performed, and
+	 *   fallback to standard dissector is requested.
+	 */
+	BPF_FLOW_DISSECTOR_CONTINUE = 129,
 };
 struct bpf_sock {
 		struct {
 			__aligned_u64 target_name; /* in/out: target_name buffer ptr */
 			__u32 target_name_len;	   /* in/out: target_name buffer len */
+			/* If the iter specific field is 32 bits, it can be put
+			 * in the first or second union. Otherwise it should be
+			 * put in the second union.
+			 */
 			union {
 				struct {
 					__u32 map_id;
 				} map;
 			};
+			union {
+				struct {
+					__u64 cgroup_id;
+					__u32 order;
+				} cgroup;
+			};
 		} iter;
 		struct  {
 			__u32 netns_ino;

kernel/bpf/Makefile+4−1

 obj-${CONFIG_BPF_LSM}	  += bpf_inode_storage.o
 obj-$(CONFIG_BPF_SYSCALL) += disasm.o
 obj-$(CONFIG_BPF_JIT) += trampoline.o
-obj-$(CONFIG_BPF_SYSCALL) += btf.o
+obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o
 obj-$(CONFIG_BPF_JIT) += dispatcher.o
 ifeq ($(CONFIG_NET),y)
 obj-$(CONFIG_BPF_SYSCALL) += devmap.o
 ifeq ($(CONFIG_PERF_EVENTS),y)
 obj-$(CONFIG_BPF_SYSCALL) += stackmap.o
 endif
+ifeq ($(CONFIG_CGROUPS),y)
+obj-$(CONFIG_BPF_SYSCALL) += cgroup_iter.o
+endif
 obj-$(CONFIG_CGROUP_BPF) += cgroup.o
 ifeq ($(CONFIG_INET),y)
 obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o

kernel/bpf/bpf_iter.c+5−0

 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
+	struct bpf_run_ctx run_ctx, *old_run_ctx;
 	int ret;
 	if (prog->aux->sleepable) {
 		rcu_read_lock_trace();
 		migrate_disable();
 		might_fault();
+		old_run_ctx = bpf_set_run_ctx(&run_ctx);
 		ret = bpf_prog_run(prog, ctx);
+		bpf_reset_run_ctx(old_run_ctx);
 		migrate_enable();
 		rcu_read_unlock_trace();
 	} else {
 		rcu_read_lock();
 		migrate_disable();
+		old_run_ctx = bpf_set_run_ctx(&run_ctx);
 		ret = bpf_prog_run(prog, ctx);
+		bpf_reset_run_ctx(old_run_ctx);
 		migrate_enable();
 		rcu_read_unlock();
 	}

kernel/bpf/bpf_local_storage.c+2−2

@@ -555,11 +555,11 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, · 554 unmodified lines
555	struct bpf_local_storage_elem, map_node))) {	555	struct bpf_local_storage_elem, map_node))) {
556	if (busy_counter) {	556	if (busy_counter) {
557	migrate_disable();	557	migrate_disable();
558	__this_cpu_inc(*busy_counter);	558	this_cpu_inc(*busy_counter);
559	}	559	}
560	bpf_selem_unlink(selem, false);	560	bpf_selem_unlink(selem, false);
561	if (busy_counter) {	561	if (busy_counter) {
562	__this_cpu_dec(*busy_counter);	562	this_cpu_dec(*busy_counter);
563	migrate_enable();	563	migrate_enable();
564	}	564	}
565	cond_resched_rcu();	565	cond_resched_rcu();

kernel/bpf/bpf_lsm.c+14−9

  */
 BTF_SET_START(bpf_lsm_current_hooks)
 /* operate on freshly allocated sk without any cgroup association */
+#ifdef CONFIG_SECURITY_NETWORK
 BTF_ID(func, bpf_lsm_sk_alloc_security)
 BTF_ID(func, bpf_lsm_sk_free_security)
+#endif
 BTF_SET_END(bpf_lsm_current_hooks)
 /* List of LSM hooks that trigger while the socket is properly locked.
  */
 BTF_SET_START(bpf_lsm_locked_sockopt_hooks)
+#ifdef CONFIG_SECURITY_NETWORK
 BTF_ID(func, bpf_lsm_socket_sock_rcv_skb)
 BTF_ID(func, bpf_lsm_sock_graft)
 BTF_ID(func, bpf_lsm_inet_csk_clone)
 BTF_ID(func, bpf_lsm_inet_conn_established)
+#endif
 BTF_SET_END(bpf_lsm_locked_sockopt_hooks)
 /* List of LSM hooks that trigger while the socket is _not_ locked,
  * in the early init phase.
  */
 BTF_SET_START(bpf_lsm_unlocked_sockopt_hooks)
+#ifdef CONFIG_SECURITY_NETWORK
 BTF_ID(func, bpf_lsm_socket_post_create)
 BTF_ID(func, bpf_lsm_socket_socketpair)
+#endif
 BTF_SET_END(bpf_lsm_unlocked_sockopt_hooks)
 #ifdef CONFIG_CGROUP_BPF
 static const struct bpf_func_proto *
 bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+	if (prog->expected_attach_type == BPF_LSM_CGROUP) {
+		func_proto = cgroup_common_func_proto(func_id, prog);
+		if (func_proto)
+			return func_proto;
+	}
 	switch (func_id) {
 	case BPF_FUNC_inode_storage_get:
 		return &bpf_inode_storage_get_proto;
 		return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL;
 	case BPF_FUNC_get_attach_cookie:
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL;
-	case BPF_FUNC_get_local_storage:
-		return prog->expected_attach_type == BPF_LSM_CGROUP ?
-			&bpf_get_local_storage_proto : NULL;
-	case BPF_FUNC_set_retval:
-		return prog->expected_attach_type == BPF_LSM_CGROUP ?
-			&bpf_set_retval_proto : NULL;
-	case BPF_FUNC_get_retval:
-		return prog->expected_attach_type == BPF_LSM_CGROUP ?
-			&bpf_get_retval_proto : NULL;
 #ifdef CONFIG_NET
 	case BPF_FUNC_setsockopt:
 		if (prog->expected_attach_type != BPF_LSM_CGROUP)

kernel/bpf/bpf_task_storage.c+4−4

@@ -26,20 +26,20 @@ static DEFINE_PER_CPU(int, bpf_task_storage_busy); · 25 unmodified lines
26	static void bpf_task_storage_lock(void)	26	static void bpf_task_storage_lock(void)
27	{	27	{
28	migrate_disable();	28	migrate_disable();
29	__this_cpu_inc(bpf_task_storage_busy);	29	this_cpu_inc(bpf_task_storage_busy);
30	}	30	}
31		31
32	static void bpf_task_storage_unlock(void)	32	static void bpf_task_storage_unlock(void)
33	{	33	{
34	__this_cpu_dec(bpf_task_storage_busy);	34	this_cpu_dec(bpf_task_storage_busy);
35	migrate_enable();	35	migrate_enable();
36	}	36	}
37		37
38	static bool bpf_task_storage_trylock(void)	38	static bool bpf_task_storage_trylock(void)
39	{	39	{
40	migrate_disable();	40	migrate_disable();
41	if (unlikely(__this_cpu_inc_return(bpf_task_storage_busy) != 1)) {	41	if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) {
42	__this_cpu_dec(bpf_task_storage_busy);	42	this_cpu_dec(bpf_task_storage_busy);
43	migrate_enable();	43	migrate_enable();
44	return false;	44	return false;
45	}	45	}

kernel/bpf/cgroup.c+134−23

@@ -1529,6 +1529,37 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor, · 1528 unmodified lines
1529	return ret;	1529	return ret;
1530	}	1530	}
1531		1531
		1532	BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
		1533	{
		1534	/* flags argument is not used now,
		1535	* but provides an ability to extend the API.
		1536	* verifier checks that its value is correct.
		1537	*/
		1538	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
		1539	struct bpf_cgroup_storage *storage;
		1540	struct bpf_cg_run_ctx *ctx;
		1541	void *ptr;
		1542
		1543	/* get current cgroup storage from BPF run context */
		1544	ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
		1545	storage = ctx->prog_item->cgroup_storage[stype];
		1546
		1547	if (stype == BPF_CGROUP_STORAGE_SHARED)
		1548	ptr = &READ_ONCE(storage->buf)->data[0];
		1549	else
		1550	ptr = this_cpu_ptr(storage->percpu_buf);
		1551
		1552	return (unsigned long)ptr;
		1553	}
		1554
		1555	const struct bpf_func_proto bpf_get_local_storage_proto = {
		1556	.func = bpf_get_local_storage,
		1557	.gpl_only = false,
		1558	.ret_type = RET_PTR_TO_MAP_VALUE,
		1559	.arg1_type = ARG_CONST_MAP_PTR,
		1560	.arg2_type = ARG_ANYTHING,
		1561	};
		1562
1532	BPF_CALL_0(bpf_get_retval)	1563	BPF_CALL_0(bpf_get_retval)
1533	{	1564	{
1534	struct bpf_cg_run_ctx *ctx =	1565	struct bpf_cg_run_ctx *ctx =
@@ -1560,32 +1591,26 @@ const struct bpf_func_proto bpf_set_retval_proto = { · 25 unmodified lines
1560	};	1591	};
1561		1592
1562	static const struct bpf_func_proto *	1593	static const struct bpf_func_proto *
1563	cgroup_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	1594	cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1564	{	1595	{
		1596	const struct bpf_func_proto *func_proto;
		1597
		1598	func_proto = cgroup_common_func_proto(func_id, prog);
		1599	if (func_proto)
		1600	return func_proto;
		1601
		1602	func_proto = cgroup_current_func_proto(func_id, prog);
		1603	if (func_proto)
		1604	return func_proto;
		1605
1565	switch (func_id) {	1606	switch (func_id) {
1566	case BPF_FUNC_get_current_uid_gid:
1567	return &bpf_get_current_uid_gid_proto;
1568	case BPF_FUNC_get_local_storage:
1569	return &bpf_get_local_storage_proto;
1570	case BPF_FUNC_get_current_cgroup_id:
1571	return &bpf_get_current_cgroup_id_proto;
1572	case BPF_FUNC_perf_event_output:	1607	case BPF_FUNC_perf_event_output:
1573	return &bpf_event_output_data_proto;	1608	return &bpf_event_output_data_proto;
1574	case BPF_FUNC_get_retval:
1575	return &bpf_get_retval_proto;
1576	case BPF_FUNC_set_retval:
1577	return &bpf_set_retval_proto;
1578	default:	1609	default:
1579	return bpf_base_func_proto(func_id);	1610	return bpf_base_func_proto(func_id);
1580	}	1611	}
1581	}	1612	}
1582		1613
1583	static const struct bpf_func_proto *
1584	cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1585	{
1586	return cgroup_base_func_proto(func_id, prog);
1587	}
1588
1589	static bool cgroup_dev_is_valid_access(int off, int size,	1614	static bool cgroup_dev_is_valid_access(int off, int size,
1590	enum bpf_access_type type,	1615	enum bpf_access_type type,
1591	const struct bpf_prog *prog,	1616	const struct bpf_prog *prog,
@@ -2098,11 +2123,17 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = { · 506 unmodified lines
2098	static const struct bpf_func_proto *	2123	static const struct bpf_func_proto *
2099	sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	2124	sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2100	{	2125	{
		2126	const struct bpf_func_proto *func_proto;
		2127
		2128	func_proto = cgroup_common_func_proto(func_id, prog);
		2129	if (func_proto)
		2130	return func_proto;
		2131
		2132	func_proto = cgroup_current_func_proto(func_id, prog);
		2133	if (func_proto)
		2134	return func_proto;
		2135
2101	switch (func_id) {	2136	switch (func_id) {
2102	case BPF_FUNC_strtol:
2103	return &bpf_strtol_proto;
2104	case BPF_FUNC_strtoul:
2105	return &bpf_strtoul_proto;
2106	case BPF_FUNC_sysctl_get_name:	2137	case BPF_FUNC_sysctl_get_name:
2107	return &bpf_sysctl_get_name_proto;	2138	return &bpf_sysctl_get_name_proto;
2108	case BPF_FUNC_sysctl_get_current_value:	2139	case BPF_FUNC_sysctl_get_current_value:
@@ -2113,8 +2144,10 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 4 unmodified lines
2113	return &bpf_sysctl_set_new_value_proto;	2144	return &bpf_sysctl_set_new_value_proto;
2114	case BPF_FUNC_ktime_get_coarse_ns:	2145	case BPF_FUNC_ktime_get_coarse_ns:
2115	return &bpf_ktime_get_coarse_ns_proto;	2146	return &bpf_ktime_get_coarse_ns_proto;
		2147	case BPF_FUNC_perf_event_output:
		2148	return &bpf_event_output_data_proto;
2116	default:	2149	default:
2117	return cgroup_base_func_proto(func_id, prog);	2150	return bpf_base_func_proto(func_id);
2118	}	2151	}
2119	}	2152	}
2120		2153
@@ -2235,6 +2268,16 @@ static const struct bpf_func_proto bpf_get_netns_cookie_sockopt_proto = { · 114 unmodified lines
2235	static const struct bpf_func_proto *	2268	static const struct bpf_func_proto *
2236	cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	2269	cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
2237	{	2270	{
		2271	const struct bpf_func_proto *func_proto;
		2272
		2273	func_proto = cgroup_common_func_proto(func_id, prog);
		2274	if (func_proto)
		2275	return func_proto;
		2276
		2277	func_proto = cgroup_current_func_proto(func_id, prog);
		2278	if (func_proto)
		2279	return func_proto;
		2280
2238	switch (func_id) {	2281	switch (func_id) {
2239	#ifdef CONFIG_NET	2282	#ifdef CONFIG_NET
2240	case BPF_FUNC_get_netns_cookie:	2283	case BPF_FUNC_get_netns_cookie:
@@ -2256,8 +2299,10 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 15 unmodified lines
2256	case BPF_FUNC_tcp_sock:	2299	case BPF_FUNC_tcp_sock:
2257	return &bpf_tcp_sock_proto;	2300	return &bpf_tcp_sock_proto;
2258	#endif	2301	#endif
		2302	case BPF_FUNC_perf_event_output:
		2303	return &bpf_event_output_data_proto;
2259	default:	2304	default:
2260	return cgroup_base_func_proto(func_id, prog);	2305	return bpf_base_func_proto(func_id);
2261	}	2306	}
2262	}	2307	}
2263		2308
@@ -2422,3 +2467,69 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = { · 158 unmodified lines
2422		2467
2423	const struct bpf_prog_ops cg_sockopt_prog_ops = {	2468	const struct bpf_prog_ops cg_sockopt_prog_ops = {
2424	};	2469	};
		2470
		2471	/* Common helpers for cgroup hooks. */
		2472	const struct bpf_func_proto *
		2473	cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
		2474	{
		2475	switch (func_id) {
		2476	case BPF_FUNC_get_local_storage:
		2477	return &bpf_get_local_storage_proto;
		2478	case BPF_FUNC_get_retval:
		2479	switch (prog->expected_attach_type) {
		2480	case BPF_CGROUP_INET_INGRESS:
		2481	case BPF_CGROUP_INET_EGRESS:
		2482	case BPF_CGROUP_SOCK_OPS:
		2483	case BPF_CGROUP_UDP4_RECVMSG:
		2484	case BPF_CGROUP_UDP6_RECVMSG:
		2485	case BPF_CGROUP_INET4_GETPEERNAME:
		2486	case BPF_CGROUP_INET6_GETPEERNAME:
		2487	case BPF_CGROUP_INET4_GETSOCKNAME:
		2488	case BPF_CGROUP_INET6_GETSOCKNAME:
		2489	return NULL;
		2490	default:
		2491	return &bpf_get_retval_proto;
		2492	}
		2493	case BPF_FUNC_set_retval:
		2494	switch (prog->expected_attach_type) {
		2495	case BPF_CGROUP_INET_INGRESS:
		2496	case BPF_CGROUP_INET_EGRESS:
		2497	case BPF_CGROUP_SOCK_OPS:
		2498	case BPF_CGROUP_UDP4_RECVMSG:
		2499	case BPF_CGROUP_UDP6_RECVMSG:
		2500	case BPF_CGROUP_INET4_GETPEERNAME:
		2501	case BPF_CGROUP_INET6_GETPEERNAME:
		2502	case BPF_CGROUP_INET4_GETSOCKNAME:
		2503	case BPF_CGROUP_INET6_GETSOCKNAME:
		2504	return NULL;
		2505	default:
		2506	return &bpf_set_retval_proto;
		2507	}
		2508	default:
		2509	return NULL;
		2510	}
		2511	}
		2512
		2513	/* Common helpers for cgroup hooks with valid process context. */
		2514	const struct bpf_func_proto *
		2515	cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
		2516	{
		2517	switch (func_id) {
		2518	case BPF_FUNC_get_current_uid_gid:
		2519	return &bpf_get_current_uid_gid_proto;
		2520	case BPF_FUNC_get_current_pid_tgid:
		2521	return &bpf_get_current_pid_tgid_proto;
		2522	case BPF_FUNC_get_current_comm:
		2523	return &bpf_get_current_comm_proto;
		2524	case BPF_FUNC_get_current_cgroup_id:
		2525	return &bpf_get_current_cgroup_id_proto;
		2526	case BPF_FUNC_get_current_ancestor_cgroup_id:
		2527	return &bpf_get_current_ancestor_cgroup_id_proto;
		2528	#ifdef CONFIG_CGROUP_NET_CLASSID
		2529	case BPF_FUNC_get_cgroup_classid:
		2530	return &bpf_get_cgroup_classid_curr_proto;
		2531	#endif
		2532	default:
		2533	return NULL;
		2534	}
		2535	}

kernel/bpf/cgroup_iter.c+282−0

@@ -0,0 +1,282 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2022 Google */
	3	#include <linux/bpf.h>
	4	#include <linux/btf_ids.h>
	5	#include <linux/cgroup.h>
	6	#include <linux/kernel.h>
	7	#include <linux/seq_file.h>
	8
	9	#include "../cgroup/cgroup-internal.h" /* cgroup_mutex and cgroup_is_dead */
	10
	11	/* cgroup_iter provides four modes of traversal to the cgroup hierarchy.
	12	*
	13	* 1. Walk the descendants of a cgroup in pre-order.
	14	* 2. Walk the descendants of a cgroup in post-order.
	15	* 3. Walk the ancestors of a cgroup.
	16	* 4. Show the given cgroup only.
	17	*
	18	* For walking descendants, cgroup_iter can walk in either pre-order or
	19	* post-order. For walking ancestors, the iter walks up from a cgroup to
	20	* the root.
	21	*
	22	* The iter program can terminate the walk early by returning 1. Walk
	23	* continues if prog returns 0.
	24	*
	25	* The prog can check (seq->num == 0) to determine whether this is
	26	* the first element. The prog may also be passed a NULL cgroup,
	27	* which means the walk has completed and the prog has a chance to
	28	* do post-processing, such as outputting an epilogue.
	29	*
	30	* Note: the iter_prog is called with cgroup_mutex held.
	31	*
	32	* Currently only one session is supported, which means, depending on the
	33	* volume of data bpf program intends to send to user space, the number
	34	* of cgroups that can be walked is limited. For example, given the current
	35	* buffer size is 8 * PAGE_SIZE, if the program sends 64B data for each
	36	* cgroup, assuming PAGE_SIZE is 4kb, the total number of cgroups that can
	37	* be walked is 512. This is a limitation of cgroup_iter. If the output data
	38	* is larger than the kernel buffer size, after all data in the kernel buffer
	39	* is consumed by user space, the subsequent read() syscall will signal
	40	* EOPNOTSUPP. In order to work around, the user may have to update their
	41	* program to reduce the volume of data sent to output. For example, skip
	42	* some uninteresting cgroups.
	43	*/
	44
	45	struct bpf_iter__cgroup {
	46	__bpf_md_ptr(struct bpf_iter_meta *, meta);
	47	__bpf_md_ptr(struct cgroup *, cgroup);
	48	};
	49
	50	struct cgroup_iter_priv {
	51	struct cgroup_subsys_state *start_css;
	52	bool visited_all;
	53	bool terminate;
	54	int order;
	55	};
	56
	57	static void cgroup_iter_seq_start(struct seq_file seq, loff_t *pos)
	58	{
	59	struct cgroup_iter_priv *p = seq->private;
	60
	61	mutex_lock(&cgroup_mutex);
	62
	63	/* cgroup_iter doesn't support read across multiple sessions. */
	64	if (*pos > 0) {
	65	if (p->visited_all)
	66	return NULL;
	67
	68	/* Haven't visited all, but because cgroup_mutex has dropped,
	69	* return -EOPNOTSUPP to indicate incomplete iteration.
	70	*/
	71	return ERR_PTR(-EOPNOTSUPP);
	72	}
	73
	74	++*pos;
	75	p->terminate = false;
	76	p->visited_all = false;
	77	if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE)
	78	return css_next_descendant_pre(NULL, p->start_css);
	79	else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST)
	80	return css_next_descendant_post(NULL, p->start_css);
	81	else /* BPF_CGROUP_ITER_SELF_ONLY and BPF_CGROUP_ITER_ANCESTORS_UP */
	82	return p->start_css;
	83	}
	84
	85	static int __cgroup_iter_seq_show(struct seq_file *seq,
	86	struct cgroup_subsys_state *css, int in_stop);
	87
	88	static void cgroup_iter_seq_stop(struct seq_file seq, void v)
	89	{
	90	struct cgroup_iter_priv *p = seq->private;
	91
	92	mutex_unlock(&cgroup_mutex);
	93
	94	/* pass NULL to the prog for post-processing */
	95	if (!v) {
	96	__cgroup_iter_seq_show(seq, NULL, true);
	97	p->visited_all = true;
	98	}
	99	}
	100
	101	static void cgroup_iter_seq_next(struct seq_file seq, void v, loff_t pos)
	102	{
	103	struct cgroup_subsys_state curr = (struct cgroup_subsys_state )v;
	104	struct cgroup_iter_priv *p = seq->private;
	105
	106	++*pos;
	107	if (p->terminate)
	108	return NULL;
	109
	110	if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE)
	111	return css_next_descendant_pre(curr, p->start_css);
	112	else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST)
	113	return css_next_descendant_post(curr, p->start_css);
	114	else if (p->order == BPF_CGROUP_ITER_ANCESTORS_UP)
	115	return curr->parent;
	116	else /* BPF_CGROUP_ITER_SELF_ONLY */
	117	return NULL;
	118	}
	119
	120	static int __cgroup_iter_seq_show(struct seq_file *seq,
	121	struct cgroup_subsys_state *css, int in_stop)
	122	{
	123	struct cgroup_iter_priv *p = seq->private;
	124	struct bpf_iter__cgroup ctx;
	125	struct bpf_iter_meta meta;
	126	struct bpf_prog *prog;
	127	int ret = 0;
	128
	129	/* cgroup is dead, skip this element */
	130	if (css && cgroup_is_dead(css->cgroup))
	131	return 0;
	132
	133	ctx.meta = &meta;
	134	ctx.cgroup = css ? css->cgroup : NULL;
	135	meta.seq = seq;
	136	prog = bpf_iter_get_info(&meta, in_stop);
	137	if (prog)
	138	ret = bpf_iter_run_prog(prog, &ctx);
	139
	140	/* if prog returns > 0, terminate after this element. */
	141	if (ret != 0)
	142	p->terminate = true;
	143
	144	return 0;
	145	}
	146
	147	static int cgroup_iter_seq_show(struct seq_file seq, void v)
	148	{
	149	return __cgroup_iter_seq_show(seq, (struct cgroup_subsys_state *)v,
	150	false);
	151	}
	152
	153	static const struct seq_operations cgroup_iter_seq_ops = {
	154	.start = cgroup_iter_seq_start,
	155	.next = cgroup_iter_seq_next,
	156	.stop = cgroup_iter_seq_stop,
	157	.show = cgroup_iter_seq_show,
	158	};
	159
	160	BTF_ID_LIST_SINGLE(bpf_cgroup_btf_id, struct, cgroup)
	161
	162	static int cgroup_iter_seq_init(void priv, struct bpf_iter_aux_info aux)
	163	{
	164	struct cgroup_iter_priv p = (struct cgroup_iter_priv )priv;
	165	struct cgroup *cgrp = aux->cgroup.start;
	166
	167	p->start_css = &cgrp->self;
	168	p->terminate = false;
	169	p->visited_all = false;
	170	p->order = aux->cgroup.order;
	171	return 0;
	172	}
	173
	174	static const struct bpf_iter_seq_info cgroup_iter_seq_info = {
	175	.seq_ops = &cgroup_iter_seq_ops,
	176	.init_seq_private = cgroup_iter_seq_init,
	177	.seq_priv_size = sizeof(struct cgroup_iter_priv),
	178	};
	179
	180	static int bpf_iter_attach_cgroup(struct bpf_prog *prog,
	181	union bpf_iter_link_info *linfo,
	182	struct bpf_iter_aux_info *aux)
	183	{
	184	int fd = linfo->cgroup.cgroup_fd;
	185	u64 id = linfo->cgroup.cgroup_id;
	186	int order = linfo->cgroup.order;
	187	struct cgroup *cgrp;
	188
	189	if (order != BPF_CGROUP_ITER_DESCENDANTS_PRE &&
	190	order != BPF_CGROUP_ITER_DESCENDANTS_POST &&
	191	order != BPF_CGROUP_ITER_ANCESTORS_UP &&
	192	order != BPF_CGROUP_ITER_SELF_ONLY)
	193	return -EINVAL;
	194
	195	if (fd && id)
	196	return -EINVAL;
	197
	198	if (fd)
	199	cgrp = cgroup_get_from_fd(fd);
	200	else if (id)
	201	cgrp = cgroup_get_from_id(id);
	202	else /* walk the entire hierarchy by default. */
	203	cgrp = cgroup_get_from_path("/");
	204
	205	if (IS_ERR(cgrp))
	206	return PTR_ERR(cgrp);
	207
	208	aux->cgroup.start = cgrp;
	209	aux->cgroup.order = order;
	210	return 0;
	211	}
	212
	213	static void bpf_iter_detach_cgroup(struct bpf_iter_aux_info *aux)
	214	{
	215	cgroup_put(aux->cgroup.start);
	216	}
	217
	218	static void bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info *aux,
	219	struct seq_file *seq)
	220	{
	221	char *buf;
	222
	223	buf = kzalloc(PATH_MAX, GFP_KERNEL);
	224	if (!buf) {
	225	seq_puts(seq, "cgroup_path:\t<unknown>\n");
	226	goto show_order;
	227	}
	228
	229	/* If cgroup_path_ns() fails, buf will be an empty string, cgroup_path
	230	* will print nothing.
	231	*
	232	* Path is in the calling process's cgroup namespace.
	233	*/
	234	cgroup_path_ns(aux->cgroup.start, buf, PATH_MAX,
	235	current->nsproxy->cgroup_ns);
	236	seq_printf(seq, "cgroup_path:\t%s\n", buf);
	237	kfree(buf);
	238
	239	show_order:
	240	if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_PRE)
	241	seq_puts(seq, "order: descendants_pre\n");
	242	else if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_POST)
	243	seq_puts(seq, "order: descendants_post\n");
	244	else if (aux->cgroup.order == BPF_CGROUP_ITER_ANCESTORS_UP)
	245	seq_puts(seq, "order: ancestors_up\n");
	246	else /* BPF_CGROUP_ITER_SELF_ONLY */
	247	seq_puts(seq, "order: self_only\n");
	248	}
	249
	250	static int bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info *aux,
	251	struct bpf_link_info *info)
	252	{
	253	info->iter.cgroup.order = aux->cgroup.order;
	254	info->iter.cgroup.cgroup_id = cgroup_id(aux->cgroup.start);
	255	return 0;
	256	}
	257
	258	DEFINE_BPF_ITER_FUNC(cgroup, struct bpf_iter_meta *meta,
	259	struct cgroup *cgroup)
	260
	261	static struct bpf_iter_reg bpf_cgroup_reg_info = {
	262	.target = "cgroup",
	263	.feature = BPF_ITER_RESCHED,
	264	.attach_target = bpf_iter_attach_cgroup,
	265	.detach_target = bpf_iter_detach_cgroup,
	266	.show_fdinfo = bpf_iter_cgroup_show_fdinfo,
	267	.fill_link_info = bpf_iter_cgroup_fill_link_info,
	268	.ctx_arg_info_size = 1,
	269	.ctx_arg_info = {
	270	{ offsetof(struct bpf_iter__cgroup, cgroup),
	271	PTR_TO_BTF_ID_OR_NULL },
	272	},
	273	.seq_info = &cgroup_iter_seq_info,
	274	};
	275
	276	static int __init bpf_cgroup_iter_init(void)
	277	{
	278	bpf_cgroup_reg_info.ctx_arg_info[0].btf_id = bpf_cgroup_btf_id[0];
	279	return bpf_iter_reg_target(&bpf_cgroup_reg_info);
	280	}
	281
	282	late_initcall(bpf_cgroup_iter_init);

kernel/bpf/hashtab.c+123−45

 #include "percpu_freelist.h"
 #include "bpf_lru_list.h"
 #include "map_in_map.h"
+#include <linux/bpf_mem_alloc.h>
 #define HTAB_CREATE_FLAG_MASK						\
 	(BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE |	\
 struct bpf_htab {
 	struct bpf_map map;
+	struct bpf_mem_alloc ma;
+	struct bpf_mem_alloc pcpu_ma;
 	struct bucket *buckets;
 	void *elems;
 	union {
 		struct bpf_lru lru;
 	};
 	struct htab_elem *__percpu *extra_elems;
-	atomic_t count;	/* number of elements in this hashtable */
+	/* number of elements in non-preallocated hashtable are kept
+	 * in either pcount or count
+	 */
+	struct percpu_counter pcount;
+	atomic_t count;
+	bool use_percpu_counter;
 	u32 n_buckets;	/* number of hash buckets */
 	u32 elem_size;	/* size of each element in bytes */
 	u32 hashrnd;
 		struct {
 			void *padding;
 			union {
-				struct bpf_htab *htab;
 				struct pcpu_freelist_node fnode;
 				struct htab_elem *batch_flink;
 			};
 		};
 	};
 	union {
-		struct rcu_head rcu;
+		/* pointer to per-cpu pointer */
+		void *ptr_to_pptr;
 		struct bpf_lru_node lru_node;
 	};
 	u32 hash;
 				   unsigned long *pflags)
 {
 	unsigned long flags;
+	bool use_raw_lock;
 	hash = hash & HASHTAB_MAP_LOCK_MASK;
-	migrate_disable();
+	use_raw_lock = htab_use_raw_lock(htab);
+	if (use_raw_lock)
+		preempt_disable();
+	else
+		migrate_disable();
 	if (unlikely(__this_cpu_inc_return(*(htab->map_locked[hash])) != 1)) {
 		__this_cpu_dec(*(htab->map_locked[hash]));
-		migrate_enable();
+		if (use_raw_lock)
+			preempt_enable();
+		else
+			migrate_enable();
 		return -EBUSY;
 	}
-	if (htab_use_raw_lock(htab))
+	if (use_raw_lock)
 		raw_spin_lock_irqsave(&b->raw_lock, flags);
 	else
 		spin_lock_irqsave(&b->lock, flags);
 				      struct bucket *b, u32 hash,
 				      unsigned long flags)
 {
+	bool use_raw_lock = htab_use_raw_lock(htab);
 	hash = hash & HASHTAB_MAP_LOCK_MASK;
-	if (htab_use_raw_lock(htab))
+	if (use_raw_lock)
 		raw_spin_unlock_irqrestore(&b->raw_lock, flags);
 	else
 		spin_unlock_irqrestore(&b->lock, flags);
 	__this_cpu_dec(*(htab->map_locked[hash]));
-	migrate_enable();
+	if (use_raw_lock)
+		preempt_enable();
+	else
+		migrate_enable();
 }
 static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node);
 	bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED);
 	int numa_node = bpf_map_attr_numa_node(attr);
-	BUILD_BUG_ON(offsetof(struct htab_elem, htab) !=
-		     offsetof(struct htab_elem, hash_node.pprev));
 	BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
 		     offsetof(struct htab_elem, hash_node.pprev));
 	htab_init_buckets(htab);
+/* compute_batch_value() computes batch value as num_online_cpus() * 2
+ * and __percpu_counter_compare() needs
+ * htab->max_entries - cur_number_of_elems to be more than batch * num_online_cpus()
+ * for percpu_counter to be faster than atomic_t. In practice the average bpf
+ * hash map size is 10k, which means that a system with 64 cpus will fill
+ * hashmap to 20% of 10k before percpu_counter becomes ineffective. Therefore
+ * define our own batch count as 32 then 10k hash map can be filled up to 80%:
+ * 10k - 8k > 32 _batch_ * 64 _cpus_
+ * and __percpu_counter_compare() will still be fast. At that point hash map
+ * collisions will dominate its performance anyway. Assume that hash map filled
+ * to 50+% isn't going to be O(1) and use the following formula to choose
+ * between percpu_counter and atomic_t.
+ */
+#define PERCPU_COUNTER_BATCH 32
+	if (attr->max_entries / 2 > num_online_cpus() * PERCPU_COUNTER_BATCH)
+		htab->use_percpu_counter = true;
+	if (htab->use_percpu_counter) {
+		err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL);
+		if (err)
+			goto free_map_locked;
+	}
 	if (prealloc) {
 		err = prealloc_init(htab);
 		if (err)
 			if (err)
 				goto free_prealloc;
 		}
+	} else {
+		err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false);
+		if (err)
+			goto free_map_locked;
+		if (percpu) {
+			err = bpf_mem_alloc_init(&htab->pcpu_ma,
+						 round_up(htab->map.value_size, 8), true);
+			if (err)
+				goto free_map_locked;
+		}
 	}
 	return &htab->map;
 	for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
 		free_percpu(htab->map_locked[i]);
 	bpf_map_area_free(htab->buckets);
+	bpf_mem_alloc_destroy(&htab->pcpu_ma);
+	bpf_mem_alloc_destroy(&htab->ma);
 free_htab:
 	lockdep_unregister_key(&htab->lockdep_key);
 	bpf_map_area_free(htab);
 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
 {
 	if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
-		free_percpu(htab_elem_get_ptr(l, htab->map.key_size));
+		bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
 	check_and_free_fields(htab, l);
-	kfree(l);
+	bpf_mem_cache_free(&htab->ma, l);
-}
-static void htab_elem_free_rcu(struct rcu_head *head)
-{
-	struct htab_elem *l = container_of(head, struct htab_elem, rcu);
-	struct bpf_htab *htab = l->htab;
-	htab_elem_free(htab, l);
 }
 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
 	}
 }
+static bool is_map_full(struct bpf_htab *htab)
+{
+	if (htab->use_percpu_counter)
+		return __percpu_counter_compare(&htab->pcount, htab->map.max_entries,
+						PERCPU_COUNTER_BATCH) >= 0;
+	return atomic_read(&htab->count) >= htab->map.max_entries;
+}
+static void inc_elem_count(struct bpf_htab *htab)
+{
+	if (htab->use_percpu_counter)
+		percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH);
+	else
+		atomic_inc(&htab->count);
+}
+static void dec_elem_count(struct bpf_htab *htab)
+{
+	if (htab->use_percpu_counter)
+		percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH);
+	else
+		atomic_dec(&htab->count);
+}
 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
 {
 	htab_put_fd_value(htab, l);
 		check_and_free_fields(htab, l);
 		__pcpu_freelist_push(&htab->freelist, &l->fnode);
 	} else {
-		atomic_dec(&htab->count);
+		dec_elem_count(htab);
-		l->htab = htab;
+		htab_elem_free(htab, l);
-		call_rcu(&l->rcu, htab_elem_free_rcu);
 	}
 }
 static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
 			    void *value, bool onallcpus)
 {
-	/* When using prealloc and not setting the initial value on all cpus,
+	/* When not setting the initial value on all cpus, zero-fill element
-	 * zero-fill element values for other cpus (just as what happens when
+	 * values for other cpus. Otherwise, bpf program has no way to ensure
-	 * not using prealloc). Otherwise, bpf program has no way to ensure
 	 * known initial values for cpus other than current one
 	 * (onallcpus=false always when coming from bpf prog).
 	 */
-	if (htab_is_prealloc(htab) && !onallcpus) {
+	if (!onallcpus) {
 		u32 size = round_up(htab->map.value_size, 8);
 		int current_cpu = raw_smp_processor_id();
 		int cpu;
 			l_new = container_of(l, struct htab_elem, fnode);
 		}
 	} else {
-		if (atomic_inc_return(&htab->count) > htab->map.max_entries)
+		if (is_map_full(htab))
-			if (!old_elem) {
+			if (!old_elem)
 				/* when map is full and update() is replacing
 				 * old element, it's ok to allocate, since
 				 * old element will be freed immediately.
 				 * Otherwise return an error
 				 */
-				l_new = ERR_PTR(-E2BIG);
+				return ERR_PTR(-E2BIG);
-				goto dec_count;
+		inc_elem_count(htab);
-			}
+		l_new = bpf_mem_cache_alloc(&htab->ma);
-		l_new = bpf_map_kmalloc_node(&htab->map, htab->elem_size,
-					     GFP_NOWAIT | __GFP_NOWARN,
-					     htab->map.numa_node);
 		if (!l_new) {
 			l_new = ERR_PTR(-ENOMEM);
 			goto dec_count;
 	memcpy(l_new->key, key, key_size);
 	if (percpu) {
-		size = round_up(size, 8);
 		if (prealloc) {
 			pptr = htab_elem_get_ptr(l_new, key_size);
 		} else {
 			/* alloc_percpu zero-fills */
-			pptr = bpf_map_alloc_percpu(&htab->map, size, 8,
+			pptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
-						    GFP_NOWAIT | __GFP_NOWARN);
 			if (!pptr) {
-				kfree(l_new);
+				bpf_mem_cache_free(&htab->ma, l_new);
 				l_new = ERR_PTR(-ENOMEM);
 				goto dec_count;
 			}
+			l_new->ptr_to_pptr = pptr;
+			pptr = *(void **)pptr;
 		}
 		pcpu_init_value(htab, pptr, value, onallcpus);
 	l_new->hash = hash;
 	return l_new;
 dec_count:
-	atomic_dec(&htab->count);
+	dec_elem_count(htab);
 	return l_new;
 }
 {
 	int i;
+	/* It's called from a worker thread, so disable migration here,
+	 * since bpf_mem_cache_free() relies on that.
+	 */
+	migrate_disable();
 	for (i = 0; i < htab->n_buckets; i++) {
 		struct hlist_nulls_head *head = select_bucket(htab, i);
 		struct hlist_nulls_node *n;
 			htab_elem_free(htab, l);
 		}
 	}
+	migrate_enable();
 }
 static void htab_free_malloced_timers(struct bpf_htab *htab)
 	 * There is no need to synchronize_rcu() here to protect map elements.
 	 */
-	/* some of free_htab_elem() callbacks for elements of this map may
+	/* htab no longer uses call_rcu() directly. bpf_mem_alloc does it
-	 * not have executed. Wait for them.
+	 * underneath and is reponsible for waiting for callbacks to finish
+	 * during bpf_mem_alloc_destroy().
 	 */
-	rcu_barrier();
 	if (!htab_is_prealloc(htab)) {
 		delete_all_elements(htab);
 	} else {
 	bpf_map_free_kptr_off_tab(map);
 	free_percpu(htab->extra_elems);
 	bpf_map_area_free(htab->buckets);
+	bpf_mem_alloc_destroy(&htab->pcpu_ma);
+	bpf_mem_alloc_destroy(&htab->ma);
+	if (htab->use_percpu_counter)
+		percpu_counter_destroy(&htab->pcount);
 	for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
 		free_percpu(htab->map_locked[i]);
 	lockdep_unregister_key(&htab->lockdep_key);
 	/* do not grab the lock unless need it (bucket_cnt > 0). */
 	if (locked) {
 		ret = htab_lock_bucket(htab, b, batch, &flags);
-		if (ret)
+		if (ret) {
-			goto next_batch;
+			rcu_read_unlock();
+			bpf_enable_instrumentation();
+			goto after_loop;
+		}
 	}
 	bucket_cnt = 0;

kernel/bpf/helpers.c+9−39

@@ -427,40 +427,7 @@ const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = { · 426 unmodified lines
427	.ret_type = RET_INTEGER,	427	.ret_type = RET_INTEGER,
428	.arg1_type = ARG_ANYTHING,	428	.arg1_type = ARG_ANYTHING,
429	};	429	};
430		430	#endif /* CONFIG_CGROUPS */
431	#ifdef CONFIG_CGROUP_BPF
432
433	BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
434	{
435	/* flags argument is not used now,
436	* but provides an ability to extend the API.
437	* verifier checks that its value is correct.
438	*/
439	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
440	struct bpf_cgroup_storage *storage;
441	struct bpf_cg_run_ctx *ctx;
442	void *ptr;
443
444	/* get current cgroup storage from BPF run context */
445	ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
446	storage = ctx->prog_item->cgroup_storage[stype];
447
448	if (stype == BPF_CGROUP_STORAGE_SHARED)
449	ptr = &READ_ONCE(storage->buf)->data[0];
450	else
451	ptr = this_cpu_ptr(storage->percpu_buf);
452
453	return (unsigned long)ptr;
454	}
455
456	const struct bpf_func_proto bpf_get_local_storage_proto = {
457	.func = bpf_get_local_storage,
458	.gpl_only = false,
459	.ret_type = RET_PTR_TO_MAP_VALUE,
460	.arg1_type = ARG_CONST_MAP_PTR,
461	.arg2_type = ARG_ANYTHING,
462	};
463	#endif
464		431
465	#define BPF_STRTOX_BASE_MASK 0x1F	432	#define BPF_STRTOX_BASE_MASK 0x1F
466		433
@@ -589,7 +556,6 @@ const struct bpf_func_proto bpf_strtoul_proto = { · 122 unmodified lines
589	.arg3_type = ARG_ANYTHING,	556	.arg3_type = ARG_ANYTHING,
590	.arg4_type = ARG_PTR_TO_LONG,	557	.arg4_type = ARG_PTR_TO_LONG,
591	};	558	};
592	#endif
593		559
594	BPF_CALL_3(bpf_strncmp, const char , s1, u32, s1_sz, const char , s2)	560	BPF_CALL_3(bpf_strncmp, const char , s1, u32, s1_sz, const char , s2)
595	{	561	{
@@ -1647,12 +1613,12 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 1051 unmodified lines
1647	return &bpf_ringbuf_submit_dynptr_proto;	1613	return &bpf_ringbuf_submit_dynptr_proto;
1648	case BPF_FUNC_ringbuf_discard_dynptr:	1614	case BPF_FUNC_ringbuf_discard_dynptr:
1649	return &bpf_ringbuf_discard_dynptr_proto;	1615	return &bpf_ringbuf_discard_dynptr_proto;
1650	case BPF_FUNC_for_each_map_elem:
1651	return &bpf_for_each_map_elem_proto;
1652	case BPF_FUNC_loop:
1653	return &bpf_loop_proto;
1654	case BPF_FUNC_strncmp:	1616	case BPF_FUNC_strncmp:
1655	return &bpf_strncmp_proto;	1617	return &bpf_strncmp_proto;
		1618	case BPF_FUNC_strtol:
		1619	return &bpf_strtol_proto;
		1620	case BPF_FUNC_strtoul:
		1621	return &bpf_strtoul_proto;
1656	case BPF_FUNC_dynptr_from_mem:	1622	case BPF_FUNC_dynptr_from_mem:
1657	return &bpf_dynptr_from_mem_proto;	1623	return &bpf_dynptr_from_mem_proto;
1658	case BPF_FUNC_dynptr_read:	1624	case BPF_FUNC_dynptr_read:
@@ -1689,6 +1655,10 @@ bpf_base_func_proto(enum bpf_func_id func_id) · 30 unmodified lines
1689	return &bpf_timer_cancel_proto;	1655	return &bpf_timer_cancel_proto;
1690	case BPF_FUNC_kptr_xchg:	1656	case BPF_FUNC_kptr_xchg:
1691	return &bpf_kptr_xchg_proto;	1657	return &bpf_kptr_xchg_proto;
		1658	case BPF_FUNC_for_each_map_elem:
		1659	return &bpf_for_each_map_elem_proto;
		1660	case BPF_FUNC_loop:
		1661	return &bpf_loop_proto;
1692	default:	1662	default:
1693	break;	1663	break;
1694	}	1664	}

kernel/bpf/memalloc.c+634−0

@@ -0,0 +1,634 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
	3	#include <linux/mm.h>
	4	#include <linux/llist.h>
	5	#include <linux/bpf.h>
	6	#include <linux/irq_work.h>
	7	#include <linux/bpf_mem_alloc.h>
	8	#include <linux/memcontrol.h>
	9	#include <asm/local.h>
	10
	11	/* Any context (including NMI) BPF specific memory allocator.
	12	*
	13	* Tracing BPF programs can attach to kprobe and fentry. Hence they
	14	* run in unknown context where calling plain kmalloc() might not be safe.
	15	*
	16	* Front-end kmalloc() with per-cpu per-bucket cache of free elements.
	17	* Refill this cache asynchronously from irq_work.
	18	*
	19	* CPU_0 buckets
	20	* 16 32 64 96 128 196 256 512 1024 2048 4096
	21	* ...
	22	* CPU_N buckets
	23	* 16 32 64 96 128 196 256 512 1024 2048 4096
	24	*
	25	* The buckets are prefilled at the start.
	26	* BPF programs always run with migration disabled.
	27	* It's safe to allocate from cache of the current cpu with irqs disabled.
	28	* Free-ing is always done into bucket of the current cpu as well.
	29	* irq_work trims extra free elements from buckets with kfree
	30	* and refills them with kmalloc, so global kmalloc logic takes care
	31	* of freeing objects allocated by one cpu and freed on another.
	32	*
	33	* Every allocated objected is padded with extra 8 bytes that contains
	34	* struct llist_node.
	35	*/
	36	#define LLIST_NODE_SZ sizeof(struct llist_node)
	37
	38	/* similar to kmalloc, but sizeof == 8 bucket is gone */
	39	static u8 size_index[24] __ro_after_init = {
	40	3, /* 8 */
	41	3, /* 16 */
	42	4, /* 24 */
	43	4, /* 32 */
	44	5, /* 40 */
	45	5, /* 48 */
	46	5, /* 56 */
	47	5, /* 64 */
	48	1, /* 72 */
	49	1, /* 80 */
	50	1, /* 88 */
	51	1, /* 96 */
	52	6, /* 104 */
	53	6, /* 112 */
	54	6, /* 120 */
	55	6, /* 128 */
	56	2, /* 136 */
	57	2, /* 144 */
	58	2, /* 152 */
	59	2, /* 160 */
	60	2, /* 168 */
	61	2, /* 176 */
	62	2, /* 184 */
	63	2 /* 192 */
	64	};
	65
	66	static int bpf_mem_cache_idx(size_t size)
	67	{
	68	if (!size \|\| size > 4096)
	69	return -1;
	70
	71	if (size <= 192)
	72	return size_index[(size - 1) / 8] - 1;
	73
	74	return fls(size - 1) - 1;
	75	}
	76
	77	#define NUM_CACHES 11
	78
	79	struct bpf_mem_cache {
	80	/* per-cpu list of free objects of size 'unit_size'.
	81	* All accesses are done with interrupts disabled and 'active' counter
	82	* protection with __llist_add() and __llist_del_first().
	83	*/
	84	struct llist_head free_llist;
	85	local_t active;
	86
	87	/* Operations on the free_list from unit_alloc/unit_free/bpf_mem_refill
	88	* are sequenced by per-cpu 'active' counter. But unit_free() cannot
	89	* fail. When 'active' is busy the unit_free() will add an object to
	90	* free_llist_extra.
	91	*/
	92	struct llist_head free_llist_extra;
	93
	94	struct irq_work refill_work;
	95	struct obj_cgroup *objcg;
	96	int unit_size;
	97	/* count of objects in free_llist */
	98	int free_cnt;
	99	int low_watermark, high_watermark, batch;
	100	int percpu_size;
	101
	102	struct rcu_head rcu;
	103	struct llist_head free_by_rcu;
	104	struct llist_head waiting_for_gp;
	105	atomic_t call_rcu_in_progress;
	106	};
	107
	108	struct bpf_mem_caches {
	109	struct bpf_mem_cache cache[NUM_CACHES];
	110	};
	111
	112	static struct llist_node notrace __llist_del_first(struct llist_head head)
	113	{
	114	struct llist_node entry, next;
	115
	116	entry = head->first;
	117	if (!entry)
	118	return NULL;
	119	next = entry->next;
	120	head->first = next;
	121	return entry;
	122	}
	123
	124	static void __alloc(struct bpf_mem_cache c, int node)
	125	{
	126	/* Allocate, but don't deplete atomic reserves that typical
	127	* GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc
	128	* will allocate from the current numa node which is what we
	129	* want here.
	130	*/
	131	gfp_t flags = GFP_NOWAIT \| __GFP_NOWARN \| __GFP_ACCOUNT;
	132
	133	if (c->percpu_size) {
	134	void **obj = kmalloc_node(c->percpu_size, flags, node);
	135	void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
	136
	137	if (!obj \|\| !pptr) {
	138	free_percpu(pptr);
	139	kfree(obj);
	140	return NULL;
	141	}
	142	obj[1] = pptr;
	143	return obj;
	144	}
	145
	146	return kmalloc_node(c->unit_size, flags, node);
	147	}
	148
	149	static struct mem_cgroup get_memcg(const struct bpf_mem_cache c)
	150	{
	151	#ifdef CONFIG_MEMCG_KMEM
	152	if (c->objcg)
	153	return get_mem_cgroup_from_objcg(c->objcg);
	154	#endif
	155
	156	#ifdef CONFIG_MEMCG
	157	return root_mem_cgroup;
	158	#else
	159	return NULL;
	160	#endif
	161	}
	162
	163	/* Mostly runs from irq_work except __init phase. */
	164	static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node)
	165	{
	166	struct mem_cgroup memcg = NULL, old_memcg;
	167	unsigned long flags;
	168	void *obj;
	169	int i;
	170
	171	memcg = get_memcg(c);
	172	old_memcg = set_active_memcg(memcg);
	173	for (i = 0; i < cnt; i++) {
	174	obj = __alloc(c, node);
	175	if (!obj)
	176	break;
	177	if (IS_ENABLED(CONFIG_PREEMPT_RT))
	178	/* In RT irq_work runs in per-cpu kthread, so disable
	179	* interrupts to avoid preemption and interrupts and
	180	* reduce the chance of bpf prog executing on this cpu
	181	* when active counter is busy.
	182	*/
	183	local_irq_save(flags);
	184	/* alloc_bulk runs from irq_work which will not preempt a bpf
	185	* program that does unit_alloc/unit_free since IRQs are
	186	* disabled there. There is no race to increment 'active'
	187	* counter. It protects free_llist from corruption in case NMI
	188	* bpf prog preempted this loop.
	189	*/
	190	WARN_ON_ONCE(local_inc_return(&c->active) != 1);
	191	__llist_add(obj, &c->free_llist);
	192	c->free_cnt++;
	193	local_dec(&c->active);
	194	if (IS_ENABLED(CONFIG_PREEMPT_RT))
	195	local_irq_restore(flags);
	196	}
	197	set_active_memcg(old_memcg);
	198	mem_cgroup_put(memcg);
	199	}
	200
	201	static void free_one(struct bpf_mem_cache c, void obj)
	202	{
	203	if (c->percpu_size) {
	204	free_percpu(((void **)obj)[1]);
	205	kfree(obj);
	206	return;
	207	}
	208
	209	kfree(obj);
	210	}
	211
	212	static void __free_rcu(struct rcu_head *head)
	213	{
	214	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);
	215	struct llist_node *llnode = llist_del_all(&c->waiting_for_gp);
	216	struct llist_node pos, t;
	217
	218	llist_for_each_safe(pos, t, llnode)
	219	free_one(c, pos);
	220	atomic_set(&c->call_rcu_in_progress, 0);
	221	}
	222
	223	static void __free_rcu_tasks_trace(struct rcu_head *head)
	224	{
	225	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);
	226
	227	call_rcu(&c->rcu, __free_rcu);
	228	}
	229
	230	static void enque_to_free(struct bpf_mem_cache c, void obj)
	231	{
	232	struct llist_node *llnode = obj;
	233
	234	/* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work.
	235	* Nothing races to add to free_by_rcu list.
	236	*/
	237	__llist_add(llnode, &c->free_by_rcu);
	238	}
	239
	240	static void do_call_rcu(struct bpf_mem_cache *c)
	241	{
	242	struct llist_node llnode, t;
	243
	244	if (atomic_xchg(&c->call_rcu_in_progress, 1))
	245	return;
	246
	247	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));
	248	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu))
	249	/* There is no concurrent __llist_add(waiting_for_gp) access.
	250	* It doesn't race with llist_del_all either.
	251	* But there could be two concurrent llist_del_all(waiting_for_gp):
	252	* from __free_rcu() and from drain_mem_cache().
	253	*/
	254	__llist_add(llnode, &c->waiting_for_gp);
	255	/* Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
	256	* Then use call_rcu() to wait for normal progs to finish
	257	* and finally do free_one() on each element.
	258	*/
	259	call_rcu_tasks_trace(&c->rcu, __free_rcu_tasks_trace);
	260	}
	261
	262	static void free_bulk(struct bpf_mem_cache *c)
	263	{
	264	struct llist_node llnode, t;
	265	unsigned long flags;
	266	int cnt;
	267
	268	do {
	269	if (IS_ENABLED(CONFIG_PREEMPT_RT))
	270	local_irq_save(flags);
	271	WARN_ON_ONCE(local_inc_return(&c->active) != 1);
	272	llnode = __llist_del_first(&c->free_llist);
	273	if (llnode)
	274	cnt = --c->free_cnt;
	275	else
	276	cnt = 0;
	277	local_dec(&c->active);
	278	if (IS_ENABLED(CONFIG_PREEMPT_RT))
	279	local_irq_restore(flags);
	280	enque_to_free(c, llnode);
	281	} while (cnt > (c->high_watermark + c->low_watermark) / 2);
	282
	283	/* and drain free_llist_extra */
	284	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra))
	285	enque_to_free(c, llnode);
	286	do_call_rcu(c);
	287	}
	288
	289	static void bpf_mem_refill(struct irq_work *work)
	290	{
	291	struct bpf_mem_cache *c = container_of(work, struct bpf_mem_cache, refill_work);
	292	int cnt;
	293
	294	/* Racy access to free_cnt. It doesn't need to be 100% accurate */
	295	cnt = c->free_cnt;
	296	if (cnt < c->low_watermark)
	297	/* irq_work runs on this cpu and kmalloc will allocate
	298	* from the current numa node which is what we want here.
	299	*/
	300	alloc_bulk(c, c->batch, NUMA_NO_NODE);
	301	else if (cnt > c->high_watermark)
	302	free_bulk(c);
	303	}
	304
	305	static void notrace irq_work_raise(struct bpf_mem_cache *c)
	306	{
	307	irq_work_queue(&c->refill_work);
	308	}
	309
	310	/* For typical bpf map case that uses bpf_mem_cache_alloc and single bucket
	311	* the freelist cache will be elem_size * 64 (or less) on each cpu.
	312	*
	313	* For bpf programs that don't have statically known allocation sizes and
	314	* assuming (low_mark + high_mark) / 2 as an average number of elements per
	315	* bucket and all buckets are used the total amount of memory in freelists
	316	* on each cpu will be:
	317	* 6416 + 6432 + 6464 + 6496 + 64128 + 64196 + 64256 + 32512 + 161024 + 82048 + 4*4096
	318	* == ~ 116 Kbyte using below heuristic.
	319	* Initialized, but unused bpf allocator (not bpf map specific one) will
	320	* consume ~ 11 Kbyte per cpu.
	321	* Typical case will be between 11K and 116K closer to 11K.
	322	* bpf progs can and should share bpf_mem_cache when possible.
	323	*/
	324
	325	static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu)
	326	{
	327	init_irq_work(&c->refill_work, bpf_mem_refill);
	328	if (c->unit_size <= 256) {
	329	c->low_watermark = 32;
	330	c->high_watermark = 96;
	331	} else {
	332	/* When page_size == 4k, order-0 cache will have low_mark == 2
	333	* and high_mark == 6 with batch alloc of 3 individual pages at
	334	* a time.
	335	* 8k allocs and above low == 1, high == 3, batch == 1.
	336	*/
	337	c->low_watermark = max(32 * 256 / c->unit_size, 1);
	338	c->high_watermark = max(96 * 256 / c->unit_size, 3);
	339	}
	340	c->batch = max((c->high_watermark - c->low_watermark) / 4 * 3, 1);
	341
	342	/* To avoid consuming memory assume that 1st run of bpf
	343	* prog won't be doing more than 4 map_update_elem from
	344	* irq disabled region
	345	*/
	346	alloc_bulk(c, c->unit_size <= 256 ? 4 : 1, cpu_to_node(cpu));
	347	}
	348
	349	/* When size != 0 bpf_mem_cache for each cpu.
	350	* This is typical bpf hash map use case when all elements have equal size.
	351	*
	352	* When size == 0 allocate 11 bpf_mem_cache-s for each cpu, then rely on
	353	* kmalloc/kfree. Max allocation size is 4096 in this case.
	354	* This is bpf_dynptr and bpf_kptr use case.
	355	*/
	356	int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
	357	{
	358	static u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096};
	359	struct bpf_mem_caches cc, __percpu pcc;
	360	struct bpf_mem_cache c, __percpu pc;
	361	struct obj_cgroup *objcg = NULL;
	362	int cpu, i, unit_size, percpu_size = 0;
	363
	364	if (size) {
	365	pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
	366	if (!pc)
	367	return -ENOMEM;
	368
	369	if (percpu)
	370	/* room for llist_node and per-cpu pointer */
	371	percpu_size = LLIST_NODE_SZ + sizeof(void *);
	372	else
	373	size += LLIST_NODE_SZ; /* room for llist_node */
	374	unit_size = size;
	375
	376	#ifdef CONFIG_MEMCG_KMEM
	377	objcg = get_obj_cgroup_from_current();
	378	#endif
	379	for_each_possible_cpu(cpu) {
	380	c = per_cpu_ptr(pc, cpu);
	381	c->unit_size = unit_size;
	382	c->objcg = objcg;
	383	c->percpu_size = percpu_size;
	384	prefill_mem_cache(c, cpu);
	385	}
	386	ma->cache = pc;
	387	return 0;
	388	}
	389
	390	/* size == 0 && percpu is an invalid combination */
	391	if (WARN_ON_ONCE(percpu))
	392	return -EINVAL;
	393
	394	pcc = __alloc_percpu_gfp(sizeof(*cc), 8, GFP_KERNEL);
	395	if (!pcc)
	396	return -ENOMEM;
	397	#ifdef CONFIG_MEMCG_KMEM
	398	objcg = get_obj_cgroup_from_current();
	399	#endif
	400	for_each_possible_cpu(cpu) {
	401	cc = per_cpu_ptr(pcc, cpu);
	402	for (i = 0; i < NUM_CACHES; i++) {
	403	c = &cc->cache[i];
	404	c->unit_size = sizes[i];
	405	c->objcg = objcg;
	406	prefill_mem_cache(c, cpu);
	407	}
	408	}
	409	ma->caches = pcc;
	410	return 0;
	411	}
	412
	413	static void drain_mem_cache(struct bpf_mem_cache *c)
	414	{
	415	struct llist_node llnode, t;
	416
	417	/* No progs are using this bpf_mem_cache, but htab_map_free() called
	418	* bpf_mem_cache_free() for all remaining elements and they can be in
	419	* free_by_rcu or in waiting_for_gp lists, so drain those lists now.
	420	*/
	421	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu))
	422	free_one(c, llnode);
	423	llist_for_each_safe(llnode, t, llist_del_all(&c->waiting_for_gp))
	424	free_one(c, llnode);
	425	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist))
	426	free_one(c, llnode);
	427	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra))
	428	free_one(c, llnode);
	429	}
	430
	431	static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)
	432	{
	433	free_percpu(ma->cache);
	434	free_percpu(ma->caches);
	435	ma->cache = NULL;
	436	ma->caches = NULL;
	437	}
	438
	439	static void free_mem_alloc(struct bpf_mem_alloc *ma)
	440	{
	441	/* waiting_for_gp lists was drained, but __free_rcu might
	442	* still execute. Wait for it now before we freeing percpu caches.
	443	*/
	444	rcu_barrier_tasks_trace();
	445	rcu_barrier();
	446	free_mem_alloc_no_barrier(ma);
	447	}
	448
	449	static void free_mem_alloc_deferred(struct work_struct *work)
	450	{
	451	struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work);
	452
	453	free_mem_alloc(ma);
	454	kfree(ma);
	455	}
	456
	457	static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress)
	458	{
	459	struct bpf_mem_alloc *copy;
	460
	461	if (!rcu_in_progress) {
	462	/* Fast path. No callbacks are pending, hence no need to do
	463	* rcu_barrier-s.
	464	*/
	465	free_mem_alloc_no_barrier(ma);
	466	return;
	467	}
	468
	469	copy = kmalloc(sizeof(*ma), GFP_KERNEL);
	470	if (!copy) {
	471	/* Slow path with inline barrier-s */
	472	free_mem_alloc(ma);
	473	return;
	474	}
	475
	476	/* Defer barriers into worker to let the rest of map memory to be freed */
	477	copy->cache = ma->cache;
	478	ma->cache = NULL;
	479	copy->caches = ma->caches;
	480	ma->caches = NULL;
	481	INIT_WORK(&copy->work, free_mem_alloc_deferred);
	482	queue_work(system_unbound_wq, &copy->work);
	483	}
	484
	485	void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma)
	486	{
	487	struct bpf_mem_caches *cc;
	488	struct bpf_mem_cache *c;
	489	int cpu, i, rcu_in_progress;
	490
	491	if (ma->cache) {
	492	rcu_in_progress = 0;
	493	for_each_possible_cpu(cpu) {
	494	c = per_cpu_ptr(ma->cache, cpu);
	495	drain_mem_cache(c);
	496	rcu_in_progress += atomic_read(&c->call_rcu_in_progress);
	497	}
	498	/* objcg is the same across cpus */
	499	if (c->objcg)
	500	obj_cgroup_put(c->objcg);
	501	destroy_mem_alloc(ma, rcu_in_progress);
	502	}
	503	if (ma->caches) {
	504	rcu_in_progress = 0;
	505	for_each_possible_cpu(cpu) {
	506	cc = per_cpu_ptr(ma->caches, cpu);
	507	for (i = 0; i < NUM_CACHES; i++) {
	508	c = &cc->cache[i];
	509	drain_mem_cache(c);
	510	rcu_in_progress += atomic_read(&c->call_rcu_in_progress);
	511	}
	512	}
	513	if (c->objcg)
	514	obj_cgroup_put(c->objcg);
	515	destroy_mem_alloc(ma, rcu_in_progress);
	516	}
	517	}
	518
	519	/* notrace is necessary here and in other functions to make sure
	520	* bpf programs cannot attach to them and cause llist corruptions.
	521	*/
	522	static void notrace unit_alloc(struct bpf_mem_cache c)
	523	{
	524	struct llist_node *llnode = NULL;
	525	unsigned long flags;
	526	int cnt = 0;
	527
	528	/* Disable irqs to prevent the following race for majority of prog types:
	529	* prog_A
	530	* bpf_mem_alloc
	531	* preemption or irq -> prog_B
	532	* bpf_mem_alloc
	533	*
	534	* but prog_B could be a perf_event NMI prog.
	535	* Use per-cpu 'active' counter to order free_list access between
	536	* unit_alloc/unit_free/bpf_mem_refill.
	537	*/
	538	local_irq_save(flags);
	539	if (local_inc_return(&c->active) == 1) {
	540	llnode = __llist_del_first(&c->free_llist);
	541	if (llnode)
	542	cnt = --c->free_cnt;
	543	}
	544	local_dec(&c->active);
	545	local_irq_restore(flags);
	546
	547	WARN_ON(cnt < 0);
	548
	549	if (cnt < c->low_watermark)
	550	irq_work_raise(c);
	551	return llnode;
	552	}
	553
	554	/* Though 'ptr' object could have been allocated on a different cpu
	555	* add it to the free_llist of the current cpu.
	556	* Let kfree() logic deal with it when it's later called from irq_work.
	557	*/
	558	static void notrace unit_free(struct bpf_mem_cache c, void ptr)
	559	{
	560	struct llist_node *llnode = ptr - LLIST_NODE_SZ;
	561	unsigned long flags;
	562	int cnt = 0;
	563
	564	BUILD_BUG_ON(LLIST_NODE_SZ > 8);
	565
	566	local_irq_save(flags);
	567	if (local_inc_return(&c->active) == 1) {
	568	__llist_add(llnode, &c->free_llist);
	569	cnt = ++c->free_cnt;
	570	} else {
	571	/* unit_free() cannot fail. Therefore add an object to atomic
	572	* llist. free_bulk() will drain it. Though free_llist_extra is
	573	* a per-cpu list we have to use atomic llist_add here, since
	574	* it also can be interrupted by bpf nmi prog that does another
	575	* unit_free() into the same free_llist_extra.
	576	*/
	577	llist_add(llnode, &c->free_llist_extra);
	578	}
	579	local_dec(&c->active);
	580	local_irq_restore(flags);
	581
	582	if (cnt > c->high_watermark)
	583	/* free few objects from current cpu into global kmalloc pool */
	584	irq_work_raise(c);
	585	}
	586
	587	/* Called from BPF program or from sys_bpf syscall.
	588	* In both cases migration is disabled.
	589	*/
	590	void notrace bpf_mem_alloc(struct bpf_mem_alloc ma, size_t size)
	591	{
	592	int idx;
	593	void *ret;
	594
	595	if (!size)
	596	return ZERO_SIZE_PTR;
	597
	598	idx = bpf_mem_cache_idx(size + LLIST_NODE_SZ);
	599	if (idx < 0)
	600	return NULL;
	601
	602	ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx);
	603	return !ret ? NULL : ret + LLIST_NODE_SZ;
	604	}
	605
	606	void notrace bpf_mem_free(struct bpf_mem_alloc ma, void ptr)
	607	{
	608	int idx;
	609
	610	if (!ptr)
	611	return;
	612
	613	idx = bpf_mem_cache_idx(__ksize(ptr - LLIST_NODE_SZ));
	614	if (idx < 0)
	615	return;
	616
	617	unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr);
	618	}
	619
	620	void notrace bpf_mem_cache_alloc(struct bpf_mem_alloc ma)
	621	{
	622	void *ret;
	623
	624	ret = unit_alloc(this_cpu_ptr(ma->cache));
	625	return !ret ? NULL : ret + LLIST_NODE_SZ;
	626	}
	627
	628	void notrace bpf_mem_cache_free(struct bpf_mem_alloc ma, void ptr)
	629	{
	630	if (!ptr)
	631	return;
	632
	633	unit_free(this_cpu_ptr(ma->cache), ptr);
	634	}

kernel/bpf/syscall.c+10−5

 		bpf_map_free_id(map, do_idr_lock);
 		btf_put(map->btf);
 		INIT_WORK(&map->work, bpf_map_free_deferred);
-		schedule_work(&map->work);
+		/* Avoid spawning kworkers, since they all might contend
+		 * for the same mutex like slab_mutex.
+		 */
+		queue_work(system_unbound_wq, &map->work);
 	}
 }
 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
-static int map_delete_elem(union bpf_attr *attr)
+static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)
 {
-	void __user *ukey = u64_to_user_ptr(attr->key);
+	bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
 	int ufd = attr->map_fd;
 	struct bpf_map *map;
 	struct fd f;
 		goto err_put;
 	}
-	key = __bpf_copy_key(ukey, map->key_size);
+	key = ___bpf_copy_key(ukey, map->key_size);
 	if (IS_ERR(key)) {
 		err = PTR_ERR(key);
 		goto err_put;
 		err = map_update_elem(&attr, uattr);
 		break;
 	case BPF_MAP_DELETE_ELEM:
-		err = map_delete_elem(&attr);
+		err = map_delete_elem(&attr, uattr);
 		break;
 	case BPF_MAP_GET_NEXT_KEY:
 		err = map_get_next_key(&attr);
 {
 	switch (cmd) {
 	case BPF_MAP_CREATE:
+	case BPF_MAP_DELETE_ELEM:
 	case BPF_MAP_UPDATE_ELEM:
 	case BPF_MAP_FREEZE:
+	case BPF_MAP_GET_FD_BY_ID:
 	case BPF_PROG_LOAD:
 	case BPF_BTF_LOAD:
 	case BPF_LINK_CREATE:

kernel/bpf/trampoline.c+4−4

@@ -895,7 +895,7 @@ u64 notrace __bpf_prog_enter(struct bpf_prog prog, struct bpf_tramp_run_ctx ru · 894 unmodified lines
895		895
896	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);	896	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
897		897
898	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {	898	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
899	inc_misses_counter(prog);	899	inc_misses_counter(prog);
900	return 0;	900	return 0;
901	}	901	}
@@ -930,7 +930,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_ · 28 unmodified lines
930	bpf_reset_run_ctx(run_ctx->saved_run_ctx);	930	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
931		931
932	update_prog_stats(prog, start);	932	update_prog_stats(prog, start);
933	__this_cpu_dec(*(prog->active));	933	this_cpu_dec(*(prog->active));
934	migrate_enable();	934	migrate_enable();
935	rcu_read_unlock();	935	rcu_read_unlock();
936	}	936	}
@@ -966,7 +966,7 @@ u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_r · 29 unmodified lines
966	migrate_disable();	966	migrate_disable();
967	might_fault();	967	might_fault();
968		968
969	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {	969	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
970	inc_misses_counter(prog);	970	inc_misses_counter(prog);
971	return 0;	971	return 0;
972	}	972	}
@@ -982,7 +982,7 @@ void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start, · 9 unmodified lines
982	bpf_reset_run_ctx(run_ctx->saved_run_ctx);	982	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
983		983
984	update_prog_stats(prog, start);	984	update_prog_stats(prog, start);
985	__this_cpu_dec(*(prog->active));	985	this_cpu_dec(*(prog->active));
986	migrate_enable();	986	migrate_enable();
987	rcu_read_unlock_trace();	987	rcu_read_unlock_trace();
988	}	988	}

kernel/bpf/verifier.c+33−61

 	id = ++env->id_gen;
 	state->refs[new_ofs].id = id;
 	state->refs[new_ofs].insn_idx = insn_idx;
+	state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0;
 	return id;
 }
 	last_idx = state->acquired_refs - 1;
 	for (i = 0; i < state->acquired_refs; i++) {
 		if (state->refs[i].id == ptr_id) {
+			/* Cannot release caller references in callbacks */
+			if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
+				return -EINVAL;
 			if (last_idx && i != last_idx)
 				memcpy(&state->refs[i], &state->refs[last_idx],
 				       sizeof(*state->refs));
 		caller->regs[BPF_REG_0] = *r0;
 	}
-	/* Transfer references to the caller */
+	/* callback_fn frame should have released its own additions to parent's
-	err = copy_reference_state(caller, callee);
+	 * reference state at this point, or check_reference_leak would
-	if (err)
+	 * complain, hence it must be the same as the caller. There is no need
-		return err;
+	 * to copy it back.
+	 */
+	if (!callee->in_callback_fn) {
+		/* Transfer references to the caller */
+		err = copy_reference_state(caller, callee);
+		if (err)
+			return err;
+	}
 	*insn_idx = callee->callsite + 1;
 	if (env->log.level & BPF_LOG_LEVEL) {
 static int check_reference_leak(struct bpf_verifier_env *env)
 {
 	struct bpf_func_state *state = cur_func(env);
+	bool refs_lingering = false;
 	int i;
+	if (state->frameno && !state->in_callback_fn)
+		return 0;
 	for (i = 0; i < state->acquired_refs; i++) {
+		if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
+			continue;
 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
 			state->refs[i].id, state->refs[i].insn_idx);
+		refs_lingering = true;
 	}
-	return state->acquired_refs ? -EINVAL : 0;
+	return refs_lingering ? -EINVAL : 0;
 }
 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
 					return -EINVAL;
 				}
+				/* We must do check_reference_leak here before
+				 * prepare_func_exit to handle the case when
+				 * state->curframe > 0, it may be a callback
+				 * function, for which reference_state must
+				 * match caller reference state when it exits.
+				 */
+				err = check_reference_leak(env);
+				if (err)
+					return err;
 				if (state->curframe) {
 					/* exit from nested function */
 					err = prepare_func_exit(env, &env->insn_idx);
 					continue;
 				}
-				err = check_reference_leak(env);
-				if (err)
-					return err;
 				err = check_return_code(env);
 				if (err)
 					return err;
 	return err;
 }
-static int check_map_prealloc(struct bpf_map *map)
-{
-	return (map->map_type != BPF_MAP_TYPE_HASH &&
-		map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
-		map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
-		!(map->map_flags & BPF_F_NO_PREALLOC);
-}
 static bool is_tracing_prog_type(enum bpf_prog_type type)
 {
 	switch (type) {
 	}
 }
-static bool is_preallocated_map(struct bpf_map *map)
-{
-	if (!check_map_prealloc(map))
-		return false;
-	if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta))
-		return false;
-	return true;
-}
 static int check_map_prog_compatibility(struct bpf_verifier_env *env,
 					struct bpf_map *map,
 					struct bpf_prog *prog)
 {
 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
-	/*
-	 * Validate that trace type programs use preallocated hash maps.
-	 *
-	 * For programs attached to PERF events this is mandatory as the
-	 * perf NMI can hit any arbitrary code sequence.
-	 *
-	 * All other trace types using preallocated hash maps are unsafe as
-	 * well because tracepoint or kprobes can be inside locked regions
-	 * of the memory allocator or at a place where a recursion into the
-	 * memory allocator would see inconsistent state.
-	 *
-	 * On RT enabled kernels run-time allocation of all trace type
-	 * programs is strictly prohibited due to lock type constraints. On
-	 * !RT kernels it is allowed for backwards compatibility reasons for
-	 * now, but warnings are emitted so developers are made aware of
-	 * the unsafety and can fix their programs before this is enforced.
-	 */
-	if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) {
-		if (prog_type == BPF_PROG_TYPE_PERF_EVENT) {
-			verbose(env, "perf_event programs can only use preallocated hash map\n");
-			return -EINVAL;
-		}
-		if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
-			verbose(env, "trace type programs can only use preallocated hash map\n");
-			return -EINVAL;
-		}
-		WARN_ONCE(1, "trace type BPF program uses run-time allocation\n");
-		verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");
-	}
 	if (map_value_has_spin_lock(map)) {
 		if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
 		case BPF_MAP_TYPE_LRU_PERCPU_HASH:
 		case BPF_MAP_TYPE_ARRAY_OF_MAPS:
 		case BPF_MAP_TYPE_HASH_OF_MAPS:
-			if (!is_preallocated_map(map)) {
-				verbose(env,
-					"Sleepable programs can only use preallocated maps\n");
-				return -EINVAL;
-			}
-			break;
 		case BPF_MAP_TYPE_RINGBUF:
 		case BPF_MAP_TYPE_INODE_STORAGE:
 		case BPF_MAP_TYPE_SK_STORAGE:

kernel/cgroup/rstat.c+48−0

@@ -3,6 +3,10 @@ · 2 unmodified lines
3		3
4	#include <linux/sched/cputime.h>	4	#include <linux/sched/cputime.h>
5		5
		6	#include <linux/bpf.h>
		7	#include <linux/btf.h>
		8	#include <linux/btf_ids.h>
		9
6	static DEFINE_SPINLOCK(cgroup_rstat_lock);	10	static DEFINE_SPINLOCK(cgroup_rstat_lock);
7	static DEFINE_PER_CPU(raw_spinlock_t, cgroup_rstat_cpu_lock);	11	static DEFINE_PER_CPU(raw_spinlock_t, cgroup_rstat_cpu_lock);
8		12
@@ -141,6 +145,31 @@ static struct cgroup cgroup_rstat_cpu_pop_updated(struct cgroup pos, · 132 unmodified lines
141	return pos;	145	return pos;
142	}	146	}
143		147
		148	/*
		149	* A hook for bpf stat collectors to attach to and flush their stats.
		150	* Together with providing bpf kfuncs for cgroup_rstat_updated() and
		151	* cgroup_rstat_flush(), this enables a complete workflow where bpf progs that
		152	* collect cgroup stats can integrate with rstat for efficient flushing.
		153	*
		154	* A static noinline declaration here could cause the compiler to optimize away
		155	* the function. A global noinline declaration will keep the definition, but may
		156	* optimize away the callsite. Therefore, __weak is needed to ensure that the
		157	* call is still emitted, by telling the compiler that we don't know what the
		158	* function might eventually be.
		159	*
		160	* __diag_* below are needed to dismiss the missing prototype warning.
		161	*/
		162	__diag_push();
		163	__diag_ignore_all("-Wmissing-prototypes",
		164	"kfuncs which will be used in BPF programs");
		165
		166	__weak noinline void bpf_rstat_flush(struct cgroup *cgrp,
		167	struct cgroup *parent, int cpu)
		168	{
		169	}
		170
		171	__diag_pop();
		172
144	/* see cgroup_rstat_flush() */	173	/* see cgroup_rstat_flush() */
145	static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)	174	static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)
146	__releases(&cgroup_rstat_lock) __acquires(&cgroup_rstat_lock)	175	__releases(&cgroup_rstat_lock) __acquires(&cgroup_rstat_lock)
@@ -168,6 +197,7 @@ static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep) · 21 unmodified lines
168	struct cgroup_subsys_state *css;	197	struct cgroup_subsys_state *css;
169		198
170	cgroup_base_stat_flush(pos, cpu);	199	cgroup_base_stat_flush(pos, cpu);
		200	bpf_rstat_flush(pos, cgroup_parent(pos), cpu);
171		201
172	rcu_read_lock();	202	rcu_read_lock();
173	list_for_each_entry_rcu(css, &pos->rstat_css_list,	203	list_for_each_entry_rcu(css, &pos->rstat_css_list,
@@ -501,3 +531,21 @@ void cgroup_base_stat_cputime_show(struct seq_file *seq) · 327 unmodified lines
501	seq_printf(seq, "core_sched.force_idle_usec %llu\n", forceidle_time);	531	seq_printf(seq, "core_sched.force_idle_usec %llu\n", forceidle_time);
502	#endif	532	#endif
503	}	533	}
		534
		535	/* Add bpf kfuncs for cgroup_rstat_updated() and cgroup_rstat_flush() */
		536	BTF_SET8_START(bpf_rstat_kfunc_ids)
		537	BTF_ID_FLAGS(func, cgroup_rstat_updated)
		538	BTF_ID_FLAGS(func, cgroup_rstat_flush, KF_SLEEPABLE)
		539	BTF_SET8_END(bpf_rstat_kfunc_ids)
		540
		541	static const struct btf_kfunc_id_set bpf_rstat_kfunc_set = {
		542	.owner = THIS_MODULE,
		543	.set = &bpf_rstat_kfunc_ids,
		544	};
		545
		546	static int __init bpf_rstat_kfunc_init(void)
		547	{
		548	return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
		549	&bpf_rstat_kfunc_set);
		550	}
		551	late_initcall(bpf_rstat_kfunc_init);

net/core/filter.c+262−373

@@ -3010,7 +3010,7 @@ BPF_CALL_0(bpf_get_cgroup_classid_curr) · 3009 unmodified lines
3010	return __task_get_classid(current);	3010	return __task_get_classid(current);
3011	}	3011	}
3012		3012
3013	static const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {	3013	const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {
3014	.func = bpf_get_cgroup_classid_curr,	3014	.func = bpf_get_cgroup_classid_curr,
3015	.gpl_only = false,	3015	.gpl_only = false,
3016	.ret_type = RET_INTEGER,	3016	.ret_type = RET_INTEGER,
@@ -4489,7 +4489,8 @@ BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key · 1472 unmodified lines
4489	void *to_orig = to;	4489	void *to_orig = to;
4490	int err;	4490	int err;
4491		4491
4492	if (unlikely(!info \|\| (flags & ~(BPF_F_TUNINFO_IPV6)))) {	4492	if (unlikely(!info \|\| (flags & ~(BPF_F_TUNINFO_IPV6 \|
		4493	BPF_F_TUNINFO_FLAGS)))) {
4493	err = -EINVAL;	4494	err = -EINVAL;
4494	goto err_clear;	4495	goto err_clear;
4495	}	4496	}
@@ -4521,7 +4522,10 @@ set_compat: · 25 unmodified lines
4521	to->tunnel_id = be64_to_cpu(info->key.tun_id);	4522	to->tunnel_id = be64_to_cpu(info->key.tun_id);
4522	to->tunnel_tos = info->key.tos;	4523	to->tunnel_tos = info->key.tos;
4523	to->tunnel_ttl = info->key.ttl;	4524	to->tunnel_ttl = info->key.ttl;
4524	to->tunnel_ext = 0;	4525	if (flags & BPF_F_TUNINFO_FLAGS)
		4526	to->tunnel_flags = info->key.tun_flags;
		4527	else
		4528	to->tunnel_ext = 0;
4525		4529
4526	if (flags & BPF_F_TUNINFO_IPV6) {	4530	if (flags & BPF_F_TUNINFO_IPV6) {
4527	memcpy(to->remote_ipv6, &info->key.u.ipv6.src,	4531	memcpy(to->remote_ipv6, &info->key.u.ipv6.src,
@@ -5014,359 +5018,259 @@ static const struct bpf_func_proto bpf_get_socket_uid_proto = { · 486 unmodified lines
5014	.arg1_type = ARG_PTR_TO_CTX,	5018	.arg1_type = ARG_PTR_TO_CTX,
5015	};	5019	};
5016		5020
5017	static int __bpf_setsockopt(struct sock *sk, int level, int optname,	5021	static int sol_socket_sockopt(struct sock *sk, int optname,
5018	char *optval, int optlen)	5022	char optval, int optlen,
5019	{	5023	bool getopt)
5020	char devname[IFNAMSIZ];	5024	{
5021	int val, valbool;	5025	switch (optname) {
5022	struct net *net;	5026	case SO_REUSEADDR:
5023	int ifindex;	5027	case SO_SNDBUF:
5024	int ret = 0;	5028	case SO_RCVBUF:
5025		5029	case SO_KEEPALIVE:
5026	if (!sk_fullsock(sk))	5030	case SO_PRIORITY:
		5031	case SO_REUSEPORT:
		5032	case SO_RCVLOWAT:
		5033	case SO_MARK:
		5034	case SO_MAX_PACING_RATE:
		5035	case SO_BINDTOIFINDEX:
		5036	case SO_TXREHASH:
		5037	if (*optlen != sizeof(int))
		5038	return -EINVAL;
		5039	break;
		5040	case SO_BINDTODEVICE:
		5041	break;
		5042	default:
5027	return -EINVAL;	5043	return -EINVAL;
		5044	}
5028		5045
5029	if (level == SOL_SOCKET) {	5046	if (getopt) {
5030	if (optlen != sizeof(int) && optname != SO_BINDTODEVICE)	5047	if (optname == SO_BINDTODEVICE)
5031	return -EINVAL;	5048	return -EINVAL;
5032	val = ((int )optval);	5049	return sk_getsockopt(sk, SOL_SOCKET, optname,
5033	valbool = val ? 1 : 0;	5050	KERNEL_SOCKPTR(optval),
5034		5051	KERNEL_SOCKPTR(optlen));
5035	/* Only some socketops are supported */	5052	}
5036	switch (optname) {
5037	case SO_RCVBUF:
5038	val = min_t(u32, val, READ_ONCE(sysctl_rmem_max));
5039	val = min_t(int, val, INT_MAX / 2);
5040	sk->sk_userlocks \|= SOCK_RCVBUF_LOCK;
5041	WRITE_ONCE(sk->sk_rcvbuf,
5042	max_t(int, val * 2, SOCK_MIN_RCVBUF));
5043	break;
5044	case SO_SNDBUF:
5045	val = min_t(u32, val, READ_ONCE(sysctl_wmem_max));
5046	val = min_t(int, val, INT_MAX / 2);
5047	sk->sk_userlocks \|= SOCK_SNDBUF_LOCK;
5048	WRITE_ONCE(sk->sk_sndbuf,
5049	max_t(int, val * 2, SOCK_MIN_SNDBUF));
5050	break;
5051	case SO_MAX_PACING_RATE: /* 32bit version */
5052	if (val != ~0U)
5053	cmpxchg(&sk->sk_pacing_status,
5054	SK_PACING_NONE,
5055	SK_PACING_NEEDED);
5056	sk->sk_max_pacing_rate = (val == ~0U) ?
5057	~0UL : (unsigned int)val;
5058	sk->sk_pacing_rate = min(sk->sk_pacing_rate,
5059	sk->sk_max_pacing_rate);
5060	break;
5061	case SO_PRIORITY:
5062	sk->sk_priority = val;
5063	break;
5064	case SO_RCVLOWAT:
5065	if (val < 0)
5066	val = INT_MAX;
5067	if (sk->sk_socket && sk->sk_socket->ops->set_rcvlowat)
5068	ret = sk->sk_socket->ops->set_rcvlowat(sk, val);
5069	else
5070	WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
5071	break;
5072	case SO_MARK:
5073	if (sk->sk_mark != val) {
5074	sk->sk_mark = val;
5075	sk_dst_reset(sk);
5076	}
5077	break;
5078	case SO_BINDTODEVICE:
5079	optlen = min_t(long, optlen, IFNAMSIZ - 1);
5080	strncpy(devname, optval, optlen);
5081	devname[optlen] = 0;
5082		5053
5083	ifindex = 0;	5054	return sk_setsockopt(sk, SOL_SOCKET, optname,
5084	if (devname[0] != '\0') {	5055	KERNEL_SOCKPTR(optval), *optlen);
5085	struct net_device *dev;	5056	}
5086		5057
5087	ret = -ENODEV;	5058	static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname,
		5059	char *optval, int optlen)
		5060	{
		5061	struct tcp_sock *tp = tcp_sk(sk);
		5062	unsigned long timeout;
		5063	int val;
5088		5064
5089	net = sock_net(sk);	5065	if (optlen != sizeof(int))
5090	dev = dev_get_by_name(net, devname);	5066	return -EINVAL;
5091	if (!dev)
5092	break;
5093	ifindex = dev->ifindex;
5094	dev_put(dev);
5095	}
5096	fallthrough;
5097	case SO_BINDTOIFINDEX:
5098	if (optname == SO_BINDTOIFINDEX)
5099	ifindex = val;
5100	ret = sock_bindtoindex(sk, ifindex, false);
5101	break;
5102	case SO_KEEPALIVE:
5103	if (sk->sk_prot->keepalive)
5104	sk->sk_prot->keepalive(sk, valbool);
5105	sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
5106	break;
5107	case SO_REUSEPORT:
5108	sk->sk_reuseport = valbool;
5109	break;
5110	case SO_TXREHASH:
5111	if (val < -1 \|\| val > 1) {
5112	ret = -EINVAL;
5113	break;
5114	}
5115	sk->sk_txrehash = (u8)val;
5116	break;
5117	default:
5118	ret = -EINVAL;
5119	}
5120	#ifdef CONFIG_INET
5121	} else if (level == SOL_IP) {
5122	if (optlen != sizeof(int) \|\| sk->sk_family != AF_INET)
5123	return -EINVAL;
5124		5067
5125	val = ((int )optval);	5068	val = (int )optval;
5126	/* Only some options are supported */
5127	switch (optname) {
5128	case IP_TOS:
5129	if (val < -1 \|\| val > 0xff) {
5130	ret = -EINVAL;
5131	} else {
5132	struct inet_sock *inet = inet_sk(sk);
5133		5069
5134	if (val == -1)	5070	/* Only some options are supported */
5135	val = 0;	5071	switch (optname) {
5136	inet->tos = val;	5072	case TCP_BPF_IW:
5137	}	5073	if (val <= 0 \|\| tp->data_segs_out > tp->syn_data)
5138	break;	5074	return -EINVAL;
5139	default:	5075	tcp_snd_cwnd_set(tp, val);
5140	ret = -EINVAL;	5076	break;
5141	}	5077	case TCP_BPF_SNDCWND_CLAMP:
5142	#if IS_ENABLED(CONFIG_IPV6)	5078	if (val <= 0)
5143	} else if (level == SOL_IPV6) {	5079	return -EINVAL;
5144	if (optlen != sizeof(int) \|\| sk->sk_family != AF_INET6)	5080	tp->snd_cwnd_clamp = val;
		5081	tp->snd_ssthresh = val;
		5082	break;
		5083	case TCP_BPF_DELACK_MAX:
		5084	timeout = usecs_to_jiffies(val);
		5085	if (timeout > TCP_DELACK_MAX \|\|
		5086	timeout < TCP_TIMEOUT_MIN)
		5087	return -EINVAL;
		5088	inet_csk(sk)->icsk_delack_max = timeout;
		5089	break;
		5090	case TCP_BPF_RTO_MIN:
		5091	timeout = usecs_to_jiffies(val);
		5092	if (timeout > TCP_RTO_MIN \|\|
		5093	timeout < TCP_TIMEOUT_MIN)
5145	return -EINVAL;	5094	return -EINVAL;
		5095	inet_csk(sk)->icsk_rto_min = timeout;
		5096	break;
		5097	default:
		5098	return -EINVAL;
		5099	}
5146		5100
5147	val = ((int )optval);	5101	return 0;
5148	/* Only some options are supported */	5102	}
5149	switch (optname) {
5150	case IPV6_TCLASS:
5151	if (val < -1 \|\| val > 0xff) {
5152	ret = -EINVAL;
5153	} else {
5154	struct ipv6_pinfo *np = inet6_sk(sk);
5155		5103
5156	if (val == -1)	5104	static int sol_tcp_sockopt(struct sock *sk, int optname,
5157	val = 0;	5105	char optval, int optlen,
5158	np->tclass = val;	5106	bool getopt)
5159	}	5107	{
5160	break;	5108	if (sk->sk_prot->setsockopt != tcp_setsockopt)
5161	default:	5109	return -EINVAL;
5162	ret = -EINVAL;
5163	}
5164	#endif
5165	} else if (level == SOL_TCP &&
5166	sk->sk_prot->setsockopt == tcp_setsockopt) {
5167	if (optname == TCP_CONGESTION) {
5168	char name[TCP_CA_NAME_MAX];
5169		5110
5170	strncpy(name, optval, min_t(long, optlen,	5111	switch (optname) {
5171	TCP_CA_NAME_MAX-1));	5112	case TCP_NODELAY:
5172	name[TCP_CA_NAME_MAX-1] = 0;	5113	case TCP_MAXSEG:
5173	ret = tcp_set_congestion_control(sk, name, false, true);	5114	case TCP_KEEPIDLE:
5174	} else {	5115	case TCP_KEEPINTVL:
5175	struct inet_connection_sock *icsk = inet_csk(sk);	5116	case TCP_KEEPCNT:
		5117	case TCP_SYNCNT:
		5118	case TCP_WINDOW_CLAMP:
		5119	case TCP_THIN_LINEAR_TIMEOUTS:
		5120	case TCP_USER_TIMEOUT:
		5121	case TCP_NOTSENT_LOWAT:
		5122	case TCP_SAVE_SYN:
		5123	if (*optlen != sizeof(int))
		5124	return -EINVAL;
		5125	break;
		5126	case TCP_CONGESTION:
		5127	if (*optlen < 2)
		5128	return -EINVAL;
		5129	break;
		5130	case TCP_SAVED_SYN:
		5131	if (*optlen < 1)
		5132	return -EINVAL;
		5133	break;
		5134	default:
		5135	if (getopt)
		5136	return -EINVAL;
		5137	return bpf_sol_tcp_setsockopt(sk, optname, optval, *optlen);
		5138	}
		5139
		5140	if (getopt) {
		5141	if (optname == TCP_SAVED_SYN) {
5176	struct tcp_sock *tp = tcp_sk(sk);	5142	struct tcp_sock *tp = tcp_sk(sk);
5177	unsigned long timeout;
5178		5143
5179	if (optlen != sizeof(int))	5144	if (!tp->saved_syn \|\|
		5145	*optlen > tcp_saved_syn_len(tp->saved_syn))
5180	return -EINVAL;	5146	return -EINVAL;
		5147	memcpy(optval, tp->saved_syn->data, *optlen);
		5148	/* It cannot free tp->saved_syn here because it
		5149	* does not know if the user space still needs it.
		5150	*/
		5151	return 0;
		5152	}
5181		5153
5182	val = ((int )optval);	5154	if (optname == TCP_CONGESTION) {
5183	/* Only some options are supported */	5155	if (!inet_csk(sk)->icsk_ca_ops)
5184	switch (optname) {	5156	return -EINVAL;
5185	case TCP_BPF_IW:	5157	/* BPF expects NULL-terminated tcp-cc string */
5186	if (val <= 0 \|\| tp->data_segs_out > tp->syn_data)	5158	optval[--(*optlen)] = '\0';
5187	ret = -EINVAL;
5188	else
5189	tcp_snd_cwnd_set(tp, val);
5190	break;
5191	case TCP_BPF_SNDCWND_CLAMP:
5192	if (val <= 0) {
5193	ret = -EINVAL;
5194	} else {
5195	tp->snd_cwnd_clamp = val;
5196	tp->snd_ssthresh = val;
5197	}
5198	break;
5199	case TCP_BPF_DELACK_MAX:
5200	timeout = usecs_to_jiffies(val);
5201	if (timeout > TCP_DELACK_MAX \|\|
5202	timeout < TCP_TIMEOUT_MIN)
5203	return -EINVAL;
5204	inet_csk(sk)->icsk_delack_max = timeout;
5205	break;
5206	case TCP_BPF_RTO_MIN:
5207	timeout = usecs_to_jiffies(val);
5208	if (timeout > TCP_RTO_MIN \|\|
5209	timeout < TCP_TIMEOUT_MIN)
5210	return -EINVAL;
5211	inet_csk(sk)->icsk_rto_min = timeout;
5212	break;
5213	case TCP_SAVE_SYN:
5214	if (val < 0 \|\| val > 1)
5215	ret = -EINVAL;
5216	else
5217	tp->save_syn = val;
5218	break;
5219	case TCP_KEEPIDLE:
5220	ret = tcp_sock_set_keepidle_locked(sk, val);
5221	break;
5222	case TCP_KEEPINTVL:
5223	if (val < 1 \|\| val > MAX_TCP_KEEPINTVL)
5224	ret = -EINVAL;
5225	else
5226	tp->keepalive_intvl = val * HZ;
5227	break;
5228	case TCP_KEEPCNT:
5229	if (val < 1 \|\| val > MAX_TCP_KEEPCNT)
5230	ret = -EINVAL;
5231	else
5232	tp->keepalive_probes = val;
5233	break;
5234	case TCP_SYNCNT:
5235	if (val < 1 \|\| val > MAX_TCP_SYNCNT)
5236	ret = -EINVAL;
5237	else
5238	icsk->icsk_syn_retries = val;
5239	break;
5240	case TCP_USER_TIMEOUT:
5241	if (val < 0)
5242	ret = -EINVAL;
5243	else
5244	icsk->icsk_user_timeout = val;
5245	break;
5246	case TCP_NOTSENT_LOWAT:
5247	tp->notsent_lowat = val;
5248	sk->sk_write_space(sk);
5249	break;
5250	case TCP_WINDOW_CLAMP:
5251	ret = tcp_set_window_clamp(sk, val);
5252	break;
5253	default:
5254	ret = -EINVAL;
5255	}
5256	}	5159	}
5257	#endif	5160
5258	} else {	5161	return do_tcp_getsockopt(sk, SOL_TCP, optname,
5259	ret = -EINVAL;	5162	KERNEL_SOCKPTR(optval),
		5163	KERNEL_SOCKPTR(optlen));
5260	}	5164	}
5261	return ret;	5165
		5166	return do_tcp_setsockopt(sk, SOL_TCP, optname,
		5167	KERNEL_SOCKPTR(optval), *optlen);
5262	}	5168	}
5263		5169
5264	static int _bpf_setsockopt(struct sock *sk, int level, int optname,	5170	static int sol_ip_sockopt(struct sock *sk, int optname,
5265	char *optval, int optlen)	5171	char optval, int optlen,
		5172	bool getopt)
5266	{	5173	{
5267	if (sk_fullsock(sk))	5174	if (sk->sk_family != AF_INET)
5268	sock_owned_by_me(sk);	5175	return -EINVAL;
5269	return __bpf_setsockopt(sk, level, optname, optval, optlen);	5176
		5177	switch (optname) {
		5178	case IP_TOS:
		5179	if (*optlen != sizeof(int))
		5180	return -EINVAL;
		5181	break;
		5182	default:
		5183	return -EINVAL;
		5184	}
		5185
		5186	if (getopt)
		5187	return do_ip_getsockopt(sk, SOL_IP, optname,
		5188	KERNEL_SOCKPTR(optval),
		5189	KERNEL_SOCKPTR(optlen));
		5190
		5191	return do_ip_setsockopt(sk, SOL_IP, optname,
		5192	KERNEL_SOCKPTR(optval), *optlen);
5270	}	5193	}
5271		5194
5272	static int __bpf_getsockopt(struct sock *sk, int level, int optname,	5195	static int sol_ipv6_sockopt(struct sock *sk, int optname,
5273	char *optval, int optlen)	5196	char optval, int optlen,
		5197	bool getopt)
5274	{	5198	{
5275	if (!sk_fullsock(sk))	5199	if (sk->sk_family != AF_INET6)
5276	goto err_clear;	5200	return -EINVAL;
5277		5201
5278	if (level == SOL_SOCKET) {	5202	switch (optname) {
5279	if (optlen != sizeof(int))	5203	case IPV6_TCLASS:
5280	goto err_clear;	5204	case IPV6_AUTOFLOWLABEL:
		5205	if (*optlen != sizeof(int))
		5206	return -EINVAL;
		5207	break;
		5208	default:
		5209	return -EINVAL;
		5210	}
5281		5211
5282	switch (optname) {	5212	if (getopt)
5283	case SO_RCVBUF:	5213	return ipv6_bpf_stub->ipv6_getsockopt(sk, SOL_IPV6, optname,
5284	((int )optval) = sk->sk_rcvbuf;	5214	KERNEL_SOCKPTR(optval),
5285	break;	5215	KERNEL_SOCKPTR(optlen));
5286	case SO_SNDBUF:
5287	((int )optval) = sk->sk_sndbuf;
5288	break;
5289	case SO_MARK:
5290	((int )optval) = sk->sk_mark;
5291	break;
5292	case SO_PRIORITY:
5293	((int )optval) = sk->sk_priority;
5294	break;
5295	case SO_BINDTOIFINDEX:
5296	((int )optval) = sk->sk_bound_dev_if;
5297	break;
5298	case SO_REUSEPORT:
5299	((int )optval) = sk->sk_reuseport;
5300	break;
5301	case SO_TXREHASH:
5302	((int )optval) = sk->sk_txrehash;
5303	break;
5304	default:
5305	goto err_clear;
5306	}
5307	#ifdef CONFIG_INET
5308	} else if (level == SOL_TCP && sk->sk_prot->getsockopt == tcp_getsockopt) {
5309	struct inet_connection_sock *icsk;
5310	struct tcp_sock *tp;
5311		5216
5312	switch (optname) {	5217	return ipv6_bpf_stub->ipv6_setsockopt(sk, SOL_IPV6, optname,
5313	case TCP_CONGESTION:	5218	KERNEL_SOCKPTR(optval), *optlen);
5314	icsk = inet_csk(sk);	5219	}
5315		5220
5316	if (!icsk->icsk_ca_ops \|\| optlen <= 1)	5221	static int __bpf_setsockopt(struct sock *sk, int level, int optname,
5317	goto err_clear;	5222	char *optval, int optlen)
5318	strncpy(optval, icsk->icsk_ca_ops->name, optlen);	5223	{
5319	optval[optlen - 1] = 0;	5224	if (!sk_fullsock(sk))
5320	break;	5225	return -EINVAL;
5321	case TCP_SAVED_SYN:
5322	tp = tcp_sk(sk);
5323		5226
5324	if (optlen <= 0 \|\| !tp->saved_syn \|\|	5227	if (level == SOL_SOCKET)
5325	optlen > tcp_saved_syn_len(tp->saved_syn))	5228	return sol_socket_sockopt(sk, optname, optval, &optlen, false);
5326	goto err_clear;	5229	else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
5327	memcpy(optval, tp->saved_syn->data, optlen);	5230	return sol_ip_sockopt(sk, optname, optval, &optlen, false);
5328	break;	5231	else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
5329	default:	5232	return sol_ipv6_sockopt(sk, optname, optval, &optlen, false);
5330	goto err_clear;	5233	else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
5331	}	5234	return sol_tcp_sockopt(sk, optname, optval, &optlen, false);
5332	} else if (level == SOL_IP) {
5333	struct inet_sock *inet = inet_sk(sk);
5334		5235
5335	if (optlen != sizeof(int) \|\| sk->sk_family != AF_INET)	5236	return -EINVAL;
5336	goto err_clear;	5237	}
5337		5238
5338	/* Only some options are supported */	5239	static int _bpf_setsockopt(struct sock *sk, int level, int optname,
5339	switch (optname) {	5240	char *optval, int optlen)
5340	case IP_TOS:	5241	{
5341	((int )optval) = (int)inet->tos;	5242	if (sk_fullsock(sk))
5342	break;	5243	sock_owned_by_me(sk);
5343	default:	5244	return __bpf_setsockopt(sk, level, optname, optval, optlen);
5344	goto err_clear;	5245	}
5345	}
5346	#if IS_ENABLED(CONFIG_IPV6)
5347	} else if (level == SOL_IPV6) {
5348	struct ipv6_pinfo *np = inet6_sk(sk);
5349		5246
5350	if (optlen != sizeof(int) \|\| sk->sk_family != AF_INET6)	5247	static int __bpf_getsockopt(struct sock *sk, int level, int optname,
5351	goto err_clear;	5248	char *optval, int optlen)
		5249	{
		5250	int err, saved_optlen = optlen;
5352		5251
5353	/* Only some options are supported */	5252	if (!sk_fullsock(sk)) {
5354	switch (optname) {	5253	err = -EINVAL;
5355	case IPV6_TCLASS:	5254	goto done;
5356	((int )optval) = (int)np->tclass;
5357	break;
5358	default:
5359	goto err_clear;
5360	}
5361	#endif
5362	#endif
5363	} else {
5364	goto err_clear;
5365	}	5255	}
5366	return 0;	5256
5367	err_clear:	5257	if (level == SOL_SOCKET)
5368	memset(optval, 0, optlen);	5258	err = sol_socket_sockopt(sk, optname, optval, &optlen, true);
5369	return -EINVAL;	5259	else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
		5260	err = sol_tcp_sockopt(sk, optname, optval, &optlen, true);
		5261	else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
		5262	err = sol_ip_sockopt(sk, optname, optval, &optlen, true);
		5263	else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
		5264	err = sol_ipv6_sockopt(sk, optname, optval, &optlen, true);
		5265	else
		5266	err = -EINVAL;
		5267
		5268	done:
		5269	if (err)
		5270	optlen = 0;
		5271	if (optlen < saved_optlen)
		5272	memset(optval + optlen, 0, saved_optlen - optlen);
		5273	return err;
5370	}	5274	}
5371		5275
5372	static int _bpf_getsockopt(struct sock *sk, int level, int optname,	5276	static int _bpf_getsockopt(struct sock *sk, int level, int optname,
@@ -7667,34 +7571,23 @@ const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto __weak; · 2294 unmodified lines
7667	static const struct bpf_func_proto *	7571	static const struct bpf_func_proto *
7668	sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	7572	sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7669	{	7573	{
		7574	const struct bpf_func_proto *func_proto;
		7575
		7576	func_proto = cgroup_common_func_proto(func_id, prog);
		7577	if (func_proto)
		7578	return func_proto;
		7579
		7580	func_proto = cgroup_current_func_proto(func_id, prog);
		7581	if (func_proto)
		7582	return func_proto;
		7583
7670	switch (func_id) {	7584	switch (func_id) {
7671	/* inet and inet6 sockets are created in a process
7672	* context so there is always a valid uid/gid
7673	*/
7674	case BPF_FUNC_get_current_uid_gid:
7675	return &bpf_get_current_uid_gid_proto;
7676	case BPF_FUNC_get_local_storage:
7677	return &bpf_get_local_storage_proto;
7678	case BPF_FUNC_get_socket_cookie:	7585	case BPF_FUNC_get_socket_cookie:
7679	return &bpf_get_socket_cookie_sock_proto;	7586	return &bpf_get_socket_cookie_sock_proto;
7680	case BPF_FUNC_get_netns_cookie:	7587	case BPF_FUNC_get_netns_cookie:
7681	return &bpf_get_netns_cookie_sock_proto;	7588	return &bpf_get_netns_cookie_sock_proto;
7682	case BPF_FUNC_perf_event_output:	7589	case BPF_FUNC_perf_event_output:
7683	return &bpf_event_output_data_proto;	7590	return &bpf_event_output_data_proto;
7684	case BPF_FUNC_get_current_pid_tgid:
7685	return &bpf_get_current_pid_tgid_proto;
7686	case BPF_FUNC_get_current_comm:
7687	return &bpf_get_current_comm_proto;
7688	#ifdef CONFIG_CGROUPS
7689	case BPF_FUNC_get_current_cgroup_id:
7690	return &bpf_get_current_cgroup_id_proto;
7691	case BPF_FUNC_get_current_ancestor_cgroup_id:
7692	return &bpf_get_current_ancestor_cgroup_id_proto;
7693	#endif
7694	#ifdef CONFIG_CGROUP_NET_CLASSID
7695	case BPF_FUNC_get_cgroup_classid:
7696	return &bpf_get_cgroup_classid_curr_proto;
7697	#endif
7698	case BPF_FUNC_sk_storage_get:	7591	case BPF_FUNC_sk_storage_get:
7699	return &bpf_sk_storage_get_cg_sock_proto;	7592	return &bpf_sk_storage_get_cg_sock_proto;
7700	case BPF_FUNC_ktime_get_coarse_ns:	7593	case BPF_FUNC_ktime_get_coarse_ns:
@@ -7707,12 +7600,17 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 6 unmodified lines
7707	static const struct bpf_func_proto *	7600	static const struct bpf_func_proto *
7708	sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	7601	sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7709	{	7602	{
		7603	const struct bpf_func_proto *func_proto;
		7604
		7605	func_proto = cgroup_common_func_proto(func_id, prog);
		7606	if (func_proto)
		7607	return func_proto;
		7608
		7609	func_proto = cgroup_current_func_proto(func_id, prog);
		7610	if (func_proto)
		7611	return func_proto;
		7612
7710	switch (func_id) {	7613	switch (func_id) {
7711	/* inet and inet6 sockets are created in a process
7712	* context so there is always a valid uid/gid
7713	*/
7714	case BPF_FUNC_get_current_uid_gid:
7715	return &bpf_get_current_uid_gid_proto;
7716	case BPF_FUNC_bind:	7614	case BPF_FUNC_bind:
7717	switch (prog->expected_attach_type) {	7615	switch (prog->expected_attach_type) {
7718	case BPF_CGROUP_INET4_CONNECT:	7616	case BPF_CGROUP_INET4_CONNECT:
@@ -7725,24 +7623,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 6 unmodified lines
7725	return &bpf_get_socket_cookie_sock_addr_proto;	7623	return &bpf_get_socket_cookie_sock_addr_proto;
7726	case BPF_FUNC_get_netns_cookie:	7624	case BPF_FUNC_get_netns_cookie:
7727	return &bpf_get_netns_cookie_sock_addr_proto;	7625	return &bpf_get_netns_cookie_sock_addr_proto;
7728	case BPF_FUNC_get_local_storage:
7729	return &bpf_get_local_storage_proto;
7730	case BPF_FUNC_perf_event_output:	7626	case BPF_FUNC_perf_event_output:
7731	return &bpf_event_output_data_proto;	7627	return &bpf_event_output_data_proto;
7732	case BPF_FUNC_get_current_pid_tgid:
7733	return &bpf_get_current_pid_tgid_proto;
7734	case BPF_FUNC_get_current_comm:
7735	return &bpf_get_current_comm_proto;
7736	#ifdef CONFIG_CGROUPS
7737	case BPF_FUNC_get_current_cgroup_id:
7738	return &bpf_get_current_cgroup_id_proto;
7739	case BPF_FUNC_get_current_ancestor_cgroup_id:
7740	return &bpf_get_current_ancestor_cgroup_id_proto;
7741	#endif
7742	#ifdef CONFIG_CGROUP_NET_CLASSID
7743	case BPF_FUNC_get_cgroup_classid:
7744	return &bpf_get_cgroup_classid_curr_proto;
7745	#endif
7746	#ifdef CONFIG_INET	7628	#ifdef CONFIG_INET
7747	case BPF_FUNC_sk_lookup_tcp:	7629	case BPF_FUNC_sk_lookup_tcp:
7748	return &bpf_sock_addr_sk_lookup_tcp_proto;	7630	return &bpf_sock_addr_sk_lookup_tcp_proto;
@@ -7823,9 +7705,13 @@ const struct bpf_func_proto bpf_sk_storage_delete_proto __weak; · 74 unmodified lines
7823	static const struct bpf_func_proto *	7705	static const struct bpf_func_proto *
7824	cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	7706	cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
7825	{	7707	{
		7708	const struct bpf_func_proto *func_proto;
		7709
		7710	func_proto = cgroup_common_func_proto(func_id, prog);
		7711	if (func_proto)
		7712	return func_proto;
		7713
7826	switch (func_id) {	7714	switch (func_id) {
7827	case BPF_FUNC_get_local_storage:
7828	return &bpf_get_local_storage_proto;
7829	case BPF_FUNC_sk_fullsock:	7715	case BPF_FUNC_sk_fullsock:
7830	return &bpf_sk_fullsock_proto;	7716	return &bpf_sk_fullsock_proto;
7831	case BPF_FUNC_sk_storage_get:	7717	case BPF_FUNC_sk_storage_get:
@@ -8065,6 +7951,12 @@ const struct bpf_func_proto bpf_sock_hash_update_proto __weak; · 233 unmodified lines
8065	static const struct bpf_func_proto *	7951	static const struct bpf_func_proto *
8066	sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	7952	sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
8067	{	7953	{
		7954	const struct bpf_func_proto *func_proto;
		7955
		7956	func_proto = cgroup_common_func_proto(func_id, prog);
		7957	if (func_proto)
		7958	return func_proto;
		7959
8068	switch (func_id) {	7960	switch (func_id) {
8069	case BPF_FUNC_setsockopt:	7961	case BPF_FUNC_setsockopt:
8070	return &bpf_sock_ops_setsockopt_proto;	7962	return &bpf_sock_ops_setsockopt_proto;
@@ -8078,8 +7970,6 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 7 unmodified lines
8078	return &bpf_sock_hash_update_proto;	7970	return &bpf_sock_hash_update_proto;
8079	case BPF_FUNC_get_socket_cookie:	7971	case BPF_FUNC_get_socket_cookie:
8080	return &bpf_get_socket_cookie_sock_ops_proto;	7972	return &bpf_get_socket_cookie_sock_ops_proto;
8081	case BPF_FUNC_get_local_storage:
8082	return &bpf_get_local_storage_proto;
8083	case BPF_FUNC_perf_event_output:	7973	case BPF_FUNC_perf_event_output:
8084	return &bpf_event_output_data_proto;	7974	return &bpf_event_output_data_proto;
8085	case BPF_FUNC_sk_storage_get:	7975	case BPF_FUNC_sk_storage_get:
@@ -10812,14 +10702,13 @@ int sk_detach_filter(struct sock *sk) · 2726 unmodified lines
10812	}	10702	}
10813	EXPORT_SYMBOL_GPL(sk_detach_filter);	10703	EXPORT_SYMBOL_GPL(sk_detach_filter);
10814		10704
10815	int sk_get_filter(struct sock sk, struct sock_filter __user ubuf,	10705	int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len)
10816	unsigned int len)
10817	{	10706	{
10818	struct sock_fprog_kern *fprog;	10707	struct sock_fprog_kern *fprog;
10819	struct sk_filter *filter;	10708	struct sk_filter *filter;
10820	int ret = 0;	10709	int ret = 0;
10821		10710
10822	lock_sock(sk);	10711	sockopt_lock_sock(sk);
10823	filter = rcu_dereference_protected(sk->sk_filter,	10712	filter = rcu_dereference_protected(sk->sk_filter,
10824	lockdep_sock_is_held(sk));	10713	lockdep_sock_is_held(sk));
10825	if (!filter)	10714	if (!filter)
@@ -10844,7 +10733,7 @@ int sk_get_filter(struct sock sk, struct sock_filter __user ubuf, · 18 unmodified lines
10844	goto out;	10733	goto out;
10845		10734
10846	ret = -EFAULT;	10735	ret = -EFAULT;
10847	if (copy_to_user(ubuf, fprog->filter, bpf_classic_proglen(fprog)))	10736	if (copy_to_sockptr(optval, fprog->filter, bpf_classic_proglen(fprog)))
10848	goto out;	10737	goto out;
10849		10738
10850	/* Instead of bytes, the API requests to return the number	10739	/* Instead of bytes, the API requests to return the number
@@ -10852,7 +10741,7 @@ int sk_get_filter(struct sock sk, struct sock_filter __user ubuf, · 1 unmodified line
10852	*/	10741	*/
10853	ret = fprog->len;	10742	ret = fprog->len;
10854	out:	10743	out:
10855	release_sock(sk);	10744	sockopt_release_sock(sk);
10856	return ret;	10745	return ret;
10857	}	10746	}
10858		10747

net/core/flow_dissector.c+10−6

 	}
 }
-bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
+u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
-		      __be16 proto, int nhoff, int hlen, unsigned int flags)
+		     __be16 proto, int nhoff, int hlen, unsigned int flags)
 {
 	struct bpf_flow_keys *flow_keys = ctx->flow_keys;
 	u32 result;
 	flow_keys->thoff = clamp_t(u16, flow_keys->thoff,
 				   flow_keys->nhoff, hlen);
-	return result == BPF_OK;
+	return result;
 }
 static bool is_pppoe_ses_hdr_valid(const struct pppoe_hdr *hdr)
 			};
 			__be16 n_proto = proto;
 			struct bpf_prog *prog;
+			u32 result;
 			if (skb) {
 				ctx.skb = skb;
 			}
 			prog = READ_ONCE(run_array->items[0].prog);
-			ret = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
+			result = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
-					       hlen, flags);
+						  hlen, flags);
+			if (result == BPF_FLOW_DISSECTOR_CONTINUE)
+				goto dissect_continue;
 			__skb_flow_bpf_to_target(&flow_keys, flow_dissector,
 						 target_container);
 			rcu_read_unlock();
-			return ret;
+			return result == BPF_OK;
 		}
+dissect_continue:
 		rcu_read_unlock();
 	}

net/core/sock.c+94−40

 			goto out;
 	}
-	return sock_bindtoindex(sk, index, true);
+	sockopt_lock_sock(sk);
+	ret = sock_bindtoindex_locked(sk, index);
+	sockopt_release_sock(sk);
 out:
 #endif
 	return ret;
 }
-static int sock_getbindtodevice(struct sock *sk, char __user *optval,
+static int sock_getbindtodevice(struct sock *sk, sockptr_t optval,
-				int __user *optlen, int len)
+				sockptr_t optlen, int len)
 {
 	int ret = -ENOPROTOOPT;
 #ifdef CONFIG_NETDEVICES
 	len = strlen(devname) + 1;
 	ret = -EFAULT;
-	if (copy_to_user(optval, devname, len))
+	if (copy_to_sockptr(optval, devname, len))
 		goto out;
 zero:
 	ret = -EFAULT;
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		goto out;
 	ret = 0;
 	return 0;
 }
+void sockopt_lock_sock(struct sock *sk)
+{
+	/* When current->bpf_ctx is set, the setsockopt is called from
+	 * a bpf prog.  bpf has ensured the sk lock has been
+	 * acquired before calling setsockopt().
+	 */
+	if (has_current_bpf_ctx())
+		return;
+	lock_sock(sk);
+}
+EXPORT_SYMBOL(sockopt_lock_sock);
+void sockopt_release_sock(struct sock *sk)
+{
+	if (has_current_bpf_ctx())
+		return;
+	release_sock(sk);
+}
+EXPORT_SYMBOL(sockopt_release_sock);
+bool sockopt_ns_capable(struct user_namespace *ns, int cap)
+{
+	return has_current_bpf_ctx() || ns_capable(ns, cap);
+}
+EXPORT_SYMBOL(sockopt_ns_capable);
+bool sockopt_capable(int cap)
+{
+	return has_current_bpf_ctx() || capable(cap);
+}
+EXPORT_SYMBOL(sockopt_capable);
 /*
  *	This is meant for all protocols to use and covers goings on
  *	at the socket level. Everything here is generic.
  */
-int sock_setsockopt(struct socket *sock, int level, int optname,
+int sk_setsockopt(struct sock *sk, int level, int optname,
-		    sockptr_t optval, unsigned int optlen)
+		  sockptr_t optval, unsigned int optlen)
 {
 	struct so_timestamping timestamping;
+	struct socket *sock = sk->sk_socket;
 	struct sock_txtime sk_txtime;
-	struct sock *sk = sock->sk;
 	int val;
 	int valbool;
 	struct linger ling;
 	valbool = val ? 1 : 0;
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 	switch (optname) {
 	case SO_DEBUG:
-		if (val && !capable(CAP_NET_ADMIN))
+		if (val && !sockopt_capable(CAP_NET_ADMIN))
 			ret = -EACCES;
 		else
 			sock_valbool_flag(sk, SOCK_DBG, valbool);
 		break;
 	case SO_SNDBUFFORCE:
-		if (!capable(CAP_NET_ADMIN)) {
+		if (!sockopt_capable(CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
 		break;
 	case SO_RCVBUFFORCE:
-		if (!capable(CAP_NET_ADMIN)) {
+		if (!sockopt_capable(CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
 	case SO_PRIORITY:
 		if ((val >= 0 && val <= 6) ||
-		    ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) ||
+		    sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) ||
-		    ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
+		    sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
 			sk->sk_priority = val;
 		else
 			ret = -EPERM;
 	case SO_RCVLOWAT:
 		if (val < 0)
 			val = INT_MAX;
-		if (sock->ops->set_rcvlowat)
+		if (sock && sock->ops->set_rcvlowat)
 			ret = sock->ops->set_rcvlowat(sk, val);
 		else
 			WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
 			clear_bit(SOCK_PASSSEC, &sock->flags);
 		break;
 	case SO_MARK:
-		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
+		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
 		__sock_set_mark(sk, val);
 		break;
 	case SO_RCVMARK:
-		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
+		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
 #ifdef CONFIG_NET_RX_BUSY_POLL
 	case SO_BUSY_POLL:
 		/* allow unprivileged users to decrease the value */
-		if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
+		if ((val > sk->sk_ll_usec) && !sockopt_capable(CAP_NET_ADMIN))
 			ret = -EPERM;
 		else {
 			if (val < 0)
 		}
 		break;
 	case SO_PREFER_BUSY_POLL:
-		if (valbool && !capable(CAP_NET_ADMIN))
+		if (valbool && !sockopt_capable(CAP_NET_ADMIN))
 			ret = -EPERM;
 		else
 			WRITE_ONCE(sk->sk_prefer_busy_poll, valbool);
 		break;
 	case SO_BUSY_POLL_BUDGET:
-		if (val > READ_ONCE(sk->sk_busy_poll_budget) && !capable(CAP_NET_ADMIN)) {
+		if (val > READ_ONCE(sk->sk_busy_poll_budget) && !sockopt_capable(CAP_NET_ADMIN)) {
 			ret = -EPERM;
 		} else {
 			if (val < 0 || val > U16_MAX)
 		 * scheduler has enough safe guards.
 		 */
 		if (sk_txtime.clockid != CLOCK_MONOTONIC &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
 		ret = -ENOPROTOOPT;
 		break;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	return ret;
 }
+int sock_setsockopt(struct socket *sock, int level, int optname,
+		    sockptr_t optval, unsigned int optlen)
+{
+	return sk_setsockopt(sock->sk, level, optname,
+			     optval, optlen);
+}
 EXPORT_SYMBOL(sock_setsockopt);
 static const struct cred *sk_get_peer_cred(struct sock *sk)
 	}
 }
-static int groups_to_user(gid_t __user *dst, const struct group_info *src)
+static int groups_to_user(sockptr_t dst, const struct group_info *src)
 {
 	struct user_namespace *user_ns = current_user_ns();
 	int i;
-	for (i = 0; i < src->ngroups; i++)
+	for (i = 0; i < src->ngroups; i++) {
-		if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
+		gid_t gid = from_kgid_munged(user_ns, src->gid[i]);
+		if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid)))
 			return -EFAULT;
+	}
 	return 0;
 }
-int sock_getsockopt(struct socket *sock, int level, int optname,
+int sk_getsockopt(struct sock *sk, int level, int optname,
-		    char __user *optval, int __user *optlen)
+		  sockptr_t optval, sockptr_t optlen)
 {
-	struct sock *sk = sock->sk;
+	struct socket *sock = sk->sk_socket;
 	union {
 		int val;
 	int lv = sizeof(int);
 	int len;
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	if (len < 0)
 		return -EINVAL;
 		cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
 		spin_unlock(&sk->sk_peer_lock);
-		if (copy_to_user(optval, &peercred, len))
+		if (copy_to_sockptr(optval, &peercred, len))
 			return -EFAULT;
 		goto lenout;
 	}
 		if (len < n * sizeof(gid_t)) {
 			len = n * sizeof(gid_t);
 			put_cred(cred);
-			return put_user(len, optlen) ? -EFAULT : -ERANGE;
+			return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE;
 		}
 		len = n * sizeof(gid_t);
-		ret = groups_to_user((gid_t __user *)optval, cred->group_info);
+		ret = groups_to_user(optval, cred->group_info);
 		put_cred(cred);
 		if (ret)
 			return ret;
 			return -ENOTCONN;
 		if (lv < len)
 			return -EINVAL;
-		if (copy_to_user(optval, address, len))
+		if (copy_to_sockptr(optval, address, len))
 			return -EFAULT;
 		goto lenout;
 	}
 		break;
 	case SO_PEERSEC:
-		return security_socket_getpeersec_stream(sock, optval, optlen, len);
+		return security_socket_getpeersec_stream(sock, optval.user, optlen.user, len);
 	case SO_MARK:
 		v.val = sk->sk_mark;
 		return sock_getbindtodevice(sk, optval, optlen, len);
 	case SO_GET_FILTER:
-		len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
+		len = sk_get_filter(sk, optval, len);
 		if (len < 0)
 			return len;
 		sk_get_meminfo(sk, meminfo);
 		len = min_t(unsigned int, len, sizeof(meminfo));
-		if (copy_to_user(optval, &meminfo, len))
+		if (copy_to_sockptr(optval, &meminfo, len))
 			return -EFAULT;
 		goto lenout;
 	if (len > lv)
 		len = lv;
-	if (copy_to_user(optval, &v, len))
+	if (copy_to_sockptr(optval, &v, len))
 		return -EFAULT;
 lenout:
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		return -EFAULT;
 	return 0;
 }
+int sock_getsockopt(struct socket *sock, int level, int optname,
+		    char __user *optval, int __user *optlen)
+{
+	return sk_getsockopt(sock->sk, level, optname,
+			     USER_SOCKPTR(optval),
+			     USER_SOCKPTR(optlen));
+}
 /*
  * Initialize an sk_lock.
  *

net/ipv4/igmp.c+13−9

 		err = ip_mc_leave_group(sk, &imr);
 	return err;
 }
 int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
-	struct ip_msfilter __user *optval, int __user *optlen)
+		 sockptr_t optval, sockptr_t optlen)
 {
-	int err, len, count, copycount;
+	int err, len, count, copycount, msf_size;
 	struct ip_mreqn	imr;
 	__be32 addr = msf->imsf_multiaddr;
 	struct ip_mc_socklist *pmc;
 	copycount = count < msf->imsf_numsrc ? count : msf->imsf_numsrc;
 	len = flex_array_size(psl, sl_addr, copycount);
 	msf->imsf_numsrc = count;
-	if (put_user(IP_MSFILTER_SIZE(copycount), optlen) ||
+	msf_size = IP_MSFILTER_SIZE(copycount);
-	    copy_to_user(optval, msf, IP_MSFILTER_SIZE(0))) {
+	if (copy_to_sockptr(optlen, &msf_size, sizeof(int)) ||
+	    copy_to_sockptr(optval, msf, IP_MSFILTER_SIZE(0))) {
 		return -EFAULT;
 	}
 	if (len &&
-	    copy_to_user(&optval->imsf_slist_flex[0], psl->sl_addr, len))
+	    copy_to_sockptr_offset(optval,
+				   offsetof(struct ip_msfilter, imsf_slist_flex),
+				   psl->sl_addr, len))
 		return -EFAULT;
 	return 0;
 done:
 }
 int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf,
-	struct sockaddr_storage __user *p)
+		 sockptr_t optval, size_t ss_offset)
 {
 	int i, count, copycount;
 	struct sockaddr_in *psin;
 	count = psl ? psl->sl_count : 0;
 	copycount = count < gsf->gf_numsrc ? count : gsf->gf_numsrc;
 	gsf->gf_numsrc = count;
-	for (i = 0; i < copycount; i++, p++) {
+	for (i = 0; i < copycount; i++) {
 		struct sockaddr_storage ss;
 		psin = (struct sockaddr_in *)&ss;
 		memset(&ss, 0, sizeof(ss));
 		psin->sin_family = AF_INET;
 		psin->sin_addr.s_addr = psl->sl_addr[i];
-		if (copy_to_user(p, &ss, sizeof(ss)))
+		if (copy_to_sockptr_offset(optval, ss_offset,
+					   &ss, sizeof(ss)))
 			return -EFAULT;
+		ss_offset += sizeof(ss);
 	}
 	return 0;
 }

net/ipv4/ip_sockglue.c+62−52

 DEFINE_STATIC_KEY_FALSE(ip4_min_ttl);
-static int do_ip_setsockopt(struct sock *sk, int level, int optname,
+int do_ip_setsockopt(struct sock *sk, int level, int optname,
-		sockptr_t optval, unsigned int optlen)
+		     sockptr_t optval, unsigned int optlen)
 {
 	struct inet_sock *inet = inet_sk(sk);
 	struct net *net = sock_net(sk);
 	err = 0;
 	if (needs_rtnl)
 		rtnl_lock();
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 	switch (optname) {
 	case IP_OPTIONS:
 	case IP_IPSEC_POLICY:
 	case IP_XFRM_POLICY:
 		err = -EPERM;
-		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
+		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
 			break;
 		err = xfrm_user_policy(sk, optname, optval, optlen);
 		break;
 	case IP_TRANSPARENT:
-		if (!!val && !ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
+		if (!!val && !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			err = -EPERM;
 			break;
 		}
 		err = -ENOPROTOOPT;
 		break;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return err;
 e_inval:
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return -EINVAL;
 	return false;
 }
-static int ip_get_mcast_msfilter(struct sock *sk, void __user *optval,
+static int ip_get_mcast_msfilter(struct sock *sk, sockptr_t optval,
-		int __user *optlen, int len)
+				 sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct group_filter, gf_slist_flex);
-	struct group_filter __user *p = optval;
 	struct group_filter gsf;
-	int num;
+	int num, gsf_size;
 	int err;
 	if (len < size0)
 		return -EINVAL;
-	if (copy_from_user(&gsf, p, size0))
+	if (copy_from_sockptr(&gsf, optval, size0))
 		return -EFAULT;
 	num = gsf.gf_numsrc;
-	err = ip_mc_gsfget(sk, &gsf, p->gf_slist_flex);
+	err = ip_mc_gsfget(sk, &gsf, optval,
+			   offsetof(struct group_filter, gf_slist_flex));
 	if (err)
 		return err;
 	if (gsf.gf_numsrc < num)
 		num = gsf.gf_numsrc;
-	if (put_user(GROUP_FILTER_SIZE(num), optlen) ||
+	gsf_size = GROUP_FILTER_SIZE(num);
-	    copy_to_user(p, &gsf, size0))
+	if (copy_to_sockptr(optlen, &gsf_size, sizeof(int)) ||
+	    copy_to_sockptr(optval, &gsf, size0))
 		return -EFAULT;
 	return 0;
 }
-static int compat_ip_get_mcast_msfilter(struct sock *sk, void __user *optval,
+static int compat_ip_get_mcast_msfilter(struct sock *sk, sockptr_t optval,
-		int __user *optlen, int len)
+					sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct compat_group_filter, gf_slist_flex);
-	struct compat_group_filter __user *p = optval;
 	struct compat_group_filter gf32;
 	struct group_filter gf;
 	int num;
 	if (len < size0)
 		return -EINVAL;
-	if (copy_from_user(&gf32, p, size0))
+	if (copy_from_sockptr(&gf32, optval, size0))
 		return -EFAULT;
 	gf.gf_interface = gf32.gf_interface;
 	num = gf.gf_numsrc = gf32.gf_numsrc;
 	gf.gf_group = gf32.gf_group;
-	err = ip_mc_gsfget(sk, &gf, p->gf_slist_flex);
+	err = ip_mc_gsfget(sk, &gf, optval,
+			   offsetof(struct compat_group_filter, gf_slist_flex));
 	if (err)
 		return err;
 	if (gf.gf_numsrc < num)
 		num = gf.gf_numsrc;
 	len = GROUP_FILTER_SIZE(num) - (sizeof(gf) - sizeof(gf32));
-	if (put_user(len, optlen) ||
+	if (copy_to_sockptr(optlen, &len, sizeof(int)) ||
-	    put_user(gf.gf_fmode, &p->gf_fmode) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_fmode),
-	    put_user(gf.gf_numsrc, &p->gf_numsrc))
+				   &gf.gf_fmode, sizeof(gf.gf_fmode)) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_numsrc),
+				   &gf.gf_numsrc, sizeof(gf.gf_numsrc)))
 		return -EFAULT;
 	return 0;
 }
-static int do_ip_getsockopt(struct sock *sk, int level, int optname,
+int do_ip_getsockopt(struct sock *sk, int level, int optname,
-			    char __user *optval, int __user *optlen)
+		     sockptr_t optval, sockptr_t optlen)
 {
 	struct inet_sock *inet = inet_sk(sk);
 	bool needs_rtnl = getsockopt_needs_rtnl(optname);
 	if (ip_mroute_opt(optname))
 		return ip_mroute_getsockopt(sk, optname, optval, optlen);
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	if (len < 0)
 		return -EINVAL;
 	if (needs_rtnl)
 		rtnl_lock();
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 	switch (optname) {
 	case IP_OPTIONS:
 			memcpy(optbuf, &inet_opt->opt,
 			       sizeof(struct ip_options) +
 			       inet_opt->opt.optlen);
-		release_sock(sk);
+		sockopt_release_sock(sk);
-		if (opt->optlen == 0)
+		if (opt->optlen == 0) {
-			return put_user(0, optlen);
+			len = 0;
+			return copy_to_sockptr(optlen, &len, sizeof(int));
+		}
 		ip_options_undo(opt);
 		len = min_t(unsigned int, len, opt->optlen);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, opt->__data, len))
+		if (copy_to_sockptr(optval, opt->__data, len))
 			return -EFAULT;
 		return 0;
 	}
 			dst_release(dst);
 		}
 		if (!val) {
-			release_sock(sk);
+			sockopt_release_sock(sk);
 			return -ENOTCONN;
 		}
 		break;
 		struct in_addr addr;
 		len = min_t(unsigned int, len, sizeof(struct in_addr));
 		addr.s_addr = inet->mc_addr;
-		release_sock(sk);
+		sockopt_release_sock(sk);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &addr, len))
+		if (copy_to_sockptr(optval, &addr, len))
 			return -EFAULT;
 		return 0;
 	}
 			err = -EINVAL;
 			goto out;
 		}
-		if (copy_from_user(&msf, optval, IP_MSFILTER_SIZE(0))) {
+		if (copy_from_sockptr(&msf, optval, IP_MSFILTER_SIZE(0))) {
 			err = -EFAULT;
 			goto out;
 		}
-		err = ip_mc_msfget(sk, &msf,
+		err = ip_mc_msfget(sk, &msf, optval, optlen);
-				   (struct ip_msfilter __user *)optval, optlen);
 		goto out;
 	}
 	case MCAST_MSFILTER:
 	{
 		struct msghdr msg;
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		if (sk->sk_type != SOCK_STREAM)
 			return -ENOPROTOOPT;
-		msg.msg_control_is_user = true;
+		if (optval.is_kernel) {
-		msg.msg_control_user = optval;
+			msg.msg_control_is_user = false;
+			msg.msg_control = optval.kernel;
+		} else {
+			msg.msg_control_is_user = true;
+			msg.msg_control_user = optval.user;
+		}
 		msg.msg_controllen = len;
 		msg.msg_flags = in_compat_syscall() ? MSG_CMSG_COMPAT : 0;
 			put_cmsg(&msg, SOL_IP, IP_TOS, sizeof(tos), &tos);
 		}
 		len -= msg.msg_controllen;
-		return put_user(len, optlen);
+		return copy_to_sockptr(optlen, &len, sizeof(int));
 	}
 	case IP_FREEBIND:
 		val = inet->freebind;
 		val = inet->min_ttl;
 		break;
 	default:
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		return -ENOPROTOOPT;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (len < sizeof(int) && len > 0 && val >= 0 && val <= 255) {
 		unsigned char ucval = (unsigned char)val;
 		len = 1;
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &ucval, 1))
+		if (copy_to_sockptr(optval, &ucval, 1))
 			return -EFAULT;
 	} else {
 		len = min_t(unsigned int, sizeof(int), len);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &val, len))
+		if (copy_to_sockptr(optval, &val, len))
 			return -EFAULT;
 	}
 	return 0;
 out:
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return err;
 {
 	int err;
-	err = do_ip_getsockopt(sk, level, optname, optval, optlen);
+	err = do_ip_getsockopt(sk, level, optname,
+			       USER_SOCKPTR(optval), USER_SOCKPTR(optlen));
 #if IS_ENABLED(CONFIG_BPFILTER_UMH)
 	if (optname >= BPFILTER_IPT_SO_GET_INFO &&

net/ipv4/ipmr.c+5−4

 }
 /* Getsock opt support for the multicast routing system. */
-int ip_mroute_getsockopt(struct sock *sk, int optname, char __user *optval, int __user *optlen)
+int ip_mroute_getsockopt(struct sock *sk, int optname, sockptr_t optval,
+			 sockptr_t optlen)
 {
 	int olr;
 	int val;
 		return -ENOPROTOOPT;
 	}
-	if (get_user(olr, optlen))
+	if (copy_from_sockptr(&olr, optlen, sizeof(int)))
 		return -EFAULT;
 	olr = min_t(unsigned int, olr, sizeof(int));
 	if (olr < 0)
 		return -EINVAL;
-	if (put_user(olr, optlen))
+	if (copy_to_sockptr(optlen, &olr, sizeof(int)))
 		return -EFAULT;
-	if (copy_to_user(optval, &val, olr))
+	if (copy_to_sockptr(optval, &val, olr))
 		return -EFAULT;
 	return 0;
 }

net/ipv4/tcp.c+59−57

 static inline bool tcp_can_repair_sock(const struct sock *sk)
 {
-	return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
+	return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
 		(sk->sk_state != TCP_LISTEN);
 }
 /*
  *	Socket option code for TCP.
  */
-static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
+int do_tcp_setsockopt(struct sock *sk, int level, int optname,
-		sockptr_t optval, unsigned int optlen)
+		      sockptr_t optval, unsigned int optlen)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct inet_connection_sock *icsk = inet_csk(sk);
 			return -EFAULT;
 		name[val] = 0;
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
-		err = tcp_set_congestion_control(sk, name, true,
+		err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(),
-						 ns_capable(sock_net(sk)->user_ns,
+						 sockopt_ns_capable(sock_net(sk)->user_ns,
-							    CAP_NET_ADMIN));
+								    CAP_NET_ADMIN));
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		return err;
 	}
 	case TCP_ULP: {
 			return -EFAULT;
 		name[val] = 0;
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		err = tcp_set_ulp(sk, name);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		return err;
 	}
 	case TCP_FASTOPEN_KEY: {
 	if (copy_from_sockptr(&val, optval, sizeof(val)))
 		return -EFAULT;
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 	switch (optname) {
 	case TCP_MAXSEG:
 		break;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	return err;
 }
 	return stats;
 }
-static int do_tcp_getsockopt(struct sock *sk, int level,
+int do_tcp_getsockopt(struct sock *sk, int level,
-		int optname, char __user *optval, int __user *optlen)
+		      int optname, sockptr_t optval, sockptr_t optlen)
 {
 	struct inet_connection_sock *icsk = inet_csk(sk);
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct net *net = sock_net(sk);
 	int val, len;
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	len = min_t(unsigned int, len, sizeof(int));
 	case TCP_INFO: {
 		struct tcp_info info;
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		tcp_get_info(sk, &info);
 		len = min_t(unsigned int, len, sizeof(info));
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &info, len))
+		if (copy_to_sockptr(optval, &info, len))
 			return -EFAULT;
 		return 0;
 	}
 		size_t sz = 0;
 		int attr;
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		ca_ops = icsk->icsk_ca_ops;
 			sz = ca_ops->get_info(sk, ~0U, &attr, &info);
 		len = min_t(unsigned int, len, sz);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &info, len))
+		if (copy_to_sockptr(optval, &info, len))
 			return -EFAULT;
 		return 0;
 	}
 		break;
 	case TCP_CONGESTION:
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
+		if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len))
 			return -EFAULT;
 		return 0;
 	case TCP_ULP:
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
 		if (!icsk->icsk_ulp_ops) {
-			if (put_user(0, optlen))
+			len = 0;
+			if (copy_to_sockptr(optlen, &len, sizeof(int)))
 				return -EFAULT;
 			return 0;
 		}
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len))
+		if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len))
 			return -EFAULT;
 		return 0;
 		u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
 		unsigned int key_len;
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		key_len = tcp_fastopen_get_cipher(net, icsk, key) *
 				TCP_FASTOPEN_KEY_LENGTH;
 		len = min_t(unsigned int, len, key_len);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, key, len))
+		if (copy_to_sockptr(optval, key, len))
 			return -EFAULT;
 		return 0;
 	}
 	case TCP_REPAIR_WINDOW: {
 		struct tcp_repair_window opt;
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		if (len != sizeof(opt))
 		opt.rcv_wnd	= tp->rcv_wnd;
 		opt.rcv_wup	= tp->rcv_wup;
-		if (copy_to_user(optval, &opt, len))
+		if (copy_to_sockptr(optval, &opt, len))
 			return -EFAULT;
 		return 0;
 	}
 		val = tp->save_syn;
 		break;
 	case TCP_SAVED_SYN: {
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		if (tp->saved_syn) {
 			if (len < tcp_saved_syn_len(tp->saved_syn)) {
-				if (put_user(tcp_saved_syn_len(tp->saved_syn),
+				len = tcp_saved_syn_len(tp->saved_syn);
-					     optlen)) {
+				if (copy_to_sockptr(optlen, &len, sizeof(int))) {
-					release_sock(sk);
+					sockopt_release_sock(sk);
 					return -EFAULT;
 				}
-				release_sock(sk);
+				sockopt_release_sock(sk);
 				return -EINVAL;
 			}
 			len = tcp_saved_syn_len(tp->saved_syn);
-			if (put_user(len, optlen)) {
+			if (copy_to_sockptr(optlen, &len, sizeof(int))) {
-				release_sock(sk);
+				sockopt_release_sock(sk);
 				return -EFAULT;
 			}
-			if (copy_to_user(optval, tp->saved_syn->data, len)) {
+			if (copy_to_sockptr(optval, tp->saved_syn->data, len)) {
-				release_sock(sk);
+				sockopt_release_sock(sk);
 				return -EFAULT;
 			}
 			tcp_saved_syn_free(tp);
-			release_sock(sk);
+			sockopt_release_sock(sk);
 		} else {
-			release_sock(sk);
+			sockopt_release_sock(sk);
 			len = 0;
-			if (put_user(len, optlen))
+			if (copy_to_sockptr(optlen, &len, sizeof(int)))
 				return -EFAULT;
 		}
 		return 0;
 		struct tcp_zerocopy_receive zc = {};
 		int err;
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		if (len < 0 ||
 		    len < offsetofend(struct tcp_zerocopy_receive, length))
 			return -EINVAL;
 		if (unlikely(len > sizeof(zc))) {
-			err = check_zeroed_user(optval + sizeof(zc),
+			err = check_zeroed_sockptr(optval, sizeof(zc),
-						len - sizeof(zc));
+						   len - sizeof(zc));
 			if (err < 1)
 				return err == 0 ? -EINVAL : err;
 			len = sizeof(zc);
-			if (put_user(len, optlen))
+			if (copy_to_sockptr(optlen, &len, sizeof(int)))
 				return -EFAULT;
 		}
-		if (copy_from_user(&zc, optval, len))
+		if (copy_from_sockptr(&zc, optval, len))
 			return -EFAULT;
 		if (zc.reserved)
 			return -EINVAL;
 		if (zc.msg_flags &  ~(TCP_VALID_ZC_MSG_FLAGS))
 			return -EINVAL;
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		err = tcp_zerocopy_receive(sk, &zc, &tss);
 		err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
 							  &zc, &len, err);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
 			goto zerocopy_rcv_cmsg;
 		switch (len) {
 zerocopy_rcv_inq:
 		zc.inq = tcp_inq_hint(sk);
 zerocopy_rcv_out:
-		if (!err && copy_to_user(optval, &zc, len))
+		if (!err && copy_to_sockptr(optval, &zc, len))
 			err = -EFAULT;
 		return err;
 	}
 		return -ENOPROTOOPT;
 	}
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		return -EFAULT;
-	if (copy_to_user(optval, &val, len))
+	if (copy_to_sockptr(optval, &val, len))
 		return -EFAULT;
 	return 0;
 }
 	if (level != SOL_TCP)
 		return icsk->icsk_af_ops->getsockopt(sk, level, optname,
 						     optval, optlen);
-	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
+	return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval),
+				 USER_SOCKPTR(optlen));
 }
 EXPORT_SYMBOL(tcp_getsockopt);

net/ipv6/af_inet6.c+2−0

@@ -1057,6 +1057,8 @@ static const struct ipv6_stub ipv6_stub_impl = { · 1056 unmodified lines
1057	static const struct ipv6_bpf_stub ipv6_bpf_stub_impl = {	1057	static const struct ipv6_bpf_stub ipv6_bpf_stub_impl = {
1058	.inet6_bind = __inet6_bind,	1058	.inet6_bind = __inet6_bind,
1059	.udp6_lib_lookup = __udp6_lib_lookup,	1059	.udp6_lib_lookup = __udp6_lib_lookup,
		1060	.ipv6_setsockopt = do_ipv6_setsockopt,
		1061	.ipv6_getsockopt = do_ipv6_getsockopt,
1060	};	1062	};
1061		1063
1062	static int __init inet6_init(void)	1064	static int __init inet6_init(void)

net/ipv6/ip6mr.c+5−5

@@ -1827,8 +1827,8 @@ int ip6_mroute_setsockopt(struct sock *sk, int optname, sockptr_t optval, · 1826 unmodified lines
1827	* Getsock opt support for the multicast routing system.	1827	* Getsock opt support for the multicast routing system.
1828	*/	1828	*/
1829		1829
1830	int ip6_mroute_getsockopt(struct sock sk, int optname, char __user optval,	1830	int ip6_mroute_getsockopt(struct sock *sk, int optname, sockptr_t optval,
1831	int __user *optlen)	1831	sockptr_t optlen)
1832	{	1832	{
1833	int olr;	1833	int olr;
1834	int val;	1834	int val;
@@ -1859,16 +1859,16 @@ int ip6_mroute_getsockopt(struct sock sk, int optname, char __user optval, · 24 unmodified lines
1859	return -ENOPROTOOPT;	1859	return -ENOPROTOOPT;
1860	}	1860	}
1861		1861
1862	if (get_user(olr, optlen))	1862	if (copy_from_sockptr(&olr, optlen, sizeof(int)))
1863	return -EFAULT;	1863	return -EFAULT;
1864		1864
1865	olr = min_t(int, olr, sizeof(int));	1865	olr = min_t(int, olr, sizeof(int));
1866	if (olr < 0)	1866	if (olr < 0)
1867	return -EINVAL;	1867	return -EINVAL;
1868		1868
1869	if (put_user(olr, optlen))	1869	if (copy_to_sockptr(optlen, &olr, sizeof(int)))
1870	return -EFAULT;	1870	return -EFAULT;
1871	if (copy_to_user(optval, &val, olr))	1871	if (copy_to_sockptr(optval, &val, olr))
1872	return -EFAULT;	1872	return -EFAULT;
1873	return 0;	1873	return 0;
1874	}	1874	}

net/ipv6/ipv6_sockglue.c+59−54

 	int err;
 	/* hop-by-hop / destination options are privileged option */
-	if (optname != IPV6_RTHDR && !ns_capable(net->user_ns, CAP_NET_RAW))
+	if (optname != IPV6_RTHDR && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW))
 		return -EPERM;
 	/* remove any sticky options header with a zero option
 	return err;
 }
-static int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
+int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
-		   sockptr_t optval, unsigned int optlen)
+		       sockptr_t optval, unsigned int optlen)
 {
 	struct ipv6_pinfo *np = inet6_sk(sk);
 	struct net *net = sock_net(sk);
 	if (needs_rtnl)
 		rtnl_lock();
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 	switch (optname) {
 		break;
 	case IPV6_TRANSPARENT:
-		if (valbool && !ns_capable(net->user_ns, CAP_NET_RAW) &&
+		if (valbool && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(net->user_ns, CAP_NET_ADMIN)) {
+		    !sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN)) {
 			retv = -EPERM;
 			break;
 		}
 	case IPV6_IPSEC_POLICY:
 	case IPV6_XFRM_POLICY:
 		retv = -EPERM;
-		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
+		if (!sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN))
 			break;
 		retv = xfrm_user_policy(sk, optname, optval, optlen);
 		break;
 		break;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return retv;
 e_inval:
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return -EINVAL;
 EXPORT_SYMBOL(ipv6_setsockopt);
 static int ipv6_getsockopt_sticky(struct sock *sk, struct ipv6_txoptions *opt,
-				  int optname, char __user *optval, int len)
+				  int optname, sockptr_t optval, int len)
 {
 	struct ipv6_opt_hdr *hdr;
 		return 0;
 	len = min_t(unsigned int, len, ipv6_optlen(hdr));
-	if (copy_to_user(optval, hdr, len))
+	if (copy_to_sockptr(optval, hdr, len))
 		return -EFAULT;
 	return len;
 }
-static int ipv6_get_msfilter(struct sock *sk, void __user *optval,
+static int ipv6_get_msfilter(struct sock *sk, sockptr_t optval,
-		int __user *optlen, int len)
+			     sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct group_filter, gf_slist_flex);
-	struct group_filter __user *p = optval;
 	struct group_filter gsf;
 	int num;
 	int err;
 	if (len < size0)
 		return -EINVAL;
-	if (copy_from_user(&gsf, p, size0))
+	if (copy_from_sockptr(&gsf, optval, size0))
 		return -EFAULT;
 	if (gsf.gf_group.ss_family != AF_INET6)
 		return -EADDRNOTAVAIL;
 	num = gsf.gf_numsrc;
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
-	err = ip6_mc_msfget(sk, &gsf, p->gf_slist_flex);
+	err = ip6_mc_msfget(sk, &gsf, optval, size0);
 	if (!err) {
 		if (num > gsf.gf_numsrc)
 			num = gsf.gf_numsrc;
-		if (put_user(GROUP_FILTER_SIZE(num), optlen) ||
+		len = GROUP_FILTER_SIZE(num);
-		    copy_to_user(p, &gsf, size0))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)) ||
+		    copy_to_sockptr(optval, &gsf, size0))
 			err = -EFAULT;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	return err;
 }
-static int compat_ipv6_get_msfilter(struct sock *sk, void __user *optval,
+static int compat_ipv6_get_msfilter(struct sock *sk, sockptr_t optval,
-		int __user *optlen)
+				    sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct compat_group_filter, gf_slist_flex);
-	struct compat_group_filter __user *p = optval;
 	struct compat_group_filter gf32;
 	struct group_filter gf;
-	int len, err;
+	int err;
 	int num;
-	if (get_user(len, optlen))
-		return -EFAULT;
 	if (len < size0)
 		return -EINVAL;
-	if (copy_from_user(&gf32, p, size0))
+	if (copy_from_sockptr(&gf32, optval, size0))
 		return -EFAULT;
 	gf.gf_interface = gf32.gf_interface;
 	gf.gf_fmode = gf32.gf_fmode;
 	if (gf.gf_group.ss_family != AF_INET6)
 		return -EADDRNOTAVAIL;
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
-	err = ip6_mc_msfget(sk, &gf, p->gf_slist_flex);
+	err = ip6_mc_msfget(sk, &gf, optval, size0);
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (err)
 		return err;
 	if (num > gf.gf_numsrc)
 		num = gf.gf_numsrc;
 	len = GROUP_FILTER_SIZE(num) - (sizeof(gf)-sizeof(gf32));
-	if (put_user(len, optlen) ||
+	if (copy_to_sockptr(optlen, &len, sizeof(int)) ||
-	    put_user(gf.gf_fmode, &p->gf_fmode) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_fmode),
-	    put_user(gf.gf_numsrc, &p->gf_numsrc))
+				   &gf.gf_fmode, sizeof(gf32.gf_fmode)) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_numsrc),
+				   &gf.gf_numsrc, sizeof(gf32.gf_numsrc)))
 		return -EFAULT;
 	return 0;
 }
-static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
+int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
-		    char __user *optval, int __user *optlen, unsigned int flags)
+		       sockptr_t optval, sockptr_t optlen)
 {
 	struct ipv6_pinfo *np = inet6_sk(sk);
 	int len;
 	if (ip6_mroute_opt(optname))
 		return ip6_mroute_getsockopt(sk, optname, optval, optlen);
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	switch (optname) {
 	case IPV6_ADDRFORM:
 		break;
 	case MCAST_MSFILTER:
 		if (in_compat_syscall())
-			return compat_ipv6_get_msfilter(sk, optval, optlen);
+			return compat_ipv6_get_msfilter(sk, optval, optlen, len);
 		return ipv6_get_msfilter(sk, optval, optlen, len);
 	case IPV6_2292PKTOPTIONS:
 	{
 		if (sk->sk_type != SOCK_STREAM)
 			return -ENOPROTOOPT;
-		msg.msg_control_user = optval;
+		if (optval.is_kernel) {
+			msg.msg_control_is_user = false;
+			msg.msg_control = optval.kernel;
+		} else {
+			msg.msg_control_is_user = true;
+			msg.msg_control_user = optval.user;
+		}
 		msg.msg_controllen = len;
-		msg.msg_flags = flags;
+		msg.msg_flags = 0;
-		msg.msg_control_is_user = true;
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		skb = np->pktoptions;
 		if (skb)
 			ip6_datagram_recv_ctl(sk, &msg, skb);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		if (!skb) {
 			if (np->rxopt.bits.rxinfo) {
 				struct in6_pktinfo src_info;
 			}
 		}
 		len -= msg.msg_controllen;
-		return put_user(len, optlen);
+		return copy_to_sockptr(optlen, &len, sizeof(int));
 	}
 	case IPV6_MTU:
 	{
 	{
 		struct ipv6_txoptions *opt;
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		opt = rcu_dereference_protected(np->opt,
 						lockdep_sock_is_held(sk));
 		len = ipv6_getsockopt_sticky(sk, opt, optname, optval, len);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		/* check if ipv6_getsockopt_sticky() returns err code */
 		if (len < 0)
 			return len;
-		return put_user(len, optlen);
+		return copy_to_sockptr(optlen, &len, sizeof(int));
 	}
 	case IPV6_RECVHOPOPTS:
 		if (!mtuinfo.ip6m_mtu)
 			return -ENOTCONN;
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &mtuinfo, len))
+		if (copy_to_sockptr(optval, &mtuinfo, len))
 			return -EFAULT;
 		return 0;
 		if (len < sizeof(freq))
 			return -EINVAL;
-		if (copy_from_user(&freq, optval, sizeof(freq)))
+		if (copy_from_sockptr(&freq, optval, sizeof(freq)))
 			return -EFAULT;
 		if (freq.flr_action != IPV6_FL_A_GET)
 		if (val < 0)
 			return val;
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &freq, len))
+		if (copy_to_sockptr(optval, &freq, len))
 			return -EFAULT;
 		return 0;
 		return -ENOPROTOOPT;
 	}
 	len = min_t(unsigned int, sizeof(int), len);
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		return -EFAULT;
-	if (copy_to_user(optval, &val, len))
+	if (copy_to_sockptr(optval, &val, len))
 		return -EFAULT;
 	return 0;
 }
 	if (level != SOL_IPV6)
 		return -ENOPROTOOPT;
-	err = do_ipv6_getsockopt(sk, level, optname, optval, optlen, 0);
+	err = do_ipv6_getsockopt(sk, level, optname,
+				 USER_SOCKPTR(optval), USER_SOCKPTR(optlen));
 #ifdef CONFIG_NETFILTER
 	/* we need to exclude all possible ENOPROTOOPTs except default case */
 	if (err == -ENOPROTOOPT && optname != IPV6_2292PKTOPTIONS) {

net/ipv6/mcast.c+4−4

@@ -580,7 +580,7 @@ done: · 579 unmodified lines
580	}	580	}
581		581
582	int ip6_mc_msfget(struct sock sk, struct group_filter gsf,	582	int ip6_mc_msfget(struct sock sk, struct group_filter gsf,
583	struct sockaddr_storage __user *p)	583	sockptr_t optval, size_t ss_offset)
584	{	584	{
585	struct ipv6_pinfo *inet6 = inet6_sk(sk);	585	struct ipv6_pinfo *inet6 = inet6_sk(sk);
586	const struct in6_addr *group;	586	const struct in6_addr *group;
@@ -612,8 +612,7 @@ int ip6_mc_msfget(struct sock sk, struct group_filter gsf, · 25 unmodified lines
612		612
613	copycount = count < gsf->gf_numsrc ? count : gsf->gf_numsrc;	613	copycount = count < gsf->gf_numsrc ? count : gsf->gf_numsrc;
614	gsf->gf_numsrc = count;	614	gsf->gf_numsrc = count;
615		615	for (i = 0; i < copycount; i++) {
616	for (i = 0; i < copycount; i++, p++) {
617	struct sockaddr_in6 *psin6;	616	struct sockaddr_in6 *psin6;
618	struct sockaddr_storage ss;	617	struct sockaddr_storage ss;
619		618
@@ -621,8 +620,9 @@ int ip6_mc_msfget(struct sock sk, struct group_filter gsf, · 1 unmodified line
621	memset(&ss, 0, sizeof(ss));	620	memset(&ss, 0, sizeof(ss));
622	psin6->sin6_family = AF_INET6;	621	psin6->sin6_family = AF_INET6;
623	psin6->sin6_addr = psl->sl_addr[i];	622	psin6->sin6_addr = psl->sl_addr[i];
624	if (copy_to_user(p, &ss, sizeof(ss)))	623	if (copy_to_sockptr_offset(optval, ss_offset, &ss, sizeof(ss)))
625	return -EFAULT;	624	return -EFAULT;
		625	ss_offset += sizeof(ss);
626	}	626	}
627	return 0;	627	return 0;
628	}	628	}

samples/bpf/map_perf_test_kern.c+28−16

 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
-	bpf_map_update_elem(&hash_map, &key, &init_val, BPF_ANY);
+	for (i = 0; i < 10; i++) {
-	value = bpf_map_lookup_elem(&hash_map, &key);
+		bpf_map_update_elem(&hash_map, &key, &init_val, BPF_ANY);
-	if (value)
+		value = bpf_map_lookup_elem(&hash_map, &key);
-		bpf_map_delete_elem(&hash_map, &key);
+		if (value)
+			bpf_map_delete_elem(&hash_map, &key);
+	}
 	return 0;
 }
 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
-	bpf_map_update_elem(&percpu_hash_map, &key, &init_val, BPF_ANY);
+	for (i = 0; i < 10; i++) {
-	value = bpf_map_lookup_elem(&percpu_hash_map, &key);
+		bpf_map_update_elem(&percpu_hash_map, &key, &init_val, BPF_ANY);
-	if (value)
+		value = bpf_map_lookup_elem(&percpu_hash_map, &key);
-		bpf_map_delete_elem(&percpu_hash_map, &key);
+		if (value)
+			bpf_map_delete_elem(&percpu_hash_map, &key);
+	}
 	return 0;
 }
 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
-	bpf_map_update_elem(&hash_map_alloc, &key, &init_val, BPF_ANY);
+	for (i = 0; i < 10; i++) {
-	value = bpf_map_lookup_elem(&hash_map_alloc, &key);
+		bpf_map_update_elem(&hash_map_alloc, &key, &init_val, BPF_ANY);
-	if (value)
+		value = bpf_map_lookup_elem(&hash_map_alloc, &key);
-		bpf_map_delete_elem(&hash_map_alloc, &key);
+		if (value)
+			bpf_map_delete_elem(&hash_map_alloc, &key);
+	}
 	return 0;
 }
 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
-	bpf_map_update_elem(&percpu_hash_map_alloc, &key, &init_val, BPF_ANY);
+	for (i = 0; i < 10; i++) {
-	value = bpf_map_lookup_elem(&percpu_hash_map_alloc, &key);
+		bpf_map_update_elem(&percpu_hash_map_alloc, &key, &init_val, BPF_ANY);
-	if (value)
+		value = bpf_map_lookup_elem(&percpu_hash_map_alloc, &key);
-		bpf_map_delete_elem(&percpu_hash_map_alloc, &key);
+		if (value)
+			bpf_map_delete_elem(&percpu_hash_map_alloc, &key);
+	}
 	return 0;
 }

samples/bpf/map_perf_test_user.c+1−1

@@ -72,7 +72,7 @@ static int test_flags = ~0; · 71 unmodified lines
72	static uint32_t num_map_entries;	72	static uint32_t num_map_entries;
73	static uint32_t inner_lru_hash_size;	73	static uint32_t inner_lru_hash_size;
74	static int lru_hash_lookup_test_entries = 32;	74	static int lru_hash_lookup_test_entries = 32;
75	static uint32_t max_cnt = 1000000;	75	static uint32_t max_cnt = 10000;
76		76
77	static int check_test_flags(enum test_type t)	77	static int check_test_flags(enum test_type t)
78	{	78	{

scripts/bpf_doc.py+71−7

 import argparse
 import re
 import sys, os
+import subprocess
+helpersDocStart = 'Start of BPF helper function descriptions:'
 class NoHelperFound(BaseException):
     pass
     @desc: textual description of the helper function
     @ret: description of the return value of the helper function
     """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.enum_val = None
     def proto_break_down(self):
         """
         Break down helper function protocol into smaller chunks: return type,
         self.commands = []
         self.desc_unique_helpers = set()
         self.define_unique_helpers = []
+        self.helper_enum_vals = {}
         self.desc_syscalls = []
         self.enum_syscalls = []
         self.enum_syscalls = re.findall('(BPF\w+)+', bpf_cmd_str)
     def parse_desc_helpers(self):
-        self.seek_to('* Start of BPF helper function descriptions:',
+        self.seek_to(helpersDocStart,
                      'Could not find start of eBPF helper descriptions list')
         while True:
             try:
                 break
     def parse_define_helpers(self):
-        # Parse the number of FN(...) in #define __BPF_FUNC_MAPPER to compare
+        # Parse FN(...) in #define __BPF_FUNC_MAPPER to compare later with the
-        # later with the number of unique function names present in description.
+        # number of unique function names present in description and use the
+        # correct enumeration value.
         # Note: seek_to(..) discards the first line below the target search text,
         # resulting in FN(unspec) being skipped and not added to self.define_unique_helpers.
         self.seek_to('#define __BPF_FUNC_MAPPER(FN)',
                      'Could not find start of eBPF helper definition list')
-        # Searches for either one or more FN(\w+) defines or a backslash for newline
+        # Searches for one FN(\w+) define or a backslash for newline
-        p = re.compile('\s*(FN\(\w+\))+|\\\\')
+        p = re.compile('\s*FN\((\w+)\)|\\\\')
         fn_defines_str = ''
+        i = 1  # 'unspec' is skipped as mentioned above
         while True:
             capture = p.match(self.line)
             if capture:
                 fn_defines_str += self.line
+                self.helper_enum_vals[capture.expand(r'bpf_\1')] = i
+                i += 1
             else:
                 break
             self.line = self.reader.readline()
         # Find the number of occurences of FN(\w+)
         self.define_unique_helpers = re.findall('FN\(\w+\)', fn_defines_str)
+    def assign_helper_values(self):
+        seen_helpers = set()
+        for helper in self.helpers:
+            proto = helper.proto_break_down()
+            name = proto['name']
+            try:
+                enum_val = self.helper_enum_vals[name]
+            except KeyError:
+                raise Exception("Helper %s is missing from enum bpf_func_id" % name)
+            # Enforce current practice of having the descriptions ordered
+            # by enum value.
+            seen_helpers.add(name)
+            desc_val = len(seen_helpers)
+            if desc_val != enum_val:
+                raise Exception("Helper %s comment order (#%d) must be aligned with its position (#%d) in enum bpf_func_id" % (name, desc_val, enum_val))
+            helper.enum_val = enum_val
     def run(self):
         self.parse_desc_syscall()
         self.parse_enum_syscall()
         self.parse_desc_helpers()
         self.parse_define_helpers()
+        self.assign_helper_values()
         self.reader.close()
 ###############################################################################
         print('')
+    def get_kernel_version(self):
+        try:
+            version = subprocess.run(['git', 'describe'], cwd=linuxRoot,
+                                     capture_output=True, check=True)
+            version = version.stdout.decode().rstrip()
+        except:
+            try:
+                version = subprocess.run(['make', 'kernelversion'], cwd=linuxRoot,
+                                         capture_output=True, check=True)
+                version = version.stdout.decode().rstrip()
+            except:
+                return 'Linux'
+        return 'Linux {version}'.format(version=version)
+    def get_last_doc_update(self, delimiter):
+        try:
+            cmd = ['git', 'log', '-1', '--pretty=format:%cs', '--no-patch',
+                   '-L',
+                   '/{}/,/\*\//:include/uapi/linux/bpf.h'.format(delimiter)]
+            date = subprocess.run(cmd, cwd=linuxRoot,
+                                  capture_output=True, check=True)
+            return date.stdout.decode().rstrip()
+        except:
+            return ''
 class PrinterHelpersRST(PrinterRST):
     """
     A printer for dumping collected information about helpers as a ReStructured
 -------------------------------------------------------------------------------
 :Manual section: 7
+:Version: {version}
+{date_field}{date}
 DESCRIPTION
 ===========
 HELPERS
 =======
 '''
+        kernelVersion = self.get_kernel_version()
+        lastUpdate = self.get_last_doc_update(helpersDocStart)
         PrinterRST.print_license(self)
-        print(header)
+        print(header.format(version=kernelVersion,
+                            date_field = ':Date: ' if lastUpdate else '',
+                            date=lastUpdate))
     def print_footer(self):
         footer = '''
             comma = ', '
             print(one_arg, end='')
-        print(') = (void *) %d;' % len(self.seen_helpers))
+        print(') = (void *) %d;' % helper.enum_val)
         print('')
 ###############################################################################

tools/bpf/bpftool/btf_dumper.c+1−1

@@ -452,7 +452,7 @@ static int btf_dumper_int(const struct btf_type *t, __u8 bit_offset, · 451 unmodified lines
452	(char )data);	452	(char )data);
453	break;	453	break;
454	case BTF_INT_BOOL:	454	case BTF_INT_BOOL:
455	jsonw_bool(jw, (int )data);	455	jsonw_bool(jw, (bool )data);
456	break;	456	break;
457	default:	457	default:
458	/* shouldn't happen */	458	/* shouldn't happen */

tools/bpf/bpftool/link.c+35−0

@@ -83,6 +83,29 @@ static bool is_iter_map_target(const char *target_name) · 82 unmodified lines
83	strcmp(target_name, "bpf_sk_storage_map") == 0;	83	strcmp(target_name, "bpf_sk_storage_map") == 0;
84	}	84	}
85		85
		86	static bool is_iter_cgroup_target(const char *target_name)
		87	{
		88	return strcmp(target_name, "cgroup") == 0;
		89	}
		90
		91	static const char *cgroup_order_string(__u32 order)
		92	{
		93	switch (order) {
		94	case BPF_CGROUP_ITER_ORDER_UNSPEC:
		95	return "order_unspec";
		96	case BPF_CGROUP_ITER_SELF_ONLY:
		97	return "self_only";
		98	case BPF_CGROUP_ITER_DESCENDANTS_PRE:
		99	return "descendants_pre";
		100	case BPF_CGROUP_ITER_DESCENDANTS_POST:
		101	return "descendants_post";
		102	case BPF_CGROUP_ITER_ANCESTORS_UP:
		103	return "ancestors_up";
		104	default: /* won't happen */
		105	return "unknown";
		106	}
		107	}
		108
86	static void show_iter_json(struct bpf_link_info info, json_writer_t wtr)	109	static void show_iter_json(struct bpf_link_info info, json_writer_t wtr)
87	{	110	{
88	const char *target_name = u64_to_ptr(info->iter.target_name);	111	const char *target_name = u64_to_ptr(info->iter.target_name);
@@ -91,6 +114,12 @@ static void show_iter_json(struct bpf_link_info info, json_writer_t wtr) · 2 unmodified lines
91		114
92	if (is_iter_map_target(target_name))	115	if (is_iter_map_target(target_name))
93	jsonw_uint_field(wtr, "map_id", info->iter.map.map_id);	116	jsonw_uint_field(wtr, "map_id", info->iter.map.map_id);
		117
		118	if (is_iter_cgroup_target(target_name)) {
		119	jsonw_lluint_field(wtr, "cgroup_id", info->iter.cgroup.cgroup_id);
		120	jsonw_string_field(wtr, "order",
		121	cgroup_order_string(info->iter.cgroup.order));
		122	}
94	}	123	}
95		124
96	static int get_prog_info(int prog_id, struct bpf_prog_info *info)	125	static int get_prog_info(int prog_id, struct bpf_prog_info *info)
@@ -208,6 +237,12 @@ static void show_iter_plain(struct bpf_link_info *info) · 111 unmodified lines
208		237
209	if (is_iter_map_target(target_name))	238	if (is_iter_map_target(target_name))
210	printf("map_id %u ", info->iter.map.map_id);	239	printf("map_id %u ", info->iter.map.map_id);
		240
		241	if (is_iter_cgroup_target(target_name)) {
		242	printf("cgroup_id %llu ", info->iter.cgroup.cgroup_id);
		243	printf("order %s ",
		244	cgroup_order_string(info->iter.cgroup.order));
		245	}
211	}	246	}
212		247
213	static int show_link_close_plain(int fd, struct bpf_link_info *info)	248	static int show_link_close_plain(int fd, struct bpf_link_info *info)

tools/include/uapi/linux/bpf.h+69−14

 	__u32	attach_type;		/* program attach type (enum bpf_attach_type) */
 };
+enum bpf_cgroup_iter_order {
+	BPF_CGROUP_ITER_ORDER_UNSPEC = 0,
+	BPF_CGROUP_ITER_SELF_ONLY,		/* process only a single object. */
+	BPF_CGROUP_ITER_DESCENDANTS_PRE,	/* walk descendants in pre-order. */
+	BPF_CGROUP_ITER_DESCENDANTS_POST,	/* walk descendants in post-order. */
+	BPF_CGROUP_ITER_ANCESTORS_UP,		/* walk ancestors upward. */
+};
 union bpf_iter_link_info {
 	struct {
 		__u32	map_fd;
 	} map;
+	struct {
+		enum bpf_cgroup_iter_order order;
+		/* At most one of cgroup_fd and cgroup_id can be non-zero. If
+		 * both are zero, the walk starts from the default cgroup v2
+		 * root. For walking v1 hierarchy, one should always explicitly
+		 * specify cgroup_fd.
+		 */
+		__u32	cgroup_fd;
+		__u64	cgroup_id;
+	} cgroup;
 };
 /* BPF syscall commands, see bpf(2) man-page for more details. */
  *
  *		**-EEXIST** if the option already exists.
  *
- *		**-EFAULT** on failrue to parse the existing header options.
+ *		**-EFAULT** on failure to parse the existing header options.
  *
  *		**-EPERM** if the helper cannot be used under the current
  *		*skops*\ **->op**.
  *		a *map* with *task* as the **key**.  From this
  *		perspective,  the usage is not much different from
  *		**bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this
- *		helper enforces the key must be an task_struct and the map must also
+ *		helper enforces the key must be a task_struct and the map must also
  *		be a **BPF_MAP_TYPE_TASK_STORAGE**.
  *
  *		Underneath, the value is stored locally at *task* instead of
  *
  * long bpf_ima_inode_hash(struct inode *inode, void *dst, u32 size)
  *	Description
- *		Returns the stored IMA hash of the *inode* (if it's avaialable).
+ *		Returns the stored IMA hash of the *inode* (if it's available).
  *		If the hash is larger than *size*, then only *size*
  *		bytes will be copied to *dst*
  *	Return
  *
  *		The argument *len_diff* can be used for querying with a planned
  *		size change. This allows to check MTU prior to changing packet
- *		ctx. Providing an *len_diff* adjustment that is larger than the
+ *		ctx. Providing a *len_diff* adjustment that is larger than the
  *		actual packet size (resulting in negative packet size) will in
- *		principle not exceed the MTU, why it is not considered a
+ *		principle not exceed the MTU, which is why it is not considered
- *		failure.  Other BPF-helpers are needed for performing the
+ *		a failure.  Other BPF helpers are needed for performing the
- *		planned size change, why the responsability for catch a negative
+ *		planned size change; therefore the responsibility for catching
- *		packet size belong in those helpers.
+ *		a negative packet size belongs in those helpers.
  *
  *		Specifying *ifindex* zero means the MTU check is performed
  *		against the current net device.  This is practical if this isn't
  *
  * int bpf_get_retval(void)
  *	Description
- *		Get the syscall's return value that will be returned to userspace.
+ *		Get the BPF program's return value that will be returned to the upper layers.
  *
- *		This helper is currently supported by cgroup programs only.
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
  *	Return
- *		The syscall's return value.
+ *		The BPF program's return value.
  *
  * int bpf_set_retval(int retval)
  *	Description
- *		Set the syscall's return value that will be returned to userspace.
+ *		Set the BPF program's return value that will be returned to the upper layers.
+ *
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
+ *
+ *		Note that there is the following corner case where the program exports an error
+ *		via bpf_set_retval but signals success via 'return 1':
+ *
+ *			bpf_set_retval(-EPERM);
+ *			return 1;
+ *
+ *		In this case, the BPF program's return value will use helper's -EPERM. This
+ *		still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case.
  *
- *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
  *
 	BPF_F_SEQ_NUMBER		= (1ULL << 3),
 };
+/* BPF_FUNC_skb_get_tunnel_key flags. */
+enum {
+	BPF_F_TUNINFO_FLAGS		= (1ULL << 4),
+};
 /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and
  * BPF_FUNC_perf_event_read_value flags.
  */
 	};
 	__u8 tunnel_tos;
 	__u8 tunnel_ttl;
-	__u16 tunnel_ext;	/* Padding, future use. */
+	union {
+		__u16 tunnel_ext;	/* compat */
+		__be16 tunnel_flags;
+	};
 	__u32 tunnel_label;
 	union {
 		__u32 local_ipv4;
 	 *    represented by BPF_REDIRECT above).
 	 */
 	BPF_LWT_REROUTE = 128,
+	/* BPF_FLOW_DISSECTOR_CONTINUE: used by BPF_PROG_TYPE_FLOW_DISSECTOR
+	 *   to indicate that no custom dissection was performed, and
+	 *   fallback to standard dissector is requested.
+	 */
+	BPF_FLOW_DISSECTOR_CONTINUE = 129,
 };
 struct bpf_sock {
 		struct {
 			__aligned_u64 target_name; /* in/out: target_name buffer ptr */
 			__u32 target_name_len;	   /* in/out: target_name buffer len */
+			/* If the iter specific field is 32 bits, it can be put
+			 * in the first or second union. Otherwise it should be
+			 * put in the second union.
+			 */
 			union {
 				struct {
 					__u32 map_id;
 				} map;
 			};
+			union {
+				struct {
+					__u64 cgroup_id;
+					__u32 order;
+				} cgroup;
+			};
 		} iter;
 		struct  {
 			__u32 netns_ino;

tools/lib/bpf/bpf_helpers.h+13−6

 /*
  * Helper function to perform a tail call with a constant/immediate map slot.
  */
-#if __clang_major__ >= 8 && defined(__bpf__)
+#if (!defined(__clang__) || __clang_major__ >= 8) && defined(__bpf__)
 static __always_inline void
 bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)
 {
 		__bpf_unreachable();
 	/*
-	 * Provide a hard guarantee that LLVM won't optimize setting r2 (map
+	 * Provide a hard guarantee that the compiler won't optimize setting r2
-	 * pointer) and r3 (constant map index) from _different paths_ ending
+	 * (map pointer) and r3 (constant map index) from _different paths_ ending
 	 * up at the _same_ call insn as otherwise we won't be able to use the
 	 * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel
 	 * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key
 	 *
 	 * Note on clobber list: we need to stay in-line with BPF calling
 	 * convention, so even if we don't end up using r0, r4, r5, we need
-	 * to mark them as clobber so that LLVM doesn't end up using them
+	 * to mark them as clobber so that the compiler doesn't end up using
-	 * before / after the call.
+	 * them before / after the call.
 	 */
-	asm volatile("r1 = %[ctx]\n\t"
+	asm volatile(
+#ifdef __clang__
+		     "r1 = %[ctx]\n\t"
 		     "r2 = %[map]\n\t"
 		     "r3 = %[slot]\n\t"
+#else
+		     "mov %%r1,%[ctx]\n\t"
+		     "mov %%r2,%[map]\n\t"
+		     "mov %%r3,%[slot]\n\t"
+#endif
 		     "call 12"
 		     :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
 		     : "r0", "r1", "r2", "r3", "r4", "r5");

tools/lib/bpf/skel_internal.h+23−0

@@ -251,6 +251,29 @@ static inline int skel_map_update_elem(int fd, const void *key, · 250 unmodified lines
251	return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);	251	return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);
252	}	252	}
253		253
		254	static inline int skel_map_delete_elem(int fd, const void *key)
		255	{
		256	const size_t attr_sz = offsetofend(union bpf_attr, flags);
		257	union bpf_attr attr;
		258
		259	memset(&attr, 0, attr_sz);
		260	attr.map_fd = fd;
		261	attr.key = (long)key;
		262
		263	return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
		264	}
		265
		266	static inline int skel_map_get_fd_by_id(__u32 id)
		267	{
		268	const size_t attr_sz = offsetofend(union bpf_attr, flags);
		269	union bpf_attr attr;
		270
		271	memset(&attr, 0, attr_sz);
		272	attr.map_id = id;
		273
		274	return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz);
		275	}
		276
254	static inline int skel_raw_tracepoint_open(const char *name, int prog_fd)	277	static inline int skel_raw_tracepoint_open(const char *name, int prog_fd)
255	{	278	{
256	const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd);	279	const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd);

tools/testing/selftests/bpf/DENYLIST.s390x+4−0

@@ -66,3 +66,7 @@ select_reuseport # intermittently fails on new s390x set · 65 unmodified lines
66	xdp_synproxy # JIT does not support calling kernel function (kfunc)	66	xdp_synproxy # JIT does not support calling kernel function (kfunc)
67	unpriv_bpf_disabled # fentry	67	unpriv_bpf_disabled # fentry
68	lru_bug # prog 'printk': failed to auto-attach: -524	68	lru_bug # prog 'printk': failed to auto-attach: -524
		69	setget_sockopt # attach unexpected error: -524 (trampoline)
		70	cb_refs # expected error message unexpected error: -524 (trampoline)
		71	cgroup_hierarchical_stats # JIT does not support calling kernel function (kfunc)
		72	htab_update # failed to attach: ERROR: strerror_r(-524)=22 (trampoline)

tools/testing/selftests/bpf/Makefile+19−18

 TEST_GEN_PROGS += test_progs-bpf_gcc
 endif
-TEST_GEN_FILES = test_lwt_ip_encap.o test_tc_edt.o
+TEST_GEN_FILES = test_lwt_ip_encap.bpf.o test_tc_edt.bpf.o
 TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c)
 # Order correspond to 'make run_tests' order
 $(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline
 $(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h
+$(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h
 # Build BPF object using Clang
 # $1 - input .c file
 LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test_subprog.c
 SKEL_BLACKLIST += $$(LSKELS)
-test_static_linked.skel.h-deps := test_static_linked1.o test_static_linked2.o
+test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o
-linked_funcs.skel.h-deps := linked_funcs1.o linked_funcs2.o
+linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o
-linked_vars.skel.h-deps := linked_vars1.o linked_vars2.o
+linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o
-linked_maps.skel.h-deps := linked_maps1.o linked_maps2.o
+linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o
 # In the subskeleton case, we want the test_subskeleton_lib.subskel.h file
 # but that's created as a side-effect of the skel.h generation.
-test_subskeleton.skel.h-deps := test_subskeleton_lib2.o test_subskeleton_lib.o test_subskeleton.o
+test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o
-test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.o test_subskeleton_lib.o
+test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o
-test_usdt.skel.h-deps := test_usdt.o test_usdt_multispec.o
+test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o
-LINKED_BPF_SRCS := $(patsubst %.o,%.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps)))
+LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps)))
 # Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on
 # $eval()) and pass control to DEFINE_TEST_RUNNER_RULES.
 TRUNNER_EXTRA_HDRS := $$(filter %.h,$(TRUNNER_EXTRA_SOURCES))
 TRUNNER_TESTS_HDR := $(TRUNNER_TESTS_DIR)/tests.h
 TRUNNER_BPF_SRCS := $$(notdir $$(wildcard $(TRUNNER_BPF_PROGS_DIR)/*.c))
-TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, $$(TRUNNER_BPF_SRCS))
+TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.bpf.o, $$(TRUNNER_BPF_SRCS))
 TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h,	\
 				 $$(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),\
 					       $$(TRUNNER_BPF_SRCS)))
 # input/output directory combination
 ifeq ($($(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs),)
 $(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs := y
-$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.o:				\
+$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o:				\
 		     $(TRUNNER_BPF_PROGS_DIR)/%.c			\
 		     $(TRUNNER_BPF_PROGS_DIR)/*.h			\
 		     $$(INCLUDE_DIR)/vmlinux.h				\
 	$$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@,			\
 					  $(TRUNNER_BPF_CFLAGS))
-$(TRUNNER_BPF_SKELS): %.skel.h: %.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
+$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
 	$$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$<
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o)
 	$(Q)diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o)
-	$(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.o=)) > $$@
+	$(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@
-	$(Q)$$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.o=)) > $$(@:.skel.h=.subskel.h)
+	$(Q)$$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h)
-$(TRUNNER_BPF_LSKELS): %.lskel.h: %.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
+$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
 	$$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$<
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o)
 	$(Q)diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o)
-	$(Q)$$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.o=_lskel)) > $$@
+	$(Q)$$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@
 $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_BPF_OBJS) $(BPFTOOL) | $(TRUNNER_OUTPUT)
-	$$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.o))
+	$$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.bpf.o))
 	$(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps))
 	$(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o)
 	$(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o)
 			     | $(TRUNNER_BINARY)-extras
 	$$(call msg,BINARY,,$$@)
 	$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) -o $$@
-	$(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.o $$@
+	$(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@
 	$(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/bootstrap/bpftool $(if $2,$2/)bpftool
 endef

tools/testing/selftests/bpf/README.rst+4−4

@@ -126,11 +126,11 @@ available in 10.0.1. The patch is available in llvm 11.0.0 trunk. · 125 unmodified lines
126		126
127	__ https://reviews.llvm.org/D78466	127	__ https://reviews.llvm.org/D78466
128		128
129	bpf_verif_scale/loop6.o test failure with Clang 12	129	bpf_verif_scale/loop6.bpf.o test failure with Clang 12
130	==================================================	130	======================================================
131		131
132	With Clang 12, the following bpf_verif_scale test failed:	132	With Clang 12, the following bpf_verif_scale test failed:
133	* ``bpf_verif_scale/loop6.o``	133	* ``bpf_verif_scale/loop6.bpf.o``
134		134
135	The verifier output looks like	135	The verifier output looks like
136		136
@@ -245,7 +245,7 @@ See `kernel llvm reloc`_ for more explanation and some examples. · 108 unmodified lines
245	Using clang 13 to compile old libbpf which has static linker support,	245	Using clang 13 to compile old libbpf which has static linker support,
246	there will be a compilation failure::	246	there will be a compilation failure::
247		247
248	libbpf: ELF relo #0 in section #6 has unexpected type 2 in .../bpf_tcp_nogpl.o	248	libbpf: ELF relo #0 in section #6 has unexpected type 2 in .../bpf_tcp_nogpl.bpf.o
249		249
250	Here, ``type 2`` refers to new relocation type ``R_BPF_64_ABS64``.	250	Here, ``type 2`` refers to new relocation type ``R_BPF_64_ABS64``.
251	To fix this issue, user newer libbpf.	251	To fix this issue, user newer libbpf.

tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h+25−0

@@ -0,0 +1,25 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	BPF_RETVAL_HOOK(ingress, "cgroup_skb/ingress", __sk_buff, -EINVAL)
	4	BPF_RETVAL_HOOK(egress, "cgroup_skb/egress", __sk_buff, -EINVAL)
	5	BPF_RETVAL_HOOK(sock_create, "cgroup/sock_create", bpf_sock, 0)
	6	BPF_RETVAL_HOOK(sock_ops, "sockops", bpf_sock_ops, -EINVAL)
	7	BPF_RETVAL_HOOK(dev, "cgroup/dev", bpf_cgroup_dev_ctx, 0)
	8	BPF_RETVAL_HOOK(bind4, "cgroup/bind4", bpf_sock_addr, 0)
	9	BPF_RETVAL_HOOK(bind6, "cgroup/bind6", bpf_sock_addr, 0)
	10	BPF_RETVAL_HOOK(connect4, "cgroup/connect4", bpf_sock_addr, 0)
	11	BPF_RETVAL_HOOK(connect6, "cgroup/connect6", bpf_sock_addr, 0)
	12	BPF_RETVAL_HOOK(post_bind4, "cgroup/post_bind4", bpf_sock_addr, 0)
	13	BPF_RETVAL_HOOK(post_bind6, "cgroup/post_bind6", bpf_sock_addr, 0)
	14	BPF_RETVAL_HOOK(sendmsg4, "cgroup/sendmsg4", bpf_sock_addr, 0)
	15	BPF_RETVAL_HOOK(sendmsg6, "cgroup/sendmsg6", bpf_sock_addr, 0)
	16	BPF_RETVAL_HOOK(sysctl, "cgroup/sysctl", bpf_sysctl, 0)
	17	BPF_RETVAL_HOOK(recvmsg4, "cgroup/recvmsg4", bpf_sock_addr, -EINVAL)
	18	BPF_RETVAL_HOOK(recvmsg6, "cgroup/recvmsg6", bpf_sock_addr, -EINVAL)
	19	BPF_RETVAL_HOOK(getsockopt, "cgroup/getsockopt", bpf_sockopt, 0)
	20	BPF_RETVAL_HOOK(setsockopt, "cgroup/setsockopt", bpf_sockopt, 0)
	21	BPF_RETVAL_HOOK(getpeername4, "cgroup/getpeername4", bpf_sock_addr, -EINVAL)
	22	BPF_RETVAL_HOOK(getpeername6, "cgroup/getpeername6", bpf_sock_addr, -EINVAL)
	23	BPF_RETVAL_HOOK(getsockname4, "cgroup/getsockname4", bpf_sock_addr, -EINVAL)
	24	BPF_RETVAL_HOOK(getsockname6, "cgroup/getsockname6", bpf_sock_addr, -EINVAL)
	25	BPF_RETVAL_HOOK(sock_release, "cgroup/sock_release", bpf_sock, 0)

tools/testing/selftests/bpf/cgroup_helpers.c+161−41

@@ -33,49 +33,52 @@ · 32 unmodified lines
33	#define CGROUP_MOUNT_DFLT "/sys/fs/cgroup"	33	#define CGROUP_MOUNT_DFLT "/sys/fs/cgroup"
34	#define NETCLS_MOUNT_PATH CGROUP_MOUNT_DFLT "/net_cls"	34	#define NETCLS_MOUNT_PATH CGROUP_MOUNT_DFLT "/net_cls"
35	#define CGROUP_WORK_DIR "/cgroup-test-work-dir"	35	#define CGROUP_WORK_DIR "/cgroup-test-work-dir"
36	#define format_cgroup_path(buf, path) \	36
		37	#define format_cgroup_path_pid(buf, path, pid) \
37	snprintf(buf, sizeof(buf), "%s%s%d%s", CGROUP_MOUNT_PATH, \	38	snprintf(buf, sizeof(buf), "%s%s%d%s", CGROUP_MOUNT_PATH, \
38	CGROUP_WORK_DIR, getpid(), path)	39	CGROUP_WORK_DIR, pid, path)
		40
		41	#define format_cgroup_path(buf, path) \
		42	format_cgroup_path_pid(buf, path, getpid())
		43
		44	#define format_parent_cgroup_path(buf, path) \
		45	format_cgroup_path_pid(buf, path, getppid())
39		46
40	#define format_classid_path(buf) \	47	#define format_classid_path(buf) \
41	snprintf(buf, sizeof(buf), "%s%s", NETCLS_MOUNT_PATH, \	48	snprintf(buf, sizeof(buf), "%s%s", NETCLS_MOUNT_PATH, \
42	CGROUP_WORK_DIR)	49	CGROUP_WORK_DIR)
43		50
44	/**	51	static int __enable_controllers(const char cgroup_path, const char controllers)
45	* enable_all_controllers() - Enable all available cgroup v2 controllers
46	*
47	* Enable all available cgroup v2 controllers in order to increase
48	* the code coverage.
49	*
50	* If successful, 0 is returned.
51	*/
52	static int enable_all_controllers(char *cgroup_path)
53	{	52	{
54	char path[PATH_MAX + 1];	53	char path[PATH_MAX + 1];
55	char buf[PATH_MAX];	54	char enable[PATH_MAX + 1];
56	char c, c2;	55	char c, c2;
57	int fd, cfd;	56	int fd, cfd;
58	ssize_t len;	57	ssize_t len;
59		58
60	snprintf(path, sizeof(path), "%s/cgroup.controllers", cgroup_path);	59	/* If not controllers are passed, enable all available controllers */
61	fd = open(path, O_RDONLY);	60	if (!controllers) {
62	if (fd < 0) {	61	snprintf(path, sizeof(path), "%s/cgroup.controllers",
63	log_err("Opening cgroup.controllers: %s", path);	62	cgroup_path);
64	return 1;	63	fd = open(path, O_RDONLY);
65	}	64	if (fd < 0) {
66		65	log_err("Opening cgroup.controllers: %s", path);
67	len = read(fd, buf, sizeof(buf) - 1);	66	return 1;
68	if (len < 0) {	67	}
		68	len = read(fd, enable, sizeof(enable) - 1);
		69	if (len < 0) {
		70	close(fd);
		71	log_err("Reading cgroup.controllers: %s", path);
		72	return 1;
		73	} else if (len == 0) { /* No controllers to enable */
		74	close(fd);
		75	return 0;
		76	}
		77	enable[len] = 0;
69	close(fd);	78	close(fd);
70	log_err("Reading cgroup.controllers: %s", path);	79	} else {
71	return 1;	80	strncpy(enable, controllers, sizeof(enable));
72	}	81	}
73	buf[len] = 0;
74	close(fd);
75
76	/* No controllers available? We're probably on cgroup v1. */
77	if (len == 0)
78	return 0;
79		82
80	snprintf(path, sizeof(path), "%s/cgroup.subtree_control", cgroup_path);	83	snprintf(path, sizeof(path), "%s/cgroup.subtree_control", cgroup_path);
81	cfd = open(path, O_RDWR);	84	cfd = open(path, O_RDWR);
@@ -84,7 +87,7 @@ static int enable_all_controllers(char *cgroup_path) · 2 unmodified lines
84	return 1;	87	return 1;
85	}	88	}
86		89
87	for (c = strtok_r(buf, " ", &c2); c; c = strtok_r(NULL, " ", &c2)) {	90	for (c = strtok_r(enable, " ", &c2); c; c = strtok_r(NULL, " ", &c2)) {
88	if (dprintf(cfd, "+%s\n", c) <= 0) {	91	if (dprintf(cfd, "+%s\n", c) <= 0) {
89	log_err("Enabling controller %s: %s", c, path);	92	log_err("Enabling controller %s: %s", c, path);
90	close(cfd);	93	close(cfd);
@@ -95,6 +98,87 @@ static int enable_all_controllers(char *cgroup_path) · 4 unmodified lines
95	return 0;	98	return 0;
96	}	99	}
97		100
		101	/**
		102	* enable_controllers() - Enable cgroup v2 controllers
		103	* @relative_path: The cgroup path, relative to the workdir
		104	* @controllers: List of controllers to enable in cgroup.controllers format
		105	*
		106	*
		107	* Enable given cgroup v2 controllers, if @controllers is NULL, enable all
		108	* available controllers.
		109	*
		110	* If successful, 0 is returned.
		111	*/
		112	int enable_controllers(const char relative_path, const char controllers)
		113	{
		114	char cgroup_path[PATH_MAX + 1];
		115
		116	format_cgroup_path(cgroup_path, relative_path);
		117	return __enable_controllers(cgroup_path, controllers);
		118	}
		119
		120	static int __write_cgroup_file(const char cgroup_path, const char file,
		121	const char *buf)
		122	{
		123	char file_path[PATH_MAX + 1];
		124	int fd;
		125
		126	snprintf(file_path, sizeof(file_path), "%s/%s", cgroup_path, file);
		127	fd = open(file_path, O_RDWR);
		128	if (fd < 0) {
		129	log_err("Opening %s", file_path);
		130	return 1;
		131	}
		132
		133	if (dprintf(fd, "%s", buf) <= 0) {
		134	log_err("Writing to %s", file_path);
		135	close(fd);
		136	return 1;
		137	}
		138	close(fd);
		139	return 0;
		140	}
		141
		142	/**
		143	* write_cgroup_file() - Write to a cgroup file
		144	* @relative_path: The cgroup path, relative to the workdir
		145	* @file: The name of the file in cgroupfs to write to
		146	* @buf: Buffer to write to the file
		147	*
		148	* Write to a file in the given cgroup's directory.
		149	*
		150	* If successful, 0 is returned.
		151	*/
		152	int write_cgroup_file(const char relative_path, const char file,
		153	const char *buf)
		154	{
		155	char cgroup_path[PATH_MAX - 24];
		156
		157	format_cgroup_path(cgroup_path, relative_path);
		158	return __write_cgroup_file(cgroup_path, file, buf);
		159	}
		160
		161	/**
		162	* write_cgroup_file_parent() - Write to a cgroup file in the parent process
		163	* workdir
		164	* @relative_path: The cgroup path, relative to the parent process workdir
		165	* @file: The name of the file in cgroupfs to write to
		166	* @buf: Buffer to write to the file
		167	*
		168	* Write to a file in the given cgroup's directory under the parent process
		169	* workdir.
		170	*
		171	* If successful, 0 is returned.
		172	*/
		173	int write_cgroup_file_parent(const char relative_path, const char file,
		174	const char *buf)
		175	{
		176	char cgroup_path[PATH_MAX - 24];
		177
		178	format_parent_cgroup_path(cgroup_path, relative_path);
		179	return __write_cgroup_file(cgroup_path, file, buf);
		180	}
		181
98	/**	182	/**
99	* setup_cgroup_environment() - Setup the cgroup environment	183	* setup_cgroup_environment() - Setup the cgroup environment
100	*	184	*
@@ -133,7 +217,9 @@ int setup_cgroup_environment(void) · 32 unmodified lines
133	return 1;	217	return 1;
134	}	218	}
135		219
136	if (enable_all_controllers(cgroup_workdir))	220	/* Enable all available controllers to increase test coverage */
		221	if (__enable_controllers(CGROUP_MOUNT_PATH, NULL) \|\|
		222	__enable_controllers(cgroup_workdir, NULL))
137	return 1;	223	return 1;
138		224
139	return 0;	225	return 0;
@@ -173,7 +259,7 @@ static int join_cgroup_from_top(const char *cgroup_path) · 33 unmodified lines
173		259
174	/**	260	/**
175	* join_cgroup() - Join a cgroup	261	* join_cgroup() - Join a cgroup
176	* @path: The cgroup path, relative to the workdir, to join	262	* @relative_path: The cgroup path, relative to the workdir, to join
177	*	263	*
178	* This function expects a cgroup to already be created, relative to the cgroup	264	* This function expects a cgroup to already be created, relative to the cgroup
179	* work dir, and it joins it. For example, passing "/my-cgroup" as the path	265	* work dir, and it joins it. For example, passing "/my-cgroup" as the path
@@ -182,11 +268,27 @@ static int join_cgroup_from_top(const char *cgroup_path) · 2 unmodified lines
182	*	268	*
183	* On success, it returns 0, otherwise on failure it returns 1.	269	* On success, it returns 0, otherwise on failure it returns 1.
184	*/	270	*/
185	int join_cgroup(const char *path)	271	int join_cgroup(const char *relative_path)
		272	{
		273	char cgroup_path[PATH_MAX + 1];
		274
		275	format_cgroup_path(cgroup_path, relative_path);
		276	return join_cgroup_from_top(cgroup_path);
		277	}
		278
		279	/**
		280	* join_parent_cgroup() - Join a cgroup in the parent process workdir
		281	* @relative_path: The cgroup path, relative to parent process workdir, to join
		282	*
		283	* See join_cgroup().
		284	*
		285	* On success, it returns 0, otherwise on failure it returns 1.
		286	*/
		287	int join_parent_cgroup(const char *relative_path)
186	{	288	{
187	char cgroup_path[PATH_MAX + 1];	289	char cgroup_path[PATH_MAX + 1];
188		290
189	format_cgroup_path(cgroup_path, path);	291	format_parent_cgroup_path(cgroup_path, relative_path);
190	return join_cgroup_from_top(cgroup_path);	292	return join_cgroup_from_top(cgroup_path);
191	}	293	}
192		294
@@ -212,9 +314,27 @@ void cleanup_cgroup_environment(void) · 19 unmodified lines
212	nftw(cgroup_workdir, nftwfunc, WALK_FD_LIMIT, FTW_DEPTH \| FTW_MOUNT);	314	nftw(cgroup_workdir, nftwfunc, WALK_FD_LIMIT, FTW_DEPTH \| FTW_MOUNT);
213	}	315	}
214		316
		317	/**
		318	* get_root_cgroup() - Get the FD of the root cgroup
		319	*
		320	* On success, it returns the file descriptor. On failure, it returns -1.
		321	* If there is a failure, it prints the error to stderr.
		322	*/
		323	int get_root_cgroup(void)
		324	{
		325	int fd;
		326
		327	fd = open(CGROUP_MOUNT_PATH, O_RDONLY);
		328	if (fd < 0) {
		329	log_err("Opening root cgroup");
		330	return -1;
		331	}
		332	return fd;
		333	}
		334
215	/**	335	/**
216	* create_and_get_cgroup() - Create a cgroup, relative to workdir, and get the FD	336	* create_and_get_cgroup() - Create a cgroup, relative to workdir, and get the FD
217	* @path: The cgroup path, relative to the workdir, to join	337	* @relative_path: The cgroup path, relative to the workdir, to join
218	*	338	*
219	* This function creates a cgroup under the top level workdir and returns the	339	* This function creates a cgroup under the top level workdir and returns the
220	* file descriptor. It is idempotent.	340	* file descriptor. It is idempotent.
@@ -222,14 +342,14 @@ void cleanup_cgroup_environment(void) · 1 unmodified line
222	* On success, it returns the file descriptor. On failure it returns -1.	342	* On success, it returns the file descriptor. On failure it returns -1.
223	* If there is a failure, it prints the error to stderr.	343	* If there is a failure, it prints the error to stderr.
224	*/	344	*/
225	int create_and_get_cgroup(const char *path)	345	int create_and_get_cgroup(const char *relative_path)
226	{	346	{
227	char cgroup_path[PATH_MAX + 1];	347	char cgroup_path[PATH_MAX + 1];
228	int fd;	348	int fd;
229		349
230	format_cgroup_path(cgroup_path, path);	350	format_cgroup_path(cgroup_path, relative_path);
231	if (mkdir(cgroup_path, 0777) && errno != EEXIST) {	351	if (mkdir(cgroup_path, 0777) && errno != EEXIST) {
232	log_err("mkdiring cgroup %s .. %s", path, cgroup_path);	352	log_err("mkdiring cgroup %s .. %s", relative_path, cgroup_path);
233	return -1;	353	return -1;
234	}	354	}
235		355
@@ -244,13 +364,13 @@ int create_and_get_cgroup(const char *path) · 8 unmodified lines
244		364
245	/**	365	/**
246	* get_cgroup_id() - Get cgroup id for a particular cgroup path	366	* get_cgroup_id() - Get cgroup id for a particular cgroup path
247	* @path: The cgroup path, relative to the workdir, to join	367	* @relative_path: The cgroup path, relative to the workdir, to join
248	*	368	*
249	* On success, it returns the cgroup id. On failure it returns 0,	369	* On success, it returns the cgroup id. On failure it returns 0,
250	* which is an invalid cgroup id.	370	* which is an invalid cgroup id.
251	* If there is a failure, it prints the error to stderr.	371	* If there is a failure, it prints the error to stderr.
252	*/	372	*/
253	unsigned long long get_cgroup_id(const char *path)	373	unsigned long long get_cgroup_id(const char *relative_path)
254	{	374	{
255	int dirfd, err, flags, mount_id, fhsize;	375	int dirfd, err, flags, mount_id, fhsize;
256	union {	376	union {
@@ -261,7 +381,7 @@ unsigned long long get_cgroup_id(const char *path) · 4 unmodified lines
261	struct file_handle fhp, fhp2;	381	struct file_handle fhp, fhp2;
262	unsigned long long ret = 0;	382	unsigned long long ret = 0;
263		383
264	format_cgroup_path(cgroup_workdir, path);	384	format_cgroup_path(cgroup_workdir, relative_path);
265		385
266	dirfd = AT_FDCWD;	386	dirfd = AT_FDCWD;
267	flags = 0;	387	flags = 0;

tools/testing/selftests/bpf/cgroup_helpers.h+13−6

@@ -10,11 +10,18 @@ · 9 unmodified lines
10	__FILE__, __LINE__, clean_errno(), ##__VA_ARGS__)	10	__FILE__, __LINE__, clean_errno(), ##__VA_ARGS__)
11		11
12	/* cgroupv2 related */	12	/* cgroupv2 related */
13	int cgroup_setup_and_join(const char *path);	13	int enable_controllers(const char relative_path, const char controllers);
14	int create_and_get_cgroup(const char *path);	14	int write_cgroup_file(const char relative_path, const char file,
15	unsigned long long get_cgroup_id(const char *path);	15	const char *buf);
16		16	int write_cgroup_file_parent(const char relative_path, const char file,
17	int join_cgroup(const char *path);	17	const char *buf);
		18	int cgroup_setup_and_join(const char *relative_path);
		19	int get_root_cgroup(void);
		20	int create_and_get_cgroup(const char *relative_path);
		21	unsigned long long get_cgroup_id(const char *relative_path);
		22
		23	int join_cgroup(const char *relative_path);
		24	int join_parent_cgroup(const char *relative_path);
18		25
19	int setup_cgroup_environment(void);	26	int setup_cgroup_environment(void);
20	void cleanup_cgroup_environment(void);	27	void cleanup_cgroup_environment(void);
@@ -26,4 +33,4 @@ int join_classid(void); · 5 unmodified lines
26	int setup_classid_environment(void);	33	int setup_classid_environment(void);
27	void cleanup_classid_environment(void);	34	void cleanup_classid_environment(void);
28		35
29	#endif /* __CGROUP_HELPERS_H */	36	#endif /* __CGROUP_HELPERS_H */

tools/testing/selftests/bpf/get_cgroup_id_user.c+1−1

@@ -48,7 +48,7 @@ static int bpf_find_map(const char test, struct bpf_object obj, · 47 unmodified lines
48	int main(int argc, char **argv)	48	int main(int argc, char **argv)
49	{	49	{
50	const char *probe_name = "syscalls/sys_enter_nanosleep";	50	const char *probe_name = "syscalls/sys_enter_nanosleep";
51	const char *file = "get_cgroup_id_kern.o";	51	const char *file = "get_cgroup_id_kern.bpf.o";
52	int err, bytes, efd, prog_fd, pmu_fd;	52	int err, bytes, efd, prog_fd, pmu_fd;
53	int cgroup_fd, cgidmap_fd, pidmap_fd;	53	int cgroup_fd, cgidmap_fd, pidmap_fd;
54	struct perf_event_attr attr = {};	54	struct perf_event_attr attr = {};

tools/testing/selftests/bpf/map_tests/task_storage_map.c+122−0

@@ -0,0 +1,122 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
	3	#define _GNU_SOURCE
	4	#include <sched.h>
	5	#include <unistd.h>
	6	#include <stdlib.h>
	7	#include <stdbool.h>
	8	#include <errno.h>
	9	#include <string.h>
	10	#include <pthread.h>
	11
	12	#include <bpf/bpf.h>
	13	#include <bpf/libbpf.h>
	14
	15	#include "test_maps.h"
	16	#include "task_local_storage_helpers.h"
	17	#include "read_bpf_task_storage_busy.skel.h"
	18
	19	struct lookup_ctx {
	20	bool start;
	21	bool stop;
	22	int pid_fd;
	23	int map_fd;
	24	int loop;
	25	};
	26
	27	static void lookup_fn(void arg)
	28	{
	29	struct lookup_ctx *ctx = arg;
	30	long value;
	31	int i = 0;
	32
	33	while (!ctx->start)
	34	usleep(1);
	35
	36	while (!ctx->stop && i++ < ctx->loop)
	37	bpf_map_lookup_elem(ctx->map_fd, &ctx->pid_fd, &value);
	38	return NULL;
	39	}
	40
	41	static void abort_lookup(struct lookup_ctx ctx, pthread_t tids, unsigned int nr)
	42	{
	43	unsigned int i;
	44
	45	ctx->stop = true;
	46	ctx->start = true;
	47	for (i = 0; i < nr; i++)
	48	pthread_join(tids[i], NULL);
	49	}
	50
	51	void test_task_storage_map_stress_lookup(void)
	52	{
	53	#define MAX_NR_THREAD 4096
	54	unsigned int i, nr = 256, loop = 8192, cpu = 0;
	55	struct read_bpf_task_storage_busy *skel;
	56	pthread_t tids[MAX_NR_THREAD];
	57	struct lookup_ctx ctx;
	58	cpu_set_t old, new;
	59	const char *cfg;
	60	int err;
	61
	62	cfg = getenv("TASK_STORAGE_MAP_NR_THREAD");
	63	if (cfg) {
	64	nr = atoi(cfg);
	65	if (nr > MAX_NR_THREAD)
	66	nr = MAX_NR_THREAD;
	67	}
	68	cfg = getenv("TASK_STORAGE_MAP_NR_LOOP");
	69	if (cfg)
	70	loop = atoi(cfg);
	71	cfg = getenv("TASK_STORAGE_MAP_PIN_CPU");
	72	if (cfg)
	73	cpu = atoi(cfg);
	74
	75	skel = read_bpf_task_storage_busy__open_and_load();
	76	err = libbpf_get_error(skel);
	77	CHECK(err, "open_and_load", "error %d\n", err);
	78
	79	/* Only for a fully preemptible kernel */
	80	if (!skel->kconfig->CONFIG_PREEMPT)
	81	return;
	82
	83	/* Save the old affinity setting */
	84	sched_getaffinity(getpid(), sizeof(old), &old);
	85
	86	/* Pinned on a specific CPU */
	87	CPU_ZERO(&new);
	88	CPU_SET(cpu, &new);
	89	sched_setaffinity(getpid(), sizeof(new), &new);
	90
	91	ctx.start = false;
	92	ctx.stop = false;
	93	ctx.pid_fd = sys_pidfd_open(getpid(), 0);
	94	ctx.map_fd = bpf_map__fd(skel->maps.task);
	95	ctx.loop = loop;
	96	for (i = 0; i < nr; i++) {
	97	err = pthread_create(&tids[i], NULL, lookup_fn, &ctx);
	98	if (err) {
	99	abort_lookup(&ctx, tids, i);
	100	CHECK(err, "pthread_create", "error %d\n", err);
	101	goto out;
	102	}
	103	}
	104
	105	ctx.start = true;
	106	for (i = 0; i < nr; i++)
	107	pthread_join(tids[i], NULL);
	108
	109	skel->bss->pid = getpid();
	110	err = read_bpf_task_storage_busy__attach(skel);
	111	CHECK(err, "attach", "error %d\n", err);
	112
	113	/* Trigger program */
	114	syscall(SYS_gettid);
	115	skel->bss->pid = 0;
	116
	117	CHECK(skel->bss->busy != 0, "bad bpf_task_storage_busy", "got %d\n", skel->bss->busy);
	118	out:
	119	read_bpf_task_storage_busy__destroy(skel);
	120	/* Restore affinity setting */
	121	sched_setaffinity(getpid(), sizeof(old), &old);
	122	}

tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c+1−1

@@ -7,7 +7,7 @@ void serial_test_bpf_obj_id(void) · 6 unmodified lines
7	{	7	{
8	const __u64 array_magic_value = 0xfaceb00c;	8	const __u64 array_magic_value = 0xfaceb00c;
9	const __u32 array_key = 0;	9	const __u32 array_key = 0;
10	const char *file = "./test_obj_id.o";	10	const char *file = "./test_obj_id.bpf.o";
11	const char *expected_prog_name = "test_obj_id";	11	const char *expected_prog_name = "test_obj_id";
12	const char *expected_map_name = "test_map_id";	12	const char *expected_map_name = "test_map_id";
13	const __u64 nsec_per_sec = 1000000000;	13	const __u64 nsec_per_sec = 1000000000;

tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c+27−27

@@ -75,45 +75,45 @@ static void scale_test(const char *file, · 74 unmodified lines
75		75
76	void test_verif_scale1()	76	void test_verif_scale1()
77	{	77	{
78	scale_test("test_verif_scale1.o", BPF_PROG_TYPE_SCHED_CLS, false);	78	scale_test("test_verif_scale1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
79	}	79	}
80		80
81	void test_verif_scale2()	81	void test_verif_scale2()
82	{	82	{
83	scale_test("test_verif_scale2.o", BPF_PROG_TYPE_SCHED_CLS, false);	83	scale_test("test_verif_scale2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
84	}	84	}
85		85
86	void test_verif_scale3()	86	void test_verif_scale3()
87	{	87	{
88	scale_test("test_verif_scale3.o", BPF_PROG_TYPE_SCHED_CLS, false);	88	scale_test("test_verif_scale3.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
89	}	89	}
90		90
91	void test_verif_scale_pyperf_global()	91	void test_verif_scale_pyperf_global()
92	{	92	{
93	scale_test("pyperf_global.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	93	scale_test("pyperf_global.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
94	}	94	}
95		95
96	void test_verif_scale_pyperf_subprogs()	96	void test_verif_scale_pyperf_subprogs()
97	{	97	{
98	scale_test("pyperf_subprogs.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	98	scale_test("pyperf_subprogs.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
99	}	99	}
100		100
101	void test_verif_scale_pyperf50()	101	void test_verif_scale_pyperf50()
102	{	102	{
103	/* full unroll by llvm */	103	/* full unroll by llvm */
104	scale_test("pyperf50.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	104	scale_test("pyperf50.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
105	}	105	}
106		106
107	void test_verif_scale_pyperf100()	107	void test_verif_scale_pyperf100()
108	{	108	{
109	/* full unroll by llvm */	109	/* full unroll by llvm */
110	scale_test("pyperf100.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	110	scale_test("pyperf100.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
111	}	111	}
112		112
113	void test_verif_scale_pyperf180()	113	void test_verif_scale_pyperf180()
114	{	114	{
115	/* full unroll by llvm */	115	/* full unroll by llvm */
116	scale_test("pyperf180.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	116	scale_test("pyperf180.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
117	}	117	}
118		118
119	void test_verif_scale_pyperf600()	119	void test_verif_scale_pyperf600()
@@ -124,13 +124,13 @@ void test_verif_scale_pyperf600() · 4 unmodified lines
124	* 16k insns in loop body.	124	* 16k insns in loop body.
125	* Total of 5 such loops. Total program size ~82k insns.	125	* Total of 5 such loops. Total program size ~82k insns.
126	*/	126	*/
127	scale_test("pyperf600.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	127	scale_test("pyperf600.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
128	}	128	}
129		129
130	void test_verif_scale_pyperf600_bpf_loop(void)	130	void test_verif_scale_pyperf600_bpf_loop(void)
131	{	131	{
132	/* use the bpf_loop helper*/	132	/* use the bpf_loop helper*/
133	scale_test("pyperf600_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	133	scale_test("pyperf600_bpf_loop.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
134	}	134	}
135		135
136	void test_verif_scale_pyperf600_nounroll()	136	void test_verif_scale_pyperf600_nounroll()
@@ -141,37 +141,37 @@ void test_verif_scale_pyperf600_nounroll() · 4 unmodified lines
141	* ~110 insns in loop body.	141	* ~110 insns in loop body.
142	* Total of 5 such loops. Total program size ~1500 insns.	142	* Total of 5 such loops. Total program size ~1500 insns.
143	*/	143	*/
144	scale_test("pyperf600_nounroll.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	144	scale_test("pyperf600_nounroll.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
145	}	145	}
146		146
147	void test_verif_scale_loop1()	147	void test_verif_scale_loop1()
148	{	148	{
149	scale_test("loop1.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	149	scale_test("loop1.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
150	}	150	}
151		151
152	void test_verif_scale_loop2()	152	void test_verif_scale_loop2()
153	{	153	{
154	scale_test("loop2.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	154	scale_test("loop2.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
155	}	155	}
156		156
157	void test_verif_scale_loop3_fail()	157	void test_verif_scale_loop3_fail()
158	{	158	{
159	scale_test("loop3.o", BPF_PROG_TYPE_RAW_TRACEPOINT, true /* fails */);	159	scale_test("loop3.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, true /* fails */);
160	}	160	}
161		161
162	void test_verif_scale_loop4()	162	void test_verif_scale_loop4()
163	{	163	{
164	scale_test("loop4.o", BPF_PROG_TYPE_SCHED_CLS, false);	164	scale_test("loop4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
165	}	165	}
166		166
167	void test_verif_scale_loop5()	167	void test_verif_scale_loop5()
168	{	168	{
169	scale_test("loop5.o", BPF_PROG_TYPE_SCHED_CLS, false);	169	scale_test("loop5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
170	}	170	}
171		171
172	void test_verif_scale_loop6()	172	void test_verif_scale_loop6()
173	{	173	{
174	scale_test("loop6.o", BPF_PROG_TYPE_KPROBE, false);	174	scale_test("loop6.bpf.o", BPF_PROG_TYPE_KPROBE, false);
175	}	175	}
176		176
177	void test_verif_scale_strobemeta()	177	void test_verif_scale_strobemeta()
@@ -180,54 +180,54 @@ void test_verif_scale_strobemeta() · 2 unmodified lines
180	* Total program size 20.8k insn.	180	* Total program size 20.8k insn.
181	* ~350k processed_insns	181	* ~350k processed_insns
182	*/	182	*/
183	scale_test("strobemeta.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	183	scale_test("strobemeta.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
184	}	184	}
185		185
186	void test_verif_scale_strobemeta_bpf_loop(void)	186	void test_verif_scale_strobemeta_bpf_loop(void)
187	{	187	{
188	/* use the bpf_loop helper*/	188	/* use the bpf_loop helper*/
189	scale_test("strobemeta_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	189	scale_test("strobemeta_bpf_loop.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
190	}	190	}
191		191
192	void test_verif_scale_strobemeta_nounroll1()	192	void test_verif_scale_strobemeta_nounroll1()
193	{	193	{
194	/* no unroll, tiny loops */	194	/* no unroll, tiny loops */
195	scale_test("strobemeta_nounroll1.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	195	scale_test("strobemeta_nounroll1.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
196	}	196	}
197		197
198	void test_verif_scale_strobemeta_nounroll2()	198	void test_verif_scale_strobemeta_nounroll2()
199	{	199	{
200	/* no unroll, tiny loops */	200	/* no unroll, tiny loops */
201	scale_test("strobemeta_nounroll2.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	201	scale_test("strobemeta_nounroll2.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
202	}	202	}
203		203
204	void test_verif_scale_strobemeta_subprogs()	204	void test_verif_scale_strobemeta_subprogs()
205	{	205	{
206	/* non-inlined subprogs */	206	/* non-inlined subprogs */
207	scale_test("strobemeta_subprogs.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);	207	scale_test("strobemeta_subprogs.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
208	}	208	}
209		209
210	void test_verif_scale_sysctl_loop1()	210	void test_verif_scale_sysctl_loop1()
211	{	211	{
212	scale_test("test_sysctl_loop1.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);	212	scale_test("test_sysctl_loop1.bpf.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);
213	}	213	}
214		214
215	void test_verif_scale_sysctl_loop2()	215	void test_verif_scale_sysctl_loop2()
216	{	216	{
217	scale_test("test_sysctl_loop2.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);	217	scale_test("test_sysctl_loop2.bpf.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);
218	}	218	}
219		219
220	void test_verif_scale_xdp_loop()	220	void test_verif_scale_xdp_loop()
221	{	221	{
222	scale_test("test_xdp_loop.o", BPF_PROG_TYPE_XDP, false);	222	scale_test("test_xdp_loop.bpf.o", BPF_PROG_TYPE_XDP, false);
223	}	223	}
224		224
225	void test_verif_scale_seg6_loop()	225	void test_verif_scale_seg6_loop()
226	{	226	{
227	scale_test("test_seg6_loop.o", BPF_PROG_TYPE_LWT_SEG6LOCAL, false);	227	scale_test("test_seg6_loop.bpf.o", BPF_PROG_TYPE_LWT_SEG6LOCAL, false);
228	}	228	}
229		229
230	void test_verif_twfw()	230	void test_verif_twfw()
231	{	231	{
232	scale_test("twfw.o", BPF_PROG_TYPE_CGROUP_SKB, false);	232	scale_test("twfw.bpf.o", BPF_PROG_TYPE_CGROUP_SKB, false);
233	}	233	}

tools/testing/selftests/bpf/prog_tests/btf.c+2−2

@@ -4651,8 +4651,8 @@ struct btf_file_test { · 4650 unmodified lines
4651	};	4651	};
4652		4652
4653	static struct btf_file_test file_tests[] = {	4653	static struct btf_file_test file_tests[] = {
4654	{ .file = "test_btf_newkv.o", },	4654	{ .file = "test_btf_newkv.bpf.o", },
4655	{ .file = "test_btf_nokv.o", .btf_kv_notfound = true, },	4655	{ .file = "test_btf_nokv.bpf.o", .btf_kv_notfound = true, },
4656	};	4656	};
4657		4657
4658	static void do_test_file(unsigned int test_num)	4658	static void do_test_file(unsigned int test_num)

tools/testing/selftests/bpf/prog_tests/btf_dump.c+5−5

@@ -52,7 +52,7 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) · 51 unmodified lines
52	int err = 0, fd = -1;	52	int err = 0, fd = -1;
53	FILE *f = NULL;	53	FILE *f = NULL;
54		54
55	snprintf(test_file, sizeof(test_file), "%s.o", t->file);	55	snprintf(test_file, sizeof(test_file), "%s.bpf.o", t->file);
56		56
57	btf = btf__parse_elf(test_file, NULL);	57	btf = btf__parse_elf(test_file, NULL);
58	if (!ASSERT_OK_PTR(btf, "btf_parse_elf")) {	58	if (!ASSERT_OK_PTR(btf, "btf_parse_elf")) {
@@ -764,8 +764,8 @@ static void test_btf_dump_struct_data(struct btf btf, struct btf_dump d, · 705 unmodified lines
764		764
765	/* union with nested struct */	765	/* union with nested struct */
766	TEST_BTF_DUMP_DATA(btf, d, "union", str, union bpf_iter_link_info, BTF_F_COMPACT,	766	TEST_BTF_DUMP_DATA(btf, d, "union", str, union bpf_iter_link_info, BTF_F_COMPACT,
767	"(union bpf_iter_link_info){.map = (struct){.map_fd = (__u32)1,},}",	767	"(union bpf_iter_link_info){.map = (struct){.map_fd = (__u32)1,},.cgroup = (struct){.order = (enum bpf_cgroup_iter_order)BPF_CGROUP_ITER_SELF_ONLY,.cgroup_fd = (__u32)1,},}",
768	{ .map = { .map_fd = 1 }});	768	{ .cgroup = { .order = 1, .cgroup_fd = 1, }});
769		769
770	/* struct skb with nested structs/unions; because type output is so	770	/* struct skb with nested structs/unions; because type output is so
771	* complex, we don't do a string comparison, just verify we return	771	* complex, we don't do a string comparison, just verify we return
@@ -841,8 +841,8 @@ static void test_btf_dump_datasec_data(char *str) · 69 unmodified lines
841	char license[4] = "GPL";	841	char license[4] = "GPL";
842	struct btf_dump *d;	842	struct btf_dump *d;
843		843
844	btf = btf__parse("xdping_kern.o", NULL);	844	btf = btf__parse("xdping_kern.bpf.o", NULL);
845	if (!ASSERT_OK_PTR(btf, "xdping_kern.o BTF not found"))	845	if (!ASSERT_OK_PTR(btf, "xdping_kern.bpf.o BTF not found"))
846	return;	846	return;
847		847
848	d = btf_dump__new(btf, btf_dump_snprintf, str, NULL);	848	d = btf_dump__new(btf, btf_dump_snprintf, str, NULL);

tools/testing/selftests/bpf/prog_tests/btf_endian.c+1−1

@@ -23,7 +23,7 @@ void test_btf_endian() { · 22 unmodified lines
23	int var_id;	23	int var_id;
24		24
25	/* Load BTF in native endianness */	25	/* Load BTF in native endianness */
26	btf = btf__parse_elf("btf_dump_test_case_syntax.o", NULL);	26	btf = btf__parse_elf("btf_dump_test_case_syntax.bpf.o", NULL);
27	if (!ASSERT_OK_PTR(btf, "parse_native_btf"))	27	if (!ASSERT_OK_PTR(btf, "parse_native_btf"))
28	goto err_out;	28	goto err_out;
29		29

tools/testing/selftests/bpf/prog_tests/cb_refs.c+48−0

@@ -0,0 +1,48 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include "bpf/libbpf.h"
	3	#include <test_progs.h>
	4	#include <network_helpers.h>
	5
	6	#include "cb_refs.skel.h"
	7
	8	static char log_buf[1024 * 1024];
	9
	10	struct {
	11	const char *prog_name;
	12	const char *err_msg;
	13	} cb_refs_tests[] = {
	14	{ "underflow_prog", "reference has not been acquired before" },
	15	{ "leak_prog", "Unreleased reference" },
	16	{ "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */
	17	{ "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */
	18	};
	19
	20	void test_cb_refs(void)
	21	{
	22	LIBBPF_OPTS(bpf_object_open_opts, opts, .kernel_log_buf = log_buf,
	23	.kernel_log_size = sizeof(log_buf),
	24	.kernel_log_level = 1);
	25	struct bpf_program *prog;
	26	struct cb_refs *skel;
	27	int i;
	28
	29	for (i = 0; i < ARRAY_SIZE(cb_refs_tests); i++) {
	30	LIBBPF_OPTS(bpf_test_run_opts, run_opts,
	31	.data_in = &pkt_v4,
	32	.data_size_in = sizeof(pkt_v4),
	33	.repeat = 1,
	34	);
	35	skel = cb_refs__open_opts(&opts);
	36	if (!ASSERT_OK_PTR(skel, "cb_refs__open_and_load"))
	37	return;
	38	prog = bpf_object__find_program_by_name(skel->obj, cb_refs_tests[i].prog_name);
	39	bpf_program__set_autoload(prog, true);
	40	if (!ASSERT_ERR(cb_refs__load(skel), "cb_refs__load"))
	41	bpf_prog_test_run_opts(bpf_program__fd(prog), &run_opts);
	42	if (!ASSERT_OK_PTR(strstr(log_buf, cb_refs_tests[i].err_msg), "expected error message")) {
	43	fprintf(stderr, "Expected: %s\n", cb_refs_tests[i].err_msg);
	44	fprintf(stderr, "Verifier: %s\n", log_buf);
	45	}
	46	cb_refs__destroy(skel);
	47	}
	48	}

tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c+48−0

@@ -10,6 +10,7 @@ · 9 unmodified lines
10		10
11	#include "cgroup_getset_retval_setsockopt.skel.h"	11	#include "cgroup_getset_retval_setsockopt.skel.h"
12	#include "cgroup_getset_retval_getsockopt.skel.h"	12	#include "cgroup_getset_retval_getsockopt.skel.h"
		13	#include "cgroup_getset_retval_hooks.skel.h"
13		14
14	#define SOL_CUSTOM 0xdeadbeef	15	#define SOL_CUSTOM 0xdeadbeef
15		16
@@ -433,6 +434,50 @@ close_bpf_object: · 417 unmodified lines
433	cgroup_getset_retval_getsockopt__destroy(obj);	434	cgroup_getset_retval_getsockopt__destroy(obj);
434	}	435	}
435		436
		437	struct exposed_hook {
		438	const char *name;
		439	int expected_err;
		440	} exposed_hooks[] = {
		441
		442	#define BPF_RETVAL_HOOK(NAME, SECTION, CTX, EXPECTED_ERR) \
		443	{ \
		444	.name = #NAME, \
		445	.expected_err = EXPECTED_ERR, \
		446	},
		447
		448	#include "cgroup_getset_retval_hooks.h"
		449
		450	#undef BPF_RETVAL_HOOK
		451	};
		452
		453	static void test_exposed_hooks(int cgroup_fd, int sock_fd)
		454	{
		455	struct cgroup_getset_retval_hooks *skel;
		456	struct bpf_program *prog;
		457	int err;
		458	int i;
		459
		460	for (i = 0; i < ARRAY_SIZE(exposed_hooks); i++) {
		461	skel = cgroup_getset_retval_hooks__open();
		462	if (!ASSERT_OK_PTR(skel, "cgroup_getset_retval_hooks__open"))
		463	continue;
		464
		465	prog = bpf_object__find_program_by_name(skel->obj, exposed_hooks[i].name);
		466	if (!ASSERT_NEQ(prog, NULL, "bpf_object__find_program_by_name"))
		467	goto close_skel;
		468
		469	err = bpf_program__set_autoload(prog, true);
		470	if (!ASSERT_OK(err, "bpf_program__set_autoload"))
		471	goto close_skel;
		472
		473	err = cgroup_getset_retval_hooks__load(skel);
		474	ASSERT_EQ(err, exposed_hooks[i].expected_err, "expected_err");
		475
		476	close_skel:
		477	cgroup_getset_retval_hooks__destroy(skel);
		478	}
		479	}
		480
436	void test_cgroup_getset_retval(void)	481	void test_cgroup_getset_retval(void)
437	{	482	{
438	int cgroup_fd = -1;	483	int cgroup_fd = -1;
@@ -476,6 +521,9 @@ void test_cgroup_getset_retval(void) · 37 unmodified lines
476	if (test__start_subtest("getsockopt-retval_sync"))	521	if (test__start_subtest("getsockopt-retval_sync"))
477	test_getsockopt_retval_sync(cgroup_fd, sock_fd);	522	test_getsockopt_retval_sync(cgroup_fd, sock_fd);
478		523
		524	if (test__start_subtest("exposed_hooks"))
		525	test_exposed_hooks(cgroup_fd, sock_fd);
		526
479	close_fd:	527	close_fd:
480	close(cgroup_fd);	528	close(cgroup_fd);
481	}	529	}

tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c+357−0

@@ -0,0 +1,357 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Functions to manage eBPF programs attached to cgroup subsystems
	4	*
	5	* Copyright 2022 Google LLC.
	6	*/
	7	#include <asm-generic/errno.h>
	8	#include <errno.h>
	9	#include <sys/types.h>
	10	#include <sys/mount.h>
	11	#include <sys/stat.h>
	12	#include <unistd.h>
	13
	14	#include <test_progs.h>
	15	#include <bpf/libbpf.h>
	16	#include <bpf/bpf.h>
	17
	18	#include "cgroup_helpers.h"
	19	#include "cgroup_hierarchical_stats.skel.h"
	20
	21	#define PAGE_SIZE 4096
	22	#define MB(x) (x << 20)
	23
	24	#define BPFFS_ROOT "/sys/fs/bpf/"
	25	#define BPFFS_VMSCAN BPFFS_ROOT"vmscan/"
	26
	27	#define CG_ROOT_NAME "root"
	28	#define CG_ROOT_ID 1
	29
	30	#define CGROUP_PATH(p, n) {.path = p"/"n, .name = n}
	31
	32	static struct {
	33	const char path, name;
	34	unsigned long long id;
	35	int fd;
	36	} cgroups[] = {
	37	CGROUP_PATH("/", "test"),
	38	CGROUP_PATH("/test", "child1"),
	39	CGROUP_PATH("/test", "child2"),
	40	CGROUP_PATH("/test/child1", "child1_1"),
	41	CGROUP_PATH("/test/child1", "child1_2"),
	42	CGROUP_PATH("/test/child2", "child2_1"),
	43	CGROUP_PATH("/test/child2", "child2_2"),
	44	};
	45
	46	#define N_CGROUPS ARRAY_SIZE(cgroups)
	47	#define N_NON_LEAF_CGROUPS 3
	48
	49	static int root_cgroup_fd;
	50	static bool mounted_bpffs;
	51
	52	/* reads file at 'path' to 'buf', returns 0 on success. */
	53	static int read_from_file(const char path, char buf, size_t size)
	54	{
	55	int fd, len;
	56
	57	fd = open(path, O_RDONLY);
	58	if (fd < 0)
	59	return fd;
	60
	61	len = read(fd, buf, size);
	62	close(fd);
	63	if (len < 0)
	64	return len;
	65
	66	buf[len] = 0;
	67	return 0;
	68	}
	69
	70	/* mounts bpffs and mkdir for reading stats, returns 0 on success. */
	71	static int setup_bpffs(void)
	72	{
	73	int err;
	74
	75	/* Mount bpffs */
	76	err = mount("bpf", BPFFS_ROOT, "bpf", 0, NULL);
	77	mounted_bpffs = !err;
	78	if (ASSERT_FALSE(err && errno != EBUSY, "mount"))
	79	return err;
	80
	81	/* Create a directory to contain stat files in bpffs */
	82	err = mkdir(BPFFS_VMSCAN, 0755);
	83	if (!ASSERT_OK(err, "mkdir"))
	84	return err;
	85
	86	return 0;
	87	}
	88
	89	static void cleanup_bpffs(void)
	90	{
	91	/* Remove created directory in bpffs */
	92	ASSERT_OK(rmdir(BPFFS_VMSCAN), "rmdir "BPFFS_VMSCAN);
	93
	94	/* Unmount bpffs, if it wasn't already mounted when we started */
	95	if (mounted_bpffs)
	96	return;
	97
	98	ASSERT_OK(umount(BPFFS_ROOT), "unmount bpffs");
	99	}
	100
	101	/* sets up cgroups, returns 0 on success. */
	102	static int setup_cgroups(void)
	103	{
	104	int i, fd, err;
	105
	106	err = setup_cgroup_environment();
	107	if (!ASSERT_OK(err, "setup_cgroup_environment"))
	108	return err;
	109
	110	root_cgroup_fd = get_root_cgroup();
	111	if (!ASSERT_GE(root_cgroup_fd, 0, "get_root_cgroup"))
	112	return root_cgroup_fd;
	113
	114	for (i = 0; i < N_CGROUPS; i++) {
	115	fd = create_and_get_cgroup(cgroups[i].path);
	116	if (!ASSERT_GE(fd, 0, "create_and_get_cgroup"))
	117	return fd;
	118
	119	cgroups[i].fd = fd;
	120	cgroups[i].id = get_cgroup_id(cgroups[i].path);
	121
	122	/*
	123	* Enable memcg controller for the entire hierarchy.
	124	* Note that stats are collected for all cgroups in a hierarchy
	125	* with memcg enabled anyway, but are only exposed for cgroups
	126	* that have memcg enabled.
	127	*/
	128	if (i < N_NON_LEAF_CGROUPS) {
	129	err = enable_controllers(cgroups[i].path, "memory");
	130	if (!ASSERT_OK(err, "enable_controllers"))
	131	return err;
	132	}
	133	}
	134	return 0;
	135	}
	136
	137	static void cleanup_cgroups(void)
	138	{
	139	close(root_cgroup_fd);
	140	for (int i = 0; i < N_CGROUPS; i++)
	141	close(cgroups[i].fd);
	142	cleanup_cgroup_environment();
	143	}
	144
	145	/* Sets up cgroup hiearchary, returns 0 on success. */
	146	static int setup_hierarchy(void)
	147	{
	148	return setup_bpffs() \|\| setup_cgroups();
	149	}
	150
	151	static void destroy_hierarchy(void)
	152	{
	153	cleanup_cgroups();
	154	cleanup_bpffs();
	155	}
	156
	157	static int reclaimer(const char *cgroup_path, size_t size)
	158	{
	159	static char size_buf[128];
	160	char buf, ptr;
	161	int err;
	162
	163	/* Join cgroup in the parent process workdir */
	164	if (join_parent_cgroup(cgroup_path))
	165	return EACCES;
	166
	167	/* Allocate memory */
	168	buf = malloc(size);
	169	if (!buf)
	170	return ENOMEM;
	171
	172	/* Write to memory to make sure it's actually allocated */
	173	for (ptr = buf; ptr < buf + size; ptr += PAGE_SIZE)
	174	*ptr = 1;
	175
	176	/* Try to reclaim memory */
	177	snprintf(size_buf, 128, "%lu", size);
	178	err = write_cgroup_file_parent(cgroup_path, "memory.reclaim", size_buf);
	179
	180	free(buf);
	181	/* memory.reclaim returns EAGAIN if the amount is not fully reclaimed */
	182	if (err && errno != EAGAIN)
	183	return errno;
	184
	185	return 0;
	186	}
	187
	188	static int induce_vmscan(void)
	189	{
	190	int i, status;
	191
	192	/*
	193	* In every leaf cgroup, run a child process that allocates some memory
	194	* and attempts to reclaim some of it.
	195	*/
	196	for (i = N_NON_LEAF_CGROUPS; i < N_CGROUPS; i++) {
	197	pid_t pid;
	198
	199	/* Create reclaimer child */
	200	pid = fork();
	201	if (pid == 0) {
	202	status = reclaimer(cgroups[i].path, MB(5));
	203	exit(status);
	204	}
	205
	206	/* Cleanup reclaimer child */
	207	waitpid(pid, &status, 0);
	208	ASSERT_TRUE(WIFEXITED(status), "reclaimer exited");
	209	ASSERT_EQ(WEXITSTATUS(status), 0, "reclaim exit code");
	210	}
	211	return 0;
	212	}
	213
	214	static unsigned long long
	215	get_cgroup_vmscan_delay(unsigned long long cgroup_id, const char *file_name)
	216	{
	217	unsigned long long vmscan = 0, id = 0;
	218	static char buf[128], path[128];
	219
	220	/* For every cgroup, read the file generated by cgroup_iter */
	221	snprintf(path, 128, "%s%s", BPFFS_VMSCAN, file_name);
	222	if (!ASSERT_OK(read_from_file(path, buf, 128), "read cgroup_iter"))
	223	return 0;
	224
	225	/* Check the output file formatting */
	226	ASSERT_EQ(sscanf(buf, "cg_id: %llu, total_vmscan_delay: %llu\n",
	227	&id, &vmscan), 2, "output format");
	228
	229	/* Check that the cgroup_id is displayed correctly */
	230	ASSERT_EQ(id, cgroup_id, "cgroup_id");
	231	/* Check that the vmscan reading is non-zero */
	232	ASSERT_GT(vmscan, 0, "vmscan_reading");
	233	return vmscan;
	234	}
	235
	236	static void check_vmscan_stats(void)
	237	{
	238	unsigned long long vmscan_readings[N_CGROUPS], vmscan_root;
	239	int i;
	240
	241	for (i = 0; i < N_CGROUPS; i++) {
	242	vmscan_readings[i] = get_cgroup_vmscan_delay(cgroups[i].id,
	243	cgroups[i].name);
	244	}
	245
	246	/* Read stats for root too */
	247	vmscan_root = get_cgroup_vmscan_delay(CG_ROOT_ID, CG_ROOT_NAME);
	248
	249	/* Check that child1 == child1_1 + child1_2 */
	250	ASSERT_EQ(vmscan_readings[1], vmscan_readings[3] + vmscan_readings[4],
	251	"child1_vmscan");
	252	/* Check that child2 == child2_1 + child2_2 */
	253	ASSERT_EQ(vmscan_readings[2], vmscan_readings[5] + vmscan_readings[6],
	254	"child2_vmscan");
	255	/* Check that test == child1 + child2 */
	256	ASSERT_EQ(vmscan_readings[0], vmscan_readings[1] + vmscan_readings[2],
	257	"test_vmscan");
	258	/* Check that root >= test */
	259	ASSERT_GE(vmscan_root, vmscan_readings[1], "root_vmscan");
	260	}
	261
	262	/* Creates iter link and pins in bpffs, returns 0 on success, -errno on failure.
	263	*/
	264	static int setup_cgroup_iter(struct cgroup_hierarchical_stats *obj,
	265	int cgroup_fd, const char *file_name)
	266	{
	267	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	268	union bpf_iter_link_info linfo = {};
	269	struct bpf_link *link;
	270	static char path[128];
	271	int err;
	272
	273	/*
	274	* Create an iter link, parameterized by cgroup_fd. We only want to
	275	* traverse one cgroup, so set the traversal order to "self".
	276	*/
	277	linfo.cgroup.cgroup_fd = cgroup_fd;
	278	linfo.cgroup.order = BPF_CGROUP_ITER_SELF_ONLY;
	279	opts.link_info = &linfo;
	280	opts.link_info_len = sizeof(linfo);
	281	link = bpf_program__attach_iter(obj->progs.dump_vmscan, &opts);
	282	if (!ASSERT_OK_PTR(link, "attach_iter"))
	283	return -EFAULT;
	284
	285	/* Pin the link to a bpffs file */
	286	snprintf(path, 128, "%s%s", BPFFS_VMSCAN, file_name);
	287	err = bpf_link__pin(link, path);
	288	ASSERT_OK(err, "pin cgroup_iter");
	289
	290	/* Remove the link, leaving only the ref held by the pinned file */
	291	bpf_link__destroy(link);
	292	return err;
	293	}
	294
	295	/* Sets up programs for collecting stats, returns 0 on success. */
	296	static int setup_progs(struct cgroup_hierarchical_stats **skel)
	297	{
	298	int i, err;
	299
	300	*skel = cgroup_hierarchical_stats__open_and_load();
	301	if (!ASSERT_OK_PTR(*skel, "open_and_load"))
	302	return 1;
	303
	304	/* Attach cgroup_iter program that will dump the stats to cgroups */
	305	for (i = 0; i < N_CGROUPS; i++) {
	306	err = setup_cgroup_iter(*skel, cgroups[i].fd, cgroups[i].name);
	307	if (!ASSERT_OK(err, "setup_cgroup_iter"))
	308	return err;
	309	}
	310
	311	/* Also dump stats for root */
	312	err = setup_cgroup_iter(*skel, root_cgroup_fd, CG_ROOT_NAME);
	313	if (!ASSERT_OK(err, "setup_cgroup_iter"))
	314	return err;
	315
	316	bpf_program__set_autoattach((*skel)->progs.dump_vmscan, false);
	317	err = cgroup_hierarchical_stats__attach(*skel);
	318	if (!ASSERT_OK(err, "attach"))
	319	return err;
	320
	321	return 0;
	322	}
	323
	324	static void destroy_progs(struct cgroup_hierarchical_stats *skel)
	325	{
	326	static char path[128];
	327	int i;
	328
	329	for (i = 0; i < N_CGROUPS; i++) {
	330	/* Delete files in bpffs that cgroup_iters are pinned in */
	331	snprintf(path, 128, "%s%s", BPFFS_VMSCAN,
	332	cgroups[i].name);
	333	ASSERT_OK(remove(path), "remove cgroup_iter pin");
	334	}
	335
	336	/* Delete root file in bpffs */
	337	snprintf(path, 128, "%s%s", BPFFS_VMSCAN, CG_ROOT_NAME);
	338	ASSERT_OK(remove(path), "remove cgroup_iter root pin");
	339	cgroup_hierarchical_stats__destroy(skel);
	340	}
	341
	342	void test_cgroup_hierarchical_stats(void)
	343	{
	344	struct cgroup_hierarchical_stats *skel = NULL;
	345
	346	if (setup_hierarchy())
	347	goto hierarchy_cleanup;
	348	if (setup_progs(&skel))
	349	goto cleanup;
	350	if (induce_vmscan())
	351	goto cleanup;
	352	check_vmscan_stats();
	353	cleanup:
	354	destroy_progs(skel);
	355	hierarchy_cleanup:
	356	destroy_hierarchy();
	357	}

tools/testing/selftests/bpf/prog_tests/cgroup_iter.c+224−0

@@ -0,0 +1,224 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2022 Google */
	3
	4	#include <test_progs.h>
	5	#include <bpf/libbpf.h>
	6	#include <bpf/btf.h>
	7	#include "cgroup_iter.skel.h"
	8	#include "cgroup_helpers.h"
	9
	10	#define ROOT 0
	11	#define PARENT 1
	12	#define CHILD1 2
	13	#define CHILD2 3
	14	#define NUM_CGROUPS 4
	15
	16	#define PROLOGUE "prologue\n"
	17	#define EPILOGUE "epilogue\n"
	18
	19	static const char *cg_path[] = {
	20	"/", "/parent", "/parent/child1", "/parent/child2"
	21	};
	22
	23	static int cg_fd[] = {-1, -1, -1, -1};
	24	static unsigned long long cg_id[] = {0, 0, 0, 0};
	25	static char expected_output[64];
	26
	27	static int setup_cgroups(void)
	28	{
	29	int fd, i = 0;
	30
	31	for (i = 0; i < NUM_CGROUPS; i++) {
	32	fd = create_and_get_cgroup(cg_path[i]);
	33	if (fd < 0)
	34	return fd;
	35
	36	cg_fd[i] = fd;
	37	cg_id[i] = get_cgroup_id(cg_path[i]);
	38	}
	39	return 0;
	40	}
	41
	42	static void cleanup_cgroups(void)
	43	{
	44	int i;
	45
	46	for (i = 0; i < NUM_CGROUPS; i++)
	47	close(cg_fd[i]);
	48	}
	49
	50	static void read_from_cgroup_iter(struct bpf_program *prog, int cgroup_fd,
	51	int order, const char *testname)
	52	{
	53	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	54	union bpf_iter_link_info linfo;
	55	struct bpf_link *link;
	56	int len, iter_fd;
	57	static char buf[128];
	58	size_t left;
	59	char *p;
	60
	61	memset(&linfo, 0, sizeof(linfo));
	62	linfo.cgroup.cgroup_fd = cgroup_fd;
	63	linfo.cgroup.order = order;
	64	opts.link_info = &linfo;
	65	opts.link_info_len = sizeof(linfo);
	66
	67	link = bpf_program__attach_iter(prog, &opts);
	68	if (!ASSERT_OK_PTR(link, "attach_iter"))
	69	return;
	70
	71	iter_fd = bpf_iter_create(bpf_link__fd(link));
	72	if (iter_fd < 0)
	73	goto free_link;
	74
	75	memset(buf, 0, sizeof(buf));
	76	left = ARRAY_SIZE(buf);
	77	p = buf;
	78	while ((len = read(iter_fd, p, left)) > 0) {
	79	p += len;
	80	left -= len;
	81	}
	82
	83	ASSERT_STREQ(buf, expected_output, testname);
	84
	85	/* read() after iter finishes should be ok. */
	86	if (len == 0)
	87	ASSERT_OK(read(iter_fd, buf, sizeof(buf)), "second_read");
	88
	89	close(iter_fd);
	90	free_link:
	91	bpf_link__destroy(link);
	92	}
	93
	94	/* Invalid cgroup. */
	95	static void test_invalid_cgroup(struct cgroup_iter *skel)
	96	{
	97	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	98	union bpf_iter_link_info linfo;
	99	struct bpf_link *link;
	100
	101	memset(&linfo, 0, sizeof(linfo));
	102	linfo.cgroup.cgroup_fd = (__u32)-1;
	103	opts.link_info = &linfo;
	104	opts.link_info_len = sizeof(linfo);
	105
	106	link = bpf_program__attach_iter(skel->progs.cgroup_id_printer, &opts);
	107	ASSERT_ERR_PTR(link, "attach_iter");
	108	bpf_link__destroy(link);
	109	}
	110
	111	/* Specifying both cgroup_fd and cgroup_id is invalid. */
	112	static void test_invalid_cgroup_spec(struct cgroup_iter *skel)
	113	{
	114	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
	115	union bpf_iter_link_info linfo;
	116	struct bpf_link *link;
	117
	118	memset(&linfo, 0, sizeof(linfo));
	119	linfo.cgroup.cgroup_fd = (__u32)cg_fd[PARENT];
	120	linfo.cgroup.cgroup_id = (__u64)cg_id[PARENT];
	121	opts.link_info = &linfo;
	122	opts.link_info_len = sizeof(linfo);
	123
	124	link = bpf_program__attach_iter(skel->progs.cgroup_id_printer, &opts);
	125	ASSERT_ERR_PTR(link, "attach_iter");
	126	bpf_link__destroy(link);
	127	}
	128
	129	/* Preorder walk prints parent and child in order. */
	130	static void test_walk_preorder(struct cgroup_iter *skel)
	131	{
	132	snprintf(expected_output, sizeof(expected_output),
	133	PROLOGUE "%8llu\n%8llu\n%8llu\n" EPILOGUE,
	134	cg_id[PARENT], cg_id[CHILD1], cg_id[CHILD2]);
	135
	136	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
	137	BPF_CGROUP_ITER_DESCENDANTS_PRE, "preorder");
	138	}
	139
	140	/* Postorder walk prints child and parent in order. */
	141	static void test_walk_postorder(struct cgroup_iter *skel)
	142	{
	143	snprintf(expected_output, sizeof(expected_output),
	144	PROLOGUE "%8llu\n%8llu\n%8llu\n" EPILOGUE,
	145	cg_id[CHILD1], cg_id[CHILD2], cg_id[PARENT]);
	146
	147	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
	148	BPF_CGROUP_ITER_DESCENDANTS_POST, "postorder");
	149	}
	150
	151	/* Walking parents prints parent and then root. */
	152	static void test_walk_ancestors_up(struct cgroup_iter *skel)
	153	{
	154	/* terminate the walk when ROOT is met. */
	155	skel->bss->terminal_cgroup = cg_id[ROOT];
	156
	157	snprintf(expected_output, sizeof(expected_output),
	158	PROLOGUE "%8llu\n%8llu\n" EPILOGUE,
	159	cg_id[PARENT], cg_id[ROOT]);
	160
	161	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
	162	BPF_CGROUP_ITER_ANCESTORS_UP, "ancestors_up");
	163
	164	skel->bss->terminal_cgroup = 0;
	165	}
	166
	167	/* Early termination prints parent only. */
	168	static void test_early_termination(struct cgroup_iter *skel)
	169	{
	170	/* terminate the walk after the first element is processed. */
	171	skel->bss->terminate_early = 1;
	172
	173	snprintf(expected_output, sizeof(expected_output),
	174	PROLOGUE "%8llu\n" EPILOGUE, cg_id[PARENT]);
	175
	176	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
	177	BPF_CGROUP_ITER_DESCENDANTS_PRE, "early_termination");
	178
	179	skel->bss->terminate_early = 0;
	180	}
	181
	182	/* Waling self prints self only. */
	183	static void test_walk_self_only(struct cgroup_iter *skel)
	184	{
	185	snprintf(expected_output, sizeof(expected_output),
	186	PROLOGUE "%8llu\n" EPILOGUE, cg_id[PARENT]);
	187
	188	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
	189	BPF_CGROUP_ITER_SELF_ONLY, "self_only");
	190	}
	191
	192	void test_cgroup_iter(void)
	193	{
	194	struct cgroup_iter *skel = NULL;
	195
	196	if (setup_cgroup_environment())
	197	return;
	198
	199	if (setup_cgroups())
	200	goto out;
	201
	202	skel = cgroup_iter__open_and_load();
	203	if (!ASSERT_OK_PTR(skel, "cgroup_iter__open_and_load"))
	204	goto out;
	205
	206	if (test__start_subtest("cgroup_iter__invalid_cgroup"))
	207	test_invalid_cgroup(skel);
	208	if (test__start_subtest("cgroup_iter__invalid_cgroup_spec"))
	209	test_invalid_cgroup_spec(skel);
	210	if (test__start_subtest("cgroup_iter__preorder"))
	211	test_walk_preorder(skel);
	212	if (test__start_subtest("cgroup_iter__postorder"))
	213	test_walk_postorder(skel);
	214	if (test__start_subtest("cgroup_iter__ancestors_up_walk"))
	215	test_walk_ancestors_up(skel);
	216	if (test__start_subtest("cgroup_iter__early_termination"))
	217	test_early_termination(skel);
	218	if (test__start_subtest("cgroup_iter__self_only"))
	219	test_walk_self_only(skel);
	220	out:
	221	cgroup_iter__destroy(skel);
	222	cleanup_cgroups();
	223	cleanup_cgroup_environment();
	224	}

tools/testing/selftests/bpf/prog_tests/connect_force_port.c+1−1

@@ -53,7 +53,7 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type) · 52 unmodified lines
53	__u16 expected_peer_port = 60000;	53	__u16 expected_peer_port = 60000;
54	struct bpf_program *prog;	54	struct bpf_program *prog;
55	struct bpf_object *obj;	55	struct bpf_object *obj;
56	const char *obj_file = v4 ? "connect_force_port4.o" : "connect_force_port6.o";	56	const char *obj_file = v4 ? "connect_force_port4.bpf.o" : "connect_force_port6.bpf.o";
57	int fd, err;	57	int fd, err;
58	__u32 duration = 0;	58	__u32 duration = 0;
59		59

tools/testing/selftests/bpf/prog_tests/core_reloc.c+37−37

@@ -13,7 +13,7 @@ static int duration = 0; · 12 unmodified lines
13		13
14	#define MODULES_CASE(name, pg_name, tp_name) { \	14	#define MODULES_CASE(name, pg_name, tp_name) { \
15	.case_name = name, \	15	.case_name = name, \
16	.bpf_obj_file = "test_core_reloc_module.o", \	16	.bpf_obj_file = "test_core_reloc_module.bpf.o", \
17	.btf_src_file = NULL, /* find in kernel module BTFs */ \	17	.btf_src_file = NULL, /* find in kernel module BTFs */ \
18	.input = "", \	18	.input = "", \
19	.input_len = 0, \	19	.input_len = 0, \
@@ -43,8 +43,8 @@ static int duration = 0; · 23 unmodified lines
43		43
44	#define FLAVORS_CASE_COMMON(name) \	44	#define FLAVORS_CASE_COMMON(name) \
45	.case_name = #name, \	45	.case_name = #name, \
46	.bpf_obj_file = "test_core_reloc_flavors.o", \	46	.bpf_obj_file = "test_core_reloc_flavors.bpf.o", \
47	.btf_src_file = "btf__core_reloc_" #name ".o", \	47	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
48	.raw_tp_name = "sys_enter", \	48	.raw_tp_name = "sys_enter", \
49	.prog_name = "test_core_flavors" \	49	.prog_name = "test_core_flavors" \
50		50
@@ -68,8 +68,8 @@ static int duration = 0; · 17 unmodified lines
68		68
69	#define NESTING_CASE_COMMON(name) \	69	#define NESTING_CASE_COMMON(name) \
70	.case_name = #name, \	70	.case_name = #name, \
71	.bpf_obj_file = "test_core_reloc_nesting.o", \	71	.bpf_obj_file = "test_core_reloc_nesting.bpf.o", \
72	.btf_src_file = "btf__core_reloc_" #name ".o", \	72	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
73	.raw_tp_name = "sys_enter", \	73	.raw_tp_name = "sys_enter", \
74	.prog_name = "test_core_nesting" \	74	.prog_name = "test_core_nesting" \
75		75
@@ -96,8 +96,8 @@ static int duration = 0; · 20 unmodified lines
96		96
97	#define ARRAYS_CASE_COMMON(name) \	97	#define ARRAYS_CASE_COMMON(name) \
98	.case_name = #name, \	98	.case_name = #name, \
99	.bpf_obj_file = "test_core_reloc_arrays.o", \	99	.bpf_obj_file = "test_core_reloc_arrays.bpf.o", \
100	.btf_src_file = "btf__core_reloc_" #name ".o", \	100	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
101	.raw_tp_name = "sys_enter", \	101	.raw_tp_name = "sys_enter", \
102	.prog_name = "test_core_arrays" \	102	.prog_name = "test_core_arrays" \
103		103
@@ -130,8 +130,8 @@ static int duration = 0; · 26 unmodified lines
130		130
131	#define PRIMITIVES_CASE_COMMON(name) \	131	#define PRIMITIVES_CASE_COMMON(name) \
132	.case_name = #name, \	132	.case_name = #name, \
133	.bpf_obj_file = "test_core_reloc_primitives.o", \	133	.bpf_obj_file = "test_core_reloc_primitives.bpf.o", \
134	.btf_src_file = "btf__core_reloc_" #name ".o", \	134	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
135	.raw_tp_name = "sys_enter", \	135	.raw_tp_name = "sys_enter", \
136	.prog_name = "test_core_primitives" \	136	.prog_name = "test_core_primitives" \
137		137
@@ -150,8 +150,8 @@ static int duration = 0; · 12 unmodified lines
150		150
151	#define MODS_CASE(name) { \	151	#define MODS_CASE(name) { \
152	.case_name = #name, \	152	.case_name = #name, \
153	.bpf_obj_file = "test_core_reloc_mods.o", \	153	.bpf_obj_file = "test_core_reloc_mods.bpf.o", \
154	.btf_src_file = "btf__core_reloc_" #name ".o", \	154	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
155	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) { \	155	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) { \
156	.a = 1, \	156	.a = 1, \
157	.b = 2, \	157	.b = 2, \
@@ -174,8 +174,8 @@ static int duration = 0; · 16 unmodified lines
174		174
175	#define PTR_AS_ARR_CASE(name) { \	175	#define PTR_AS_ARR_CASE(name) { \
176	.case_name = #name, \	176	.case_name = #name, \
177	.bpf_obj_file = "test_core_reloc_ptr_as_arr.o", \	177	.bpf_obj_file = "test_core_reloc_ptr_as_arr.bpf.o", \
178	.btf_src_file = "btf__core_reloc_" #name ".o", \	178	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
179	.input = (const char *)&(struct core_reloc_##name []){ \	179	.input = (const char *)&(struct core_reloc_##name []){ \
180	{ .a = 1 }, \	180	{ .a = 1 }, \
181	{ .a = 2 }, \	181	{ .a = 2 }, \
@@ -203,8 +203,8 @@ static int duration = 0; · 21 unmodified lines
203		203
204	#define INTS_CASE_COMMON(name) \	204	#define INTS_CASE_COMMON(name) \
205	.case_name = #name, \	205	.case_name = #name, \
206	.bpf_obj_file = "test_core_reloc_ints.o", \	206	.bpf_obj_file = "test_core_reloc_ints.bpf.o", \
207	.btf_src_file = "btf__core_reloc_" #name ".o", \	207	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
208	.raw_tp_name = "sys_enter", \	208	.raw_tp_name = "sys_enter", \
209	.prog_name = "test_core_ints"	209	.prog_name = "test_core_ints"
210		210
@@ -223,18 +223,18 @@ static int duration = 0; · 12 unmodified lines
223		223
224	#define FIELD_EXISTS_CASE_COMMON(name) \	224	#define FIELD_EXISTS_CASE_COMMON(name) \
225	.case_name = #name, \	225	.case_name = #name, \
226	.bpf_obj_file = "test_core_reloc_existence.o", \	226	.bpf_obj_file = "test_core_reloc_existence.bpf.o", \
227	.btf_src_file = "btf__core_reloc_" #name ".o", \	227	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
228	.raw_tp_name = "sys_enter", \	228	.raw_tp_name = "sys_enter", \
229	.prog_name = "test_core_existence"	229	.prog_name = "test_core_existence"
230		230
231	#define BITFIELDS_CASE_COMMON(objfile, test_name_prefix, name) \	231	#define BITFIELDS_CASE_COMMON(objfile, test_name_prefix, name) \
232	.case_name = test_name_prefix#name, \	232	.case_name = test_name_prefix#name, \
233	.bpf_obj_file = objfile, \	233	.bpf_obj_file = objfile, \
234	.btf_src_file = "btf__core_reloc_" #name ".o"	234	.btf_src_file = "btf__core_reloc_" #name ".bpf.o"
235		235
236	#define BITFIELDS_CASE(name, ...) { \	236	#define BITFIELDS_CASE(name, ...) { \
237	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \	237	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.bpf.o", \
238	"probed:", name), \	238	"probed:", name), \
239	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \	239	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \
240	.input_len = sizeof(struct core_reloc_##name), \	240	.input_len = sizeof(struct core_reloc_##name), \
@@ -244,7 +244,7 @@ static int duration = 0; · 3 unmodified lines
244	.raw_tp_name = "sys_enter", \	244	.raw_tp_name = "sys_enter", \
245	.prog_name = "test_core_bitfields", \	245	.prog_name = "test_core_bitfields", \
246	}, { \	246	}, { \
247	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \	247	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.bpf.o", \
248	"direct:", name), \	248	"direct:", name), \
249	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \	249	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__, \
250	.input_len = sizeof(struct core_reloc_##name), \	250	.input_len = sizeof(struct core_reloc_##name), \
@@ -256,14 +256,14 @@ static int duration = 0; · 5 unmodified lines
256		256
257		257
258	#define BITFIELDS_ERR_CASE(name) { \	258	#define BITFIELDS_ERR_CASE(name) { \
259	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \	259	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.bpf.o", \
260	"probed:", name), \	260	"probed:", name), \
261	.fails = true, \	261	.fails = true, \
262	.run_btfgen_fails = true, \	262	.run_btfgen_fails = true, \
263	.raw_tp_name = "sys_enter", \	263	.raw_tp_name = "sys_enter", \
264	.prog_name = "test_core_bitfields", \	264	.prog_name = "test_core_bitfields", \
265	}, { \	265	}, { \
266	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \	266	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.bpf.o", \
267	"direct:", name), \	267	"direct:", name), \
268	.fails = true, \	268	.fails = true, \
269	.run_btfgen_fails = true, \	269	.run_btfgen_fails = true, \
@@ -272,8 +272,8 @@ static int duration = 0; · 2 unmodified lines
272		272
273	#define SIZE_CASE_COMMON(name) \	273	#define SIZE_CASE_COMMON(name) \
274	.case_name = #name, \	274	.case_name = #name, \
275	.bpf_obj_file = "test_core_reloc_size.o", \	275	.bpf_obj_file = "test_core_reloc_size.bpf.o", \
276	.btf_src_file = "btf__core_reloc_" #name ".o", \	276	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
277	.raw_tp_name = "sys_enter", \	277	.raw_tp_name = "sys_enter", \
278	.prog_name = "test_core_size"	278	.prog_name = "test_core_size"
279		279
@@ -307,13 +307,13 @@ static int duration = 0; · 27 unmodified lines
307	#define SIZE_ERR_CASE(name) { \	307	#define SIZE_ERR_CASE(name) { \
308	SIZE_CASE_COMMON(name), \	308	SIZE_CASE_COMMON(name), \
309	.fails = true, \	309	.fails = true, \
310	.run_btfgen_fails = true, \	310	.run_btfgen_fails = true, \
311	}	311	}
312		312
313	#define TYPE_BASED_CASE_COMMON(name) \	313	#define TYPE_BASED_CASE_COMMON(name) \
314	.case_name = #name, \	314	.case_name = #name, \
315	.bpf_obj_file = "test_core_reloc_type_based.o", \	315	.bpf_obj_file = "test_core_reloc_type_based.bpf.o", \
316	.btf_src_file = "btf__core_reloc_" #name ".o", \	316	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
317	.raw_tp_name = "sys_enter", \	317	.raw_tp_name = "sys_enter", \
318	.prog_name = "test_core_type_based"	318	.prog_name = "test_core_type_based"
319		319
@@ -331,8 +331,8 @@ static int duration = 0; · 11 unmodified lines
331		331
332	#define TYPE_ID_CASE_COMMON(name) \	332	#define TYPE_ID_CASE_COMMON(name) \
333	.case_name = #name, \	333	.case_name = #name, \
334	.bpf_obj_file = "test_core_reloc_type_id.o", \	334	.bpf_obj_file = "test_core_reloc_type_id.bpf.o", \
335	.btf_src_file = "btf__core_reloc_" #name ".o", \	335	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
336	.raw_tp_name = "sys_enter", \	336	.raw_tp_name = "sys_enter", \
337	.prog_name = "test_core_type_id"	337	.prog_name = "test_core_type_id"
338		338
@@ -350,8 +350,8 @@ static int duration = 0; · 11 unmodified lines
350		350
351	#define ENUMVAL_CASE_COMMON(name) \	351	#define ENUMVAL_CASE_COMMON(name) \
352	.case_name = #name, \	352	.case_name = #name, \
353	.bpf_obj_file = "test_core_reloc_enumval.o", \	353	.bpf_obj_file = "test_core_reloc_enumval.bpf.o", \
354	.btf_src_file = "btf__core_reloc_" #name ".o", \	354	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
355	.raw_tp_name = "sys_enter", \	355	.raw_tp_name = "sys_enter", \
356	.prog_name = "test_core_enumval"	356	.prog_name = "test_core_enumval"
357		357
@@ -369,8 +369,8 @@ static int duration = 0; · 11 unmodified lines
369		369
370	#define ENUM64VAL_CASE_COMMON(name) \	370	#define ENUM64VAL_CASE_COMMON(name) \
371	.case_name = #name, \	371	.case_name = #name, \
372	.bpf_obj_file = "test_core_reloc_enum64val.o", \	372	.bpf_obj_file = "test_core_reloc_enum64val.bpf.o", \
373	.btf_src_file = "btf__core_reloc_" #name ".o", \	373	.btf_src_file = "btf__core_reloc_" #name ".bpf.o", \
374	.raw_tp_name = "sys_enter", \	374	.raw_tp_name = "sys_enter", \
375	.prog_name = "test_core_enum64val"	375	.prog_name = "test_core_enum64val"
376		376
@@ -547,7 +547,7 @@ static const struct core_reloc_test_case test_cases[] = { · 170 unmodified lines
547	/* validate we can find kernel image and use its BTF for relocs */	547	/* validate we can find kernel image and use its BTF for relocs */
548	{	548	{
549	.case_name = "kernel",	549	.case_name = "kernel",
550	.bpf_obj_file = "test_core_reloc_kernel.o",	550	.bpf_obj_file = "test_core_reloc_kernel.bpf.o",
551	.btf_src_file = NULL, /* load from /lib/modules/$(uname -r) */	551	.btf_src_file = NULL, /* load from /lib/modules/$(uname -r) */
552	.input = "",	552	.input = "",
553	.input_len = 0,	553	.input_len = 0,
@@ -629,8 +629,8 @@ static const struct core_reloc_test_case test_cases[] = { · 75 unmodified lines
629	/* validate edge cases of capturing relocations */	629	/* validate edge cases of capturing relocations */
630	{	630	{
631	.case_name = "misc",	631	.case_name = "misc",
632	.bpf_obj_file = "test_core_reloc_misc.o",	632	.bpf_obj_file = "test_core_reloc_misc.bpf.o",
633	.btf_src_file = "btf__core_reloc_misc.o",	633	.btf_src_file = "btf__core_reloc_misc.bpf.o",
634	.input = (const char *)&(struct core_reloc_misc_extensible[]){	634	.input = (const char *)&(struct core_reloc_misc_extensible[]){
635	{ .a = 1 },	635	{ .a = 1 },
636	{ .a = 2 }, /* not read */	636	{ .a = 2 }, /* not read */

tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c+22−22

@@ -174,8 +174,8 @@ static void test_target_no_callees(void) · 173 unmodified lines
174	const char *prog_name[] = {	174	const char *prog_name[] = {
175	"fexit/test_pkt_md_access",	175	"fexit/test_pkt_md_access",
176	};	176	};
177	test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.o",	177	test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.bpf.o",
178	"./test_pkt_md_access.o",	178	"./test_pkt_md_access.bpf.o",
179	ARRAY_SIZE(prog_name),	179	ARRAY_SIZE(prog_name),
180	prog_name, true, NULL);	180	prog_name, true, NULL);
181	}	181	}
@@ -188,8 +188,8 @@ static void test_target_yes_callees(void) · 6 unmodified lines
188	"fexit/test_pkt_access_subprog2",	188	"fexit/test_pkt_access_subprog2",
189	"fexit/test_pkt_access_subprog3",	189	"fexit/test_pkt_access_subprog3",
190	};	190	};
191	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",	191	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.bpf.o",
192	"./test_pkt_access.o",	192	"./test_pkt_access.bpf.o",
193	ARRAY_SIZE(prog_name),	193	ARRAY_SIZE(prog_name),
194	prog_name, true, NULL);	194	prog_name, true, NULL);
195	}	195	}
@@ -206,8 +206,8 @@ static void test_func_replace(void) · 10 unmodified lines
206	"freplace/get_constant",	206	"freplace/get_constant",
207	"freplace/test_pkt_write_access_subprog",	207	"freplace/test_pkt_write_access_subprog",
208	};	208	};
209	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",	209	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.bpf.o",
210	"./test_pkt_access.o",	210	"./test_pkt_access.bpf.o",
211	ARRAY_SIZE(prog_name),	211	ARRAY_SIZE(prog_name),
212	prog_name, true, NULL);	212	prog_name, true, NULL);
213	}	213	}
@@ -217,8 +217,8 @@ static void test_func_replace_verify(void) · 3 unmodified lines
217	const char *prog_name[] = {	217	const char *prog_name[] = {
218	"freplace/do_bind",	218	"freplace/do_bind",
219	};	219	};
220	test_fexit_bpf2bpf_common("./freplace_connect4.o",	220	test_fexit_bpf2bpf_common("./freplace_connect4.bpf.o",
221	"./connect4_prog.o",	221	"./connect4_prog.bpf.o",
222	ARRAY_SIZE(prog_name),	222	ARRAY_SIZE(prog_name),
223	prog_name, false, NULL);	223	prog_name, false, NULL);
224	}	224	}
@@ -227,7 +227,7 @@ static int test_second_attach(struct bpf_object *obj) · 2 unmodified lines
227	{	227	{
228	const char *prog_name = "security_new_get_constant";	228	const char *prog_name = "security_new_get_constant";
229	const char *tgt_name = "get_constant";	229	const char *tgt_name = "get_constant";
230	const char *tgt_obj_file = "./test_pkt_access.o";	230	const char *tgt_obj_file = "./test_pkt_access.bpf.o";
231	struct bpf_program *prog = NULL;	231	struct bpf_program *prog = NULL;
232	struct bpf_object *tgt_obj;	232	struct bpf_object *tgt_obj;
233	struct bpf_link *link;	233	struct bpf_link *link;
@@ -272,8 +272,8 @@ static void test_func_replace_multi(void) · 38 unmodified lines
272	const char *prog_name[] = {	272	const char *prog_name[] = {
273	"freplace/get_constant",	273	"freplace/get_constant",
274	};	274	};
275	test_fexit_bpf2bpf_common("./freplace_get_constant.o",	275	test_fexit_bpf2bpf_common("./freplace_get_constant.bpf.o",
276	"./test_pkt_access.o",	276	"./test_pkt_access.bpf.o",
277	ARRAY_SIZE(prog_name),	277	ARRAY_SIZE(prog_name),
278	prog_name, true, test_second_attach);	278	prog_name, true, test_second_attach);
279	}	279	}
@@ -281,10 +281,10 @@ static void test_func_replace_multi(void) · 1 unmodified line
281	static void test_fmod_ret_freplace(void)	281	static void test_fmod_ret_freplace(void)
282	{	282	{
283	struct bpf_object freplace_obj = NULL, pkt_obj, *fmod_obj = NULL;	283	struct bpf_object freplace_obj = NULL, pkt_obj, *fmod_obj = NULL;
284	const char *freplace_name = "./freplace_get_constant.o";	284	const char *freplace_name = "./freplace_get_constant.bpf.o";
285	const char *fmod_ret_name = "./fmod_ret_freplace.o";	285	const char *fmod_ret_name = "./fmod_ret_freplace.bpf.o";
286	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts);	286	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts);
287	const char *tgt_name = "./test_pkt_access.o";	287	const char *tgt_name = "./test_pkt_access.bpf.o";
288	struct bpf_link *freplace_link = NULL;	288	struct bpf_link *freplace_link = NULL;
289	struct bpf_program *prog;	289	struct bpf_program *prog;
290	__u32 duration = 0;	290	__u32 duration = 0;
@@ -339,8 +339,8 @@ static void test_func_sockmap_update(void) · 48 unmodified lines
339	const char *prog_name[] = {	339	const char *prog_name[] = {
340	"freplace/cls_redirect",	340	"freplace/cls_redirect",
341	};	341	};
342	test_fexit_bpf2bpf_common("./freplace_cls_redirect.o",	342	test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o",
343	"./test_cls_redirect.o",	343	"./test_cls_redirect.bpf.o",
344	ARRAY_SIZE(prog_name),	344	ARRAY_SIZE(prog_name),
345	prog_name, false, NULL);	345	prog_name, false, NULL);
346	}	346	}
@@ -385,15 +385,15 @@ close_prog: · 38 unmodified lines
385	static void test_func_replace_return_code(void)	385	static void test_func_replace_return_code(void)
386	{	386	{
387	/* test invalid return code in the replaced program */	387	/* test invalid return code in the replaced program */
388	test_obj_load_failure_common("./freplace_connect_v4_prog.o",	388	test_obj_load_failure_common("./freplace_connect_v4_prog.bpf.o",
389	"./connect4_prog.o");	389	"./connect4_prog.bpf.o");
390	}	390	}
391		391
392	static void test_func_map_prog_compatibility(void)	392	static void test_func_map_prog_compatibility(void)
393	{	393	{
394	/* test with spin lock map value in the replaced program */	394	/* test with spin lock map value in the replaced program */
395	test_obj_load_failure_common("./freplace_attach_probe.o",	395	test_obj_load_failure_common("./freplace_attach_probe.bpf.o",
396	"./test_attach_probe.o");	396	"./test_attach_probe.bpf.o");
397	}	397	}
398		398
399	static void test_func_replace_global_func(void)	399	static void test_func_replace_global_func(void)
@@ -402,8 +402,8 @@ static void test_func_replace_global_func(void) · 2 unmodified lines
402	"freplace/test_pkt_access",	402	"freplace/test_pkt_access",
403	};	403	};
404		404
405	test_fexit_bpf2bpf_common("./freplace_global_func.o",	405	test_fexit_bpf2bpf_common("./freplace_global_func.bpf.o",
406	"./test_pkt_access.o",	406	"./test_pkt_access.bpf.o",
407	ARRAY_SIZE(prog_name),	407	ARRAY_SIZE(prog_name),
408	prog_name, false, NULL);	408	prog_name, false, NULL);
409	}	409	}

tools/testing/selftests/bpf/prog_tests/flow_dissector.c+43−1

 #include "bpf_flow.skel.h"
+#define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */
 #ifndef IP_MF
 #define IP_MF 0x2000
 #endif
 	} pkt;
 	struct bpf_flow_keys keys;
 	__u32 flags;
+	__u32 retval;
 };
 #define VLAN_HLEN	4
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6",
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "802.1q-ipv4",
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "802.1ad-ipv6",
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv4-frag",
 			.dport = 8080,
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG,
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv4-no-frag",
 			.is_frag = true,
 			.is_first_frag = true,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-frag",
 			.dport = 8080,
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG,
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-no-frag",
 			.is_frag = true,
 			.is_first_frag = true,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-flow-label",
 			.dport = 8080,
 			.flow_label = __bpf_constant_htonl(0xbeeef),
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-no-flow-label",
 			.flow_label = __bpf_constant_htonl(0xbeeef),
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL,
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipip-encap",
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipip-no-encap",
 			.is_encap = true,
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP,
+		.retval = BPF_OK,
+	},
+	{
+		.name = "ipip-encap-dissector-continue",
+		.pkt.ipip = {
+			.eth.h_proto = __bpf_constant_htons(ETH_P_IP),
+			.iph.ihl = 5,
+			.iph.protocol = IPPROTO_IPIP,
+			.iph.tot_len = __bpf_constant_htons(MAGIC_BYTES),
+			.iph.saddr = __bpf_constant_htonl(FLOW_CONTINUE_SADDR),
+			.iph_inner.ihl = 5,
+			.iph_inner.protocol = IPPROTO_TCP,
+			.iph_inner.tot_len =
+				__bpf_constant_htons(MAGIC_BYTES) -
+				sizeof(struct iphdr),
+			.tcp.doff = 5,
+			.tcp.source = 99,
+			.tcp.dest = 9090,
+		},
+		.retval = BPF_FLOW_DISSECTOR_CONTINUE,
 	},
 };
 		err = tx_tap(tap_fd, &tests[i].pkt, sizeof(tests[i].pkt));
 		CHECK(err < 0, "tx_tap", "err %d errno %d\n", err, errno);
+		/* check the stored flow_keys only if BPF_OK expected */
+		if (tests[i].retval != BPF_OK)
+			continue;
 		err = bpf_map_lookup_elem(keys_fd, &key, &flow_keys);
 		ASSERT_OK(err, "bpf_map_lookup_elem");
 		err = bpf_prog_test_run_opts(prog_fd, &topts);
 		ASSERT_OK(err, "test_run");
-		ASSERT_EQ(topts.retval, 1, "test_run retval");
+		ASSERT_EQ(topts.retval, tests[i].retval, "test_run retval");
+		/* check the resulting flow_keys only if BPF_OK returned */
+		if (topts.retval != BPF_OK)
+			continue;
 		ASSERT_EQ(topts.data_size_out, sizeof(flow_keys),
 			  "test_run data_size_out");
 		CHECK_FLOW_KEYS(tests[i].name, flow_keys, tests[i].keys);

tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c+1−1

@@ -44,7 +44,7 @@ void serial_test_flow_dissector_load_bytes(void) · 43 unmodified lines
44	ASSERT_OK(err, "test_run");	44	ASSERT_OK(err, "test_run");
45	ASSERT_EQ(topts.data_size_out, sizeof(flow_keys),	45	ASSERT_EQ(topts.data_size_out, sizeof(flow_keys),
46	"test_run data_size_out");	46	"test_run data_size_out");
47	ASSERT_EQ(topts.retval, 1, "test_run retval");	47	ASSERT_EQ(topts.retval, BPF_OK, "test_run retval");
48		48
49	if (fd >= -1)	49	if (fd >= -1)
50	close(fd);	50	close(fd);

tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c+2−2

@@ -84,8 +84,8 @@ static void get_stack_print_output(void ctx, int cpu, void data, __u32 size) · 83 unmodified lines
84		84
85	void test_get_stack_raw_tp(void)	85	void test_get_stack_raw_tp(void)
86	{	86	{
87	const char *file = "./test_get_stack_rawtp.o";	87	const char *file = "./test_get_stack_rawtp.bpf.o";
88	const char *file_err = "./test_get_stack_rawtp_err.o";	88	const char *file_err = "./test_get_stack_rawtp_err.bpf.o";
89	const char *prog_name = "bpf_prog1";	89	const char *prog_name = "bpf_prog1";
90	int i, err, prog_fd, exp_cnt = MAX_CNT_RAWTP;	90	int i, err, prog_fd, exp_cnt = MAX_CNT_RAWTP;
91	struct perf_buffer *pb = NULL;	91	struct perf_buffer *pb = NULL;

tools/testing/selftests/bpf/prog_tests/global_data.c+1−1

@@ -131,7 +131,7 @@ static void test_global_data_rdonly(struct bpf_object *obj, __u32 duration) · 130 unmodified lines
131		131
132	void test_global_data(void)	132	void test_global_data(void)
133	{	133	{
134	const char *file = "./test_global_data.o";	134	const char *file = "./test_global_data.bpf.o";
135	struct bpf_object *obj;	135	struct bpf_object *obj;
136	int err, prog_fd;	136	int err, prog_fd;
137	LIBBPF_OPTS(bpf_test_run_opts, topts,	137	LIBBPF_OPTS(bpf_test_run_opts, topts,

tools/testing/selftests/bpf/prog_tests/global_data_init.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3		3
4	void test_global_data_init(void)	4	void test_global_data_init(void)
5	{	5	{
6	const char *file = "./test_global_data.o";	6	const char *file = "./test_global_data.bpf.o";
7	int err = -ENOMEM, map_fd, zero = 0;	7	int err = -ENOMEM, map_fd, zero = 0;
8	__u8 buff = NULL, newval = NULL;	8	__u8 buff = NULL, newval = NULL;
9	struct bpf_object *obj;	9	struct bpf_object *obj;

tools/testing/selftests/bpf/prog_tests/global_func_args.c+1−1

@@ -39,7 +39,7 @@ static void test_global_func_args0(struct bpf_object *obj) · 38 unmodified lines
39		39
40	void test_global_func_args(void)	40	void test_global_func_args(void)
41	{	41	{
42	const char *file = "./test_global_func_args.o";	42	const char *file = "./test_global_func_args.bpf.o";
43	struct bpf_object *obj;	43	struct bpf_object *obj;
44	int err, prog_fd;	44	int err, prog_fd;
45	LIBBPF_OPTS(bpf_test_run_opts, topts,	45	LIBBPF_OPTS(bpf_test_run_opts, topts,

tools/testing/selftests/bpf/prog_tests/htab_update.c+126−0

@@ -0,0 +1,126 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
	3	#define _GNU_SOURCE
	4	#include <sched.h>
	5	#include <stdbool.h>
	6	#include <test_progs.h>
	7	#include "htab_update.skel.h"
	8
	9	struct htab_update_ctx {
	10	int fd;
	11	int loop;
	12	bool stop;
	13	};
	14
	15	static void test_reenter_update(void)
	16	{
	17	struct htab_update *skel;
	18	unsigned int key, value;
	19	int err;
	20
	21	skel = htab_update__open();
	22	if (!ASSERT_OK_PTR(skel, "htab_update__open"))
	23	return;
	24
	25	/* lookup_elem_raw() may be inlined and find_kernel_btf_id() will return -ESRCH */
	26	bpf_program__set_autoload(skel->progs.lookup_elem_raw, true);
	27	err = htab_update__load(skel);
	28	if (!ASSERT_TRUE(!err \|\| err == -ESRCH, "htab_update__load") \|\| err)
	29	goto out;
	30
	31	skel->bss->pid = getpid();
	32	err = htab_update__attach(skel);
	33	if (!ASSERT_OK(err, "htab_update__attach"))
	34	goto out;
	35
	36	/* Will trigger the reentrancy of bpf_map_update_elem() */
	37	key = 0;
	38	value = 0;
	39	err = bpf_map_update_elem(bpf_map__fd(skel->maps.htab), &key, &value, 0);
	40	if (!ASSERT_OK(err, "add element"))
	41	goto out;
	42
	43	ASSERT_EQ(skel->bss->update_err, -EBUSY, "no reentrancy");
	44	out:
	45	htab_update__destroy(skel);
	46	}
	47
	48	static void htab_update_thread(void arg)
	49	{
	50	struct htab_update_ctx *ctx = arg;
	51	cpu_set_t cpus;
	52	int i;
	53
	54	/* Pinned on CPU 0 */
	55	CPU_ZERO(&cpus);
	56	CPU_SET(0, &cpus);
	57	pthread_setaffinity_np(pthread_self(), sizeof(cpus), &cpus);
	58
	59	i = 0;
	60	while (i++ < ctx->loop && !ctx->stop) {
	61	unsigned int key = 0, value = 0;
	62	int err;
	63
	64	err = bpf_map_update_elem(ctx->fd, &key, &value, 0);
	65	if (err) {
	66	ctx->stop = true;
	67	return (void *)(long)err;
	68	}
	69	}
	70
	71	return NULL;
	72	}
	73
	74	static void test_concurrent_update(void)
	75	{
	76	struct htab_update_ctx ctx;
	77	struct htab_update *skel;
	78	unsigned int i, nr;
	79	pthread_t *tids;
	80	int err;
	81
	82	skel = htab_update__open_and_load();
	83	if (!ASSERT_OK_PTR(skel, "htab_update__open_and_load"))
	84	return;
	85
	86	ctx.fd = bpf_map__fd(skel->maps.htab);
	87	ctx.loop = 1000;
	88	ctx.stop = false;
	89
	90	nr = 4;
	91	tids = calloc(nr, sizeof(*tids));
	92	if (!ASSERT_NEQ(tids, NULL, "no mem"))
	93	goto out;
	94
	95	for (i = 0; i < nr; i++) {
	96	err = pthread_create(&tids[i], NULL, htab_update_thread, &ctx);
	97	if (!ASSERT_OK(err, "pthread_create")) {
	98	unsigned int j;
	99
	100	ctx.stop = true;
	101	for (j = 0; j < i; j++)
	102	pthread_join(tids[j], NULL);
	103	goto out;
	104	}
	105	}
	106
	107	for (i = 0; i < nr; i++) {
	108	void *thread_err = NULL;
	109
	110	pthread_join(tids[i], &thread_err);
	111	ASSERT_EQ(thread_err, NULL, "update error");
	112	}
	113
	114	out:
	115	if (tids)
	116	free(tids);
	117	htab_update__destroy(skel);
	118	}
	119
	120	void test_htab_update(void)
	121	{
	122	if (test__start_subtest("reenter_update"))
	123	test_reenter_update();
	124	if (test__start_subtest("concurrent_update"))
	125	test_concurrent_update();
	126	}

tools/testing/selftests/bpf/prog_tests/kfree_skb.c+1−1

@@ -69,7 +69,7 @@ void serial_test_kfree_skb(void) · 68 unmodified lines
69	const int zero = 0;	69	const int zero = 0;
70	bool test_ok[2];	70	bool test_ok[2];
71		71
72	err = bpf_prog_test_load("./test_pkt_access.o", BPF_PROG_TYPE_SCHED_CLS,	72	err = bpf_prog_test_load("./test_pkt_access.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
73	&obj, &prog_fd);	73	&obj, &prog_fd);
74	if (CHECK(err, "prog_load sched cls", "err %d errno %d\n", err, errno))	74	if (CHECK(err, "prog_load sched cls", "err %d errno %d\n", err, errno))
75	return;	75	return;

tools/testing/selftests/bpf/prog_tests/kfunc_call.c+1−1

@@ -109,7 +109,7 @@ static void test_destructive(void) · 108 unmodified lines
109	{	109	{
110	__u64 save_caps = 0;	110	__u64 save_caps = 0;
111		111
112	ASSERT_OK(test_destructive_open_and_load(), "succesful_load");	112	ASSERT_OK(test_destructive_open_and_load(), "successful_load");
113		113
114	if (!ASSERT_OK(cap_disable_effective(1ULL << CAP_SYS_BOOT, &save_caps), "drop_caps"))	114	if (!ASSERT_OK(cap_disable_effective(1ULL << CAP_SYS_BOOT, &save_caps), "drop_caps"))
115	return;	115	return;

tools/testing/selftests/bpf/prog_tests/l4lb_all.c+2−2

@@ -90,7 +90,7 @@ out: · 89 unmodified lines
90	void test_l4lb_all(void)	90	void test_l4lb_all(void)
91	{	91	{
92	if (test__start_subtest("l4lb_inline"))	92	if (test__start_subtest("l4lb_inline"))
93	test_l4lb("test_l4lb.o");	93	test_l4lb("test_l4lb.bpf.o");
94	if (test__start_subtest("l4lb_noinline"))	94	if (test__start_subtest("l4lb_noinline"))
95	test_l4lb("test_l4lb_noinline.o");	95	test_l4lb("test_l4lb_noinline.bpf.o");
96	}	96	}

tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c+2−2

@@ -27,8 +27,8 @@ void test_load_bytes_relative(void) · 26 unmodified lines
27	if (CHECK_FAIL(server_fd < 0))	27	if (CHECK_FAIL(server_fd < 0))
28	goto close_cgroup_fd;	28	goto close_cgroup_fd;
29		29
30	err = bpf_prog_test_load("./load_bytes_relative.o", BPF_PROG_TYPE_CGROUP_SKB,	30	err = bpf_prog_test_load("./load_bytes_relative.bpf.o", BPF_PROG_TYPE_CGROUP_SKB,
31	&obj, &prog_fd);	31	&obj, &prog_fd);
32	if (CHECK_FAIL(err))	32	if (CHECK_FAIL(err))
33	goto close_server_fd;	33	goto close_server_fd;
34		34

tools/testing/selftests/bpf/prog_tests/map_lock.c+1−1

@@ -49,7 +49,7 @@ out: · 48 unmodified lines
49		49
50	void test_map_lock(void)	50	void test_map_lock(void)
51	{	51	{
52	const char *file = "./test_map_lock.o";	52	const char *file = "./test_map_lock.bpf.o";
53	int prog_fd, map_fd[2], vars[17] = {};	53	int prog_fd, map_fd[2], vars[17] = {};
54	pthread_t thread_id[6];	54	pthread_t thread_id[6];
55	struct bpf_object *obj = NULL;	55	struct bpf_object *obj = NULL;

tools/testing/selftests/bpf/prog_tests/pinning.c+2−2

@@ -26,13 +26,13 @@ __u32 get_map_id(struct bpf_object obj, const char name) · 25 unmodified lines
26		26
27	void test_pinning(void)	27	void test_pinning(void)
28	{	28	{
29	const char *file_invalid = "./test_pinning_invalid.o";	29	const char *file_invalid = "./test_pinning_invalid.bpf.o";
30	const char *custpinpath = "/sys/fs/bpf/custom/pinmap";	30	const char *custpinpath = "/sys/fs/bpf/custom/pinmap";
31	const char *nopinpath = "/sys/fs/bpf/nopinmap";	31	const char *nopinpath = "/sys/fs/bpf/nopinmap";
32	const char *nopinpath2 = "/sys/fs/bpf/nopinmap2";	32	const char *nopinpath2 = "/sys/fs/bpf/nopinmap2";
33	const char *custpath = "/sys/fs/bpf/custom";	33	const char *custpath = "/sys/fs/bpf/custom";
34	const char *pinpath = "/sys/fs/bpf/pinmap";	34	const char *pinpath = "/sys/fs/bpf/pinmap";
35	const char *file = "./test_pinning.o";	35	const char *file = "./test_pinning.bpf.o";
36	__u32 map_id, map_id2, duration = 0;	36	__u32 map_id, map_id2, duration = 0;
37	struct stat statbuf = {};	37	struct stat statbuf = {};
38	struct bpf_object *obj;	38	struct bpf_object *obj;

tools/testing/selftests/bpf/prog_tests/pkt_access.c+1−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4		4
5	void test_pkt_access(void)	5	void test_pkt_access(void)
6	{	6	{
7	const char *file = "./test_pkt_access.o";	7	const char *file = "./test_pkt_access.bpf.o";
8	struct bpf_object *obj;	8	struct bpf_object *obj;
9	int err, prog_fd;	9	int err, prog_fd;
10	LIBBPF_OPTS(bpf_test_run_opts, topts,	10	LIBBPF_OPTS(bpf_test_run_opts, topts,

tools/testing/selftests/bpf/prog_tests/pkt_md_access.c+1−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4		4
5	void test_pkt_md_access(void)	5	void test_pkt_md_access(void)
6	{	6	{
7	const char *file = "./test_pkt_md_access.o";	7	const char *file = "./test_pkt_md_access.bpf.o";
8	struct bpf_object *obj;	8	struct bpf_object *obj;
9	int err, prog_fd;	9	int err, prog_fd;
10	LIBBPF_OPTS(bpf_test_run_opts, topts,	10	LIBBPF_OPTS(bpf_test_run_opts, topts,

tools/testing/selftests/bpf/prog_tests/probe_user.c+1−1

@@ -11,7 +11,7 @@ void serial_test_probe_user(void) · 10 unmodified lines
11	#endif	11	#endif
12	};	12	};
13	enum { prog_count = ARRAY_SIZE(prog_names) };	13	enum { prog_count = ARRAY_SIZE(prog_names) };
14	const char *obj_file = "./test_probe_user.o";	14	const char *obj_file = "./test_probe_user.bpf.o";
15	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, );	15	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, );
16	int err, results_map_fd, sock_fd, duration = 0;	16	int err, results_map_fd, sock_fd, duration = 0;
17	struct sockaddr curr, orig, tmp;	17	struct sockaddr curr, orig, tmp;

tools/testing/selftests/bpf/prog_tests/queue_stack_map.c+2−2

@@ -28,9 +28,9 @@ static void test_queue_stack_map_by_type(int type) · 27 unmodified lines
28	vals[i] = rand();	28	vals[i] = rand();
29		29
30	if (type == QUEUE)	30	if (type == QUEUE)
31	strncpy(file, "./test_queue_map.o", sizeof(file));	31	strncpy(file, "./test_queue_map.bpf.o", sizeof(file));
32	else if (type == STACK)	32	else if (type == STACK)
33	strncpy(file, "./test_stack_map.o", sizeof(file));	33	strncpy(file, "./test_stack_map.bpf.o", sizeof(file));
34	else	34	else
35	return;	35	return;
36		36

tools/testing/selftests/bpf/prog_tests/rdonly_maps.c+1−1

@@ -16,7 +16,7 @@ struct rdonly_map_subtest { · 15 unmodified lines
16		16
17	void test_rdonly_maps(void)	17	void test_rdonly_maps(void)
18	{	18	{
19	const char *file = "test_rdonly_maps.o";	19	const char *file = "test_rdonly_maps.bpf.o";
20	struct rdonly_map_subtest subtests[] = {	20	struct rdonly_map_subtest subtests[] = {
21	{ "skip loop", "skip_loop", 0, 0 },	21	{ "skip loop", "skip_loop", 0, 0 },
22	{ "part loop", "part_loop", 3, 2 + 3 + 4 },	22	{ "part loop", "part_loop", 3, 2 + 3 + 4 },

tools/testing/selftests/bpf/prog_tests/reference_tracking.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3		3
4	void test_reference_tracking(void)	4	void test_reference_tracking(void)
5	{	5	{
6	const char *file = "test_sk_lookup_kern.o";	6	const char *file = "test_sk_lookup_kern.bpf.o";
7	const char *obj_name = "ref_track";	7	const char *obj_name = "ref_track";
8	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,	8	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
9	.object_name = obj_name,	9	.object_name = obj_name,

tools/testing/selftests/bpf/prog_tests/resolve_btfids.c+1−1

@@ -101,7 +101,7 @@ static int resolve_symbols(void) · 100 unmodified lines
101	int type_id;	101	int type_id;
102	__u32 nr;	102	__u32 nr;
103		103
104	btf = btf__parse_elf("btf_data.o", NULL);	104	btf = btf__parse_elf("btf_data.bpf.o", NULL);
105	if (CHECK(libbpf_get_error(btf), "resolve",	105	if (CHECK(libbpf_get_error(btf), "resolve",
106	"Failed to load BTF from btf_data.o\n"))	106	"Failed to load BTF from btf_data.o\n"))
107	return -1;	107	return -1;

tools/testing/selftests/bpf/prog_tests/select_reuseport.c+2−2

@@ -91,9 +91,9 @@ static int prepare_bpf_obj(void) · 90 unmodified lines
91	struct bpf_map *map;	91	struct bpf_map *map;
92	int err;	92	int err;
93		93
94	obj = bpf_object__open("test_select_reuseport_kern.o");	94	obj = bpf_object__open("test_select_reuseport_kern.bpf.o");
95	err = libbpf_get_error(obj);	95	err = libbpf_get_error(obj);
96	RET_ERR(err, "open test_select_reuseport_kern.o",	96	RET_ERR(err, "open test_select_reuseport_kern.bpf.o",
97	"obj:%p PTR_ERR(obj):%d\n", obj, err);	97	"obj:%p PTR_ERR(obj):%d\n", obj, err);
98		98
99	map = bpf_object__find_map_by_name(obj, "outer_map");	99	map = bpf_object__find_map_by_name(obj, "outer_map");

tools/testing/selftests/bpf/prog_tests/setget_sockopt.c+125−0

@@ -0,0 +1,125 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) Meta Platforms, Inc. and affiliates. */
	3
	4	#define _GNU_SOURCE
	5	#include <sched.h>
	6	#include <linux/socket.h>
	7	#include <net/if.h>
	8
	9	#include "test_progs.h"
	10	#include "cgroup_helpers.h"
	11	#include "network_helpers.h"
	12
	13	#include "setget_sockopt.skel.h"
	14
	15	#define CG_NAME "/setget-sockopt-test"
	16
	17	static const char addr4_str[] = "127.0.0.1";
	18	static const char addr6_str[] = "::1";
	19	static struct setget_sockopt *skel;
	20	static int cg_fd;
	21
	22	static int create_netns(void)
	23	{
	24	if (!ASSERT_OK(unshare(CLONE_NEWNET), "create netns"))
	25	return -1;
	26
	27	if (!ASSERT_OK(system("ip link set dev lo up"), "set lo up"))
	28	return -1;
	29
	30	if (!ASSERT_OK(system("ip link add dev binddevtest1 type veth peer name binddevtest2"),
	31	"add veth"))
	32	return -1;
	33
	34	if (!ASSERT_OK(system("ip link set dev binddevtest1 up"),
	35	"bring veth up"))
	36	return -1;
	37
	38	return 0;
	39	}
	40
	41	static void test_tcp(int family)
	42	{
	43	struct setget_sockopt__bss *bss = skel->bss;
	44	int sfd, cfd;
	45
	46	memset(bss, 0, sizeof(*bss));
	47
	48	sfd = start_server(family, SOCK_STREAM,
	49	family == AF_INET6 ? addr6_str : addr4_str, 0, 0);
	50	if (!ASSERT_GE(sfd, 0, "start_server"))
	51	return;
	52
	53	cfd = connect_to_fd(sfd, 0);
	54	if (!ASSERT_GE(cfd, 0, "connect_to_fd_server")) {
	55	close(sfd);
	56	return;
	57	}
	58	close(sfd);
	59	close(cfd);
	60
	61	ASSERT_EQ(bss->nr_listen, 1, "nr_listen");
	62	ASSERT_EQ(bss->nr_connect, 1, "nr_connect");
	63	ASSERT_EQ(bss->nr_active, 1, "nr_active");
	64	ASSERT_EQ(bss->nr_passive, 1, "nr_passive");
	65	ASSERT_EQ(bss->nr_socket_post_create, 2, "nr_socket_post_create");
	66	ASSERT_EQ(bss->nr_binddev, 2, "nr_bind");
	67	}
	68
	69	static void test_udp(int family)
	70	{
	71	struct setget_sockopt__bss *bss = skel->bss;
	72	int sfd;
	73
	74	memset(bss, 0, sizeof(*bss));
	75
	76	sfd = start_server(family, SOCK_DGRAM,
	77	family == AF_INET6 ? addr6_str : addr4_str, 0, 0);
	78	if (!ASSERT_GE(sfd, 0, "start_server"))
	79	return;
	80	close(sfd);
	81
	82	ASSERT_GE(bss->nr_socket_post_create, 1, "nr_socket_post_create");
	83	ASSERT_EQ(bss->nr_binddev, 1, "nr_bind");
	84	}
	85
	86	void test_setget_sockopt(void)
	87	{
	88	cg_fd = test__join_cgroup(CG_NAME);
	89	if (cg_fd < 0)
	90	return;
	91
	92	if (create_netns())
	93	goto done;
	94
	95	skel = setget_sockopt__open();
	96	if (!ASSERT_OK_PTR(skel, "open skel"))
	97	goto done;
	98
	99	strcpy(skel->rodata->veth, "binddevtest1");
	100	skel->rodata->veth_ifindex = if_nametoindex("binddevtest1");
	101	if (!ASSERT_GT(skel->rodata->veth_ifindex, 0, "if_nametoindex"))
	102	goto done;
	103
	104	if (!ASSERT_OK(setget_sockopt__load(skel), "load skel"))
	105	goto done;
	106
	107	skel->links.skops_sockopt =
	108	bpf_program__attach_cgroup(skel->progs.skops_sockopt, cg_fd);
	109	if (!ASSERT_OK_PTR(skel->links.skops_sockopt, "attach cgroup"))
	110	goto done;
	111
	112	skel->links.socket_post_create =
	113	bpf_program__attach_cgroup(skel->progs.socket_post_create, cg_fd);
	114	if (!ASSERT_OK_PTR(skel->links.socket_post_create, "attach_cgroup"))
	115	goto done;
	116
	117	test_tcp(AF_INET6);
	118	test_tcp(AF_INET);
	119	test_udp(AF_INET6);
	120	test_udp(AF_INET);
	121
	122	done:
	123	setget_sockopt__destroy(skel);
	124	close(cg_fd);
	125	}

tools/testing/selftests/bpf/prog_tests/sk_assign.c+1−1

@@ -47,7 +47,7 @@ configure_stack(void) · 46 unmodified lines
47	if (CHECK_FAIL(system("tc qdisc add dev lo clsact")))	47	if (CHECK_FAIL(system("tc qdisc add dev lo clsact")))
48	return false;	48	return false;
49	sprintf(tc_cmd, "%s %s %s %s", "tc filter add dev lo ingress bpf",	49	sprintf(tc_cmd, "%s %s %s %s", "tc filter add dev lo ingress bpf",
50	"direct-action object-file ./test_sk_assign.o",	50	"direct-action object-file ./test_sk_assign.bpf.o",
51	"section tc",	51	"section tc",
52	(env.verbosity < VERBOSE_VERY) ? " 2>/dev/null" : "verbose");	52	(env.verbosity < VERBOSE_VERY) ? " 2>/dev/null" : "verbose");
53	if (CHECK(system(tc_cmd), "BPF load failed;",	53	if (CHECK(system(tc_cmd), "BPF load failed;",

tools/testing/selftests/bpf/prog_tests/skb_ctx.c+1−1

@@ -31,7 +31,7 @@ void test_skb_ctx(void) · 30 unmodified lines
31	struct bpf_object *obj;	31	struct bpf_object *obj;
32	int err, prog_fd, i;	32	int err, prog_fd, i;
33		33
34	err = bpf_prog_test_load("./test_skb_ctx.o", BPF_PROG_TYPE_SCHED_CLS,	34	err = bpf_prog_test_load("./test_skb_ctx.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
35	&obj, &prog_fd);	35	&obj, &prog_fd);
36	if (!ASSERT_OK(err, "load"))	36	if (!ASSERT_OK(err, "load"))
37	return;	37	return;

tools/testing/selftests/bpf/prog_tests/skb_helpers.c+1−1

@@ -20,7 +20,7 @@ void test_skb_helpers(void) · 19 unmodified lines
20	struct bpf_object *obj;	20	struct bpf_object *obj;
21	int err, prog_fd;	21	int err, prog_fd;
22		22
23	err = bpf_prog_test_load("./test_skb_helpers.o",	23	err = bpf_prog_test_load("./test_skb_helpers.bpf.o",
24	BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);	24	BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
25	if (!ASSERT_OK(err, "load"))	25	if (!ASSERT_OK(err, "load"))
26	return;	26	return;

tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c+1−1

@@ -174,7 +174,7 @@ static void run_test(int cgroup_fd) · 173 unmodified lines
174	pthread_t tid;	174	pthread_t tid;
175	int err;	175	int err;
176		176
177	obj = bpf_object__open_file("sockopt_inherit.o", NULL);	177	obj = bpf_object__open_file("sockopt_inherit.bpf.o", NULL);
178	if (!ASSERT_OK_PTR(obj, "obj_open"))	178	if (!ASSERT_OK_PTR(obj, "obj_open"))
179	return;	179	return;
180		180

tools/testing/selftests/bpf/prog_tests/sockopt_multi.c+1−1

@@ -310,7 +310,7 @@ void test_sockopt_multi(void) · 309 unmodified lines
310	if (CHECK_FAIL(cg_child < 0))	310	if (CHECK_FAIL(cg_child < 0))
311	goto out;	311	goto out;
312		312
313	obj = bpf_object__open_file("sockopt_multi.o", NULL);	313	obj = bpf_object__open_file("sockopt_multi.bpf.o", NULL);
314	if (!ASSERT_OK_PTR(obj, "obj_load"))	314	if (!ASSERT_OK_PTR(obj, "obj_load"))
315	goto out;	315	goto out;
316		316

tools/testing/selftests/bpf/prog_tests/spinlock.c+1−1

@@ -19,7 +19,7 @@ static void spin_lock_thread(void arg) · 18 unmodified lines
19		19
20	void test_spinlock(void)	20	void test_spinlock(void)
21	{	21	{
22	const char *file = "./test_spin_lock.o";	22	const char *file = "./test_spin_lock.bpf.o";
23	pthread_t thread_id[4];	23	pthread_t thread_id[4];
24	struct bpf_object *obj = NULL;	24	struct bpf_object *obj = NULL;
25	int prog_fd;	25	int prog_fd;

tools/testing/selftests/bpf/prog_tests/stacktrace_map.c+1−1

@@ -6,7 +6,7 @@ void test_stacktrace_map(void) · 5 unmodified lines
6	int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;	6	int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
7	const char *prog_name = "oncpu";	7	const char *prog_name = "oncpu";
8	int err, prog_fd, stack_trace_len;	8	int err, prog_fd, stack_trace_len;
9	const char *file = "./test_stacktrace_map.o";	9	const char *file = "./test_stacktrace_map.bpf.o";
10	__u32 key, val, duration = 0;	10	__u32 key, val, duration = 0;
11	struct bpf_program *prog;	11	struct bpf_program *prog;
12	struct bpf_object *obj;	12	struct bpf_object *obj;

tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c+1−1

@@ -5,7 +5,7 @@ void test_stacktrace_map_raw_tp(void) · 4 unmodified lines
5	{	5	{
6	const char *prog_name = "oncpu";	6	const char *prog_name = "oncpu";
7	int control_map_fd, stackid_hmap_fd, stackmap_fd;	7	int control_map_fd, stackid_hmap_fd, stackmap_fd;
8	const char *file = "./test_stacktrace_map.o";	8	const char *file = "./test_stacktrace_map.bpf.o";
9	__u32 key, val, duration = 0;	9	__u32 key, val, duration = 0;
10	int err, prog_fd;	10	int err, prog_fd;
11	struct bpf_program *prog;	11	struct bpf_program *prog;

tools/testing/selftests/bpf/prog_tests/tailcalls.c+18−18

@@ -20,8 +20,8 @@ static void test_tailcall_1(void) · 19 unmodified lines
20	.repeat = 1,	20	.repeat = 1,
21	);	21	);
22		22
23	err = bpf_prog_test_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj,	23	err = bpf_prog_test_load("tailcall1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
24	&prog_fd);	24	&prog_fd);
25	if (CHECK_FAIL(err))	25	if (CHECK_FAIL(err))
26	return;	26	return;
27		27
@@ -156,8 +156,8 @@ static void test_tailcall_2(void) · 128 unmodified lines
156	.repeat = 1,	156	.repeat = 1,
157	);	157	);
158		158
159	err = bpf_prog_test_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj,	159	err = bpf_prog_test_load("tailcall2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
160	&prog_fd);	160	&prog_fd);
161	if (CHECK_FAIL(err))	161	if (CHECK_FAIL(err))
162	return;	162	return;
163		163
@@ -299,7 +299,7 @@ out: · 135 unmodified lines
299	*/	299	*/
300	static void test_tailcall_3(void)	300	static void test_tailcall_3(void)
301	{	301	{
302	test_tailcall_count("tailcall3.o");	302	test_tailcall_count("tailcall3.bpf.o");
303	}	303	}
304		304
305	/* test_tailcall_6 checks that the count value of the tail call limit	305	/* test_tailcall_6 checks that the count value of the tail call limit
@@ -307,7 +307,7 @@ static void test_tailcall_3(void) · 1 unmodified line
307	*/	307	*/
308	static void test_tailcall_6(void)	308	static void test_tailcall_6(void)
309	{	309	{
310	test_tailcall_count("tailcall6.o");	310	test_tailcall_count("tailcall6.bpf.o");
311	}	311	}
312		312
313	/* test_tailcall_4 checks that the kernel properly selects indirect jump	313	/* test_tailcall_4 checks that the kernel properly selects indirect jump
@@ -329,8 +329,8 @@ static void test_tailcall_4(void) · 15 unmodified lines
329	.repeat = 1,	329	.repeat = 1,
330	);	330	);
331		331
332	err = bpf_prog_test_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj,	332	err = bpf_prog_test_load("tailcall4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
333	&prog_fd);	333	&prog_fd);
334	if (CHECK_FAIL(err))	334	if (CHECK_FAIL(err))
335	return;	335	return;
336		336
@@ -419,8 +419,8 @@ static void test_tailcall_5(void) · 82 unmodified lines
419	.repeat = 1,	419	.repeat = 1,
420	);	420	);
421		421
422	err = bpf_prog_test_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj,	422	err = bpf_prog_test_load("tailcall5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
423	&prog_fd);	423	&prog_fd);
424	if (CHECK_FAIL(err))	424	if (CHECK_FAIL(err))
425	return;	425	return;
426		426
@@ -507,8 +507,8 @@ static void test_tailcall_bpf2bpf_1(void) · 80 unmodified lines
507	.repeat = 1,	507	.repeat = 1,
508	);	508	);
509		509
510	err = bpf_prog_test_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS,	510	err = bpf_prog_test_load("tailcall_bpf2bpf1.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
511	&obj, &prog_fd);	511	&obj, &prog_fd);
512	if (CHECK_FAIL(err))	512	if (CHECK_FAIL(err))
513	return;	513	return;
514		514
@@ -591,8 +591,8 @@ static void test_tailcall_bpf2bpf_2(void) · 76 unmodified lines
591	.repeat = 1,	591	.repeat = 1,
592	);	592	);
593		593
594	err = bpf_prog_test_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS,	594	err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
595	&obj, &prog_fd);	595	&obj, &prog_fd);
596	if (CHECK_FAIL(err))	596	if (CHECK_FAIL(err))
597	return;	597	return;
598		598
@@ -671,8 +671,8 @@ static void test_tailcall_bpf2bpf_3(void) · 72 unmodified lines
671	.repeat = 1,	671	.repeat = 1,
672	);	672	);
673		673
674	err = bpf_prog_test_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS,	674	err = bpf_prog_test_load("tailcall_bpf2bpf3.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
675	&obj, &prog_fd);	675	&obj, &prog_fd);
676	if (CHECK_FAIL(err))	676	if (CHECK_FAIL(err))
677	return;	677	return;
678		678
@@ -766,8 +766,8 @@ static void test_tailcall_bpf2bpf_4(bool noise) · 87 unmodified lines
766	.repeat = 1,	766	.repeat = 1,
767	);	767	);
768		768
769	err = bpf_prog_test_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS,	769	err = bpf_prog_test_load("tailcall_bpf2bpf4.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
770	&obj, &prog_fd);	770	&obj, &prog_fd);
771	if (CHECK_FAIL(err))	771	if (CHECK_FAIL(err))
772	return;	772	return;
773		773

tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3		3
4	void test_task_fd_query_rawtp(void)	4	void test_task_fd_query_rawtp(void)
5	{	5	{
6	const char *file = "./test_get_stack_rawtp.o";	6	const char *file = "./test_get_stack_rawtp.bpf.o";
7	__u64 probe_offset, probe_addr;	7	__u64 probe_offset, probe_addr;
8	__u32 len, prog_id, fd_type;	8	__u32 len, prog_id, fd_type;
9	struct bpf_object *obj;	9	struct bpf_object *obj;

tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c+1−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	static void test_task_fd_query_tp_core(const char *probe_name,	4	static void test_task_fd_query_tp_core(const char *probe_name,
5	const char *tp_name)	5	const char *tp_name)
6	{	6	{
7	const char *file = "./test_tracepoint.o";	7	const char *file = "./test_tracepoint.bpf.o";
8	int err, bytes, efd, prog_fd, pmu_fd;	8	int err, bytes, efd, prog_fd, pmu_fd;
9	struct perf_event_attr attr = {};	9	struct perf_event_attr attr = {};
10	__u64 probe_offset, probe_addr;	10	__u64 probe_offset, probe_addr;

tools/testing/selftests/bpf/prog_tests/tcp_estats.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3		3
4	void test_tcp_estats(void)	4	void test_tcp_estats(void)
5	{	5	{
6	const char *file = "./test_tcp_estats.o";	6	const char *file = "./test_tcp_estats.bpf.o";
7	int err, prog_fd;	7	int err, prog_fd;
8	struct bpf_object *obj;	8	struct bpf_object *obj;
9	__u32 duration = 0;	9	__u32 duration = 0;

tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c+1−9

@@ -9,18 +9,10 @@ · 8 unmodified lines
9		9
10	#include "bprm_opts.skel.h"	10	#include "bprm_opts.skel.h"
11	#include "network_helpers.h"	11	#include "network_helpers.h"
12		12	#include "task_local_storage_helpers.h"
13	#ifndef __NR_pidfd_open
14	#define __NR_pidfd_open 434
15	#endif
16		13
17	static const char * const bash_envp[] = { "TMPDIR=shouldnotbeset", NULL };	14	static const char * const bash_envp[] = { "TMPDIR=shouldnotbeset", NULL };
18		15
19	static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
20	{
21	return syscall(__NR_pidfd_open, pid, flags);
22	}
23
24	static int update_storage(int map_fd, int secureexec)	16	static int update_storage(int map_fd, int secureexec)
25	{	17	{
26	int task_fd, ret = 0;	18	int task_fd, ret = 0;

tools/testing/selftests/bpf/prog_tests/test_global_funcs.c+17−17

@@ -65,23 +65,23 @@ struct test_def { · 64 unmodified lines
65	void test_test_global_funcs(void)	65	void test_test_global_funcs(void)
66	{	66	{
67	struct test_def tests[] = {	67	struct test_def tests[] = {
68	{ "test_global_func1.o", "combined stack size of 4 calls is 544" },	68	{ "test_global_func1.bpf.o", "combined stack size of 4 calls is 544" },
69	{ "test_global_func2.o" },	69	{ "test_global_func2.bpf.o" },
70	{ "test_global_func3.o" , "the call stack of 8 frames" },	70	{ "test_global_func3.bpf.o", "the call stack of 8 frames" },
71	{ "test_global_func4.o" },	71	{ "test_global_func4.bpf.o" },
72	{ "test_global_func5.o" , "expected pointer to ctx, but got PTR" },	72	{ "test_global_func5.bpf.o", "expected pointer to ctx, but got PTR" },
73	{ "test_global_func6.o" , "modified ctx ptr R2" },	73	{ "test_global_func6.bpf.o", "modified ctx ptr R2" },
74	{ "test_global_func7.o" , "foo() doesn't return scalar" },	74	{ "test_global_func7.bpf.o", "foo() doesn't return scalar" },
75	{ "test_global_func8.o" },	75	{ "test_global_func8.bpf.o" },
76	{ "test_global_func9.o" },	76	{ "test_global_func9.bpf.o" },
77	{ "test_global_func10.o", "invalid indirect read from stack" },	77	{ "test_global_func10.bpf.o", "invalid indirect read from stack" },
78	{ "test_global_func11.o", "Caller passes invalid args into func#1" },	78	{ "test_global_func11.bpf.o", "Caller passes invalid args into func#1" },
79	{ "test_global_func12.o", "invalid mem access 'mem_or_null'" },	79	{ "test_global_func12.bpf.o", "invalid mem access 'mem_or_null'" },
80	{ "test_global_func13.o", "Caller passes invalid args into func#1" },	80	{ "test_global_func13.bpf.o", "Caller passes invalid args into func#1" },
81	{ "test_global_func14.o", "reference type('FWD S') size cannot be determined" },	81	{ "test_global_func14.bpf.o", "reference type('FWD S') size cannot be determined" },
82	{ "test_global_func15.o", "At program exit the register R0 has value" },	82	{ "test_global_func15.bpf.o", "At program exit the register R0 has value" },
83	{ "test_global_func16.o", "invalid indirect read from stack" },	83	{ "test_global_func16.bpf.o", "invalid indirect read from stack" },
84	{ "test_global_func17.o", "Caller passes invalid args into func#1" },	84	{ "test_global_func17.bpf.o", "Caller passes invalid args into func#1" },
85	};	85	};
86	libbpf_print_fn_t old_print_fn = NULL;	86	libbpf_print_fn_t old_print_fn = NULL;
87	int err, i, duration = 0;	87	int err, i, duration = 0;

tools/testing/selftests/bpf/prog_tests/test_local_storage.c+1−9

@@ -11,15 +11,7 @@ · 10 unmodified lines
11		11
12	#include "local_storage.skel.h"	12	#include "local_storage.skel.h"
13	#include "network_helpers.h"	13	#include "network_helpers.h"
14		14	#include "task_local_storage_helpers.h"
15	#ifndef __NR_pidfd_open
16	#define __NR_pidfd_open 434
17	#endif
18
19	static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
20	{
21	return syscall(__NR_pidfd_open, pid, flags);
22	}
23		15
24	static unsigned int duration;	16	static unsigned int duration;
25		17

tools/testing/selftests/bpf/prog_tests/test_overhead.c+1−1

@@ -72,7 +72,7 @@ void test_test_overhead(void) · 71 unmodified lines
72	if (CHECK_FAIL(prctl(PR_GET_NAME, comm, 0L, 0L, 0L)))	72	if (CHECK_FAIL(prctl(PR_GET_NAME, comm, 0L, 0L, 0L)))
73	return;	73	return;
74		74
75	obj = bpf_object__open_file("./test_overhead.o", NULL);	75	obj = bpf_object__open_file("./test_overhead.bpf.o", NULL);
76	if (!ASSERT_OK_PTR(obj, "obj_open_file"))	76	if (!ASSERT_OK_PTR(obj, "obj_open_file"))
77	return;	77	return;
78		78

tools/testing/selftests/bpf/prog_tests/tp_attach_query.c+1−1

@@ -6,7 +6,7 @@ void serial_test_tp_attach_query(void) · 5 unmodified lines
6	const int num_progs = 3;	6	const int num_progs = 3;
7	int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];	7	int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
8	__u32 duration = 0, info_len, saved_prog_ids[num_progs];	8	__u32 duration = 0, info_len, saved_prog_ids[num_progs];
9	const char *file = "./test_tracepoint.o";	9	const char *file = "./test_tracepoint.bpf.o";
10	struct perf_event_query_bpf *query;	10	struct perf_event_query_bpf *query;
11	struct perf_event_attr attr = {};	11	struct perf_event_attr attr = {};
12	struct bpf_object *obj[num_progs];	12	struct bpf_object *obj[num_progs];

tools/testing/selftests/bpf/prog_tests/trampoline_count.c+1−1

@@ -35,7 +35,7 @@ static struct bpf_program load_prog(char file, char name, struct inst inst) · 34 unmodified lines
35	/* TODO: use different target function to run in concurrent mode */	35	/* TODO: use different target function to run in concurrent mode */
36	void serial_test_trampoline_count(void)	36	void serial_test_trampoline_count(void)
37	{	37	{
38	char *file = "test_trampoline_count.o";	38	char *file = "test_trampoline_count.bpf.o";
39	char *const progs[] = { "fentry_test", "fmod_ret_test", "fexit_test" };	39	char *const progs[] = { "fentry_test", "fmod_ret_test", "fexit_test" };
40	struct inst inst[MAX_TRAMP_PROGS + 1] = {};	40	struct inst inst[MAX_TRAMP_PROGS + 1] = {};
41	struct bpf_program *prog;	41	struct bpf_program *prog;

tools/testing/selftests/bpf/prog_tests/xdp.c+1−1

@@ -8,7 +8,7 @@ void test_xdp(void) · 7 unmodified lines
8	struct vip key6 = {.protocol = 6, .family = AF_INET6};	8	struct vip key6 = {.protocol = 6, .family = AF_INET6};
9	struct iptnl_info value4 = {.family = AF_INET};	9	struct iptnl_info value4 = {.family = AF_INET};
10	struct iptnl_info value6 = {.family = AF_INET6};	10	struct iptnl_info value6 = {.family = AF_INET6};
11	const char *file = "./test_xdp.o";	11	const char *file = "./test_xdp.bpf.o";
12	struct bpf_object *obj;	12	struct bpf_object *obj;
13	char buf[128];	13	char buf[128];
14	struct ipv6hdr iph6;	14	struct ipv6hdr iph6;

tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c+1−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4		4
5	static void test_xdp_update_frags(void)	5	static void test_xdp_update_frags(void)
6	{	6	{
7	const char *file = "./test_xdp_update_frags.o";	7	const char *file = "./test_xdp_update_frags.bpf.o";
8	int err, prog_fd, max_skb_frags, buf_size, num;	8	int err, prog_fd, max_skb_frags, buf_size, num;
9	struct bpf_program *prog;	9	struct bpf_program *prog;
10	struct bpf_object *obj;	10	struct bpf_object *obj;

tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c+5−5

@@ -4,7 +4,7 @@ · 3 unmodified lines
4		4
5	static void test_xdp_adjust_tail_shrink(void)	5	static void test_xdp_adjust_tail_shrink(void)
6	{	6	{
7	const char *file = "./test_xdp_adjust_tail_shrink.o";	7	const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
8	__u32 expect_sz;	8	__u32 expect_sz;
9	struct bpf_object *obj;	9	struct bpf_object *obj;
10	int err, prog_fd;	10	int err, prog_fd;
@@ -39,7 +39,7 @@ static void test_xdp_adjust_tail_shrink(void) · 28 unmodified lines
39		39
40	static void test_xdp_adjust_tail_grow(void)	40	static void test_xdp_adjust_tail_grow(void)
41	{	41	{
42	const char *file = "./test_xdp_adjust_tail_grow.o";	42	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
43	struct bpf_object *obj;	43	struct bpf_object *obj;
44	char buf[4096]; /* avoid segfault: large buf to hold grow results */	44	char buf[4096]; /* avoid segfault: large buf to hold grow results */
45	__u32 expect_sz;	45	__u32 expect_sz;
@@ -73,7 +73,7 @@ static void test_xdp_adjust_tail_grow(void) · 27 unmodified lines
73		73
74	static void test_xdp_adjust_tail_grow2(void)	74	static void test_xdp_adjust_tail_grow2(void)
75	{	75	{
76	const char *file = "./test_xdp_adjust_tail_grow.o";	76	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
77	char buf[4096]; /* avoid segfault: large buf to hold grow results */	77	char buf[4096]; /* avoid segfault: large buf to hold grow results */
78	int tailroom = 320; /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info))*/;	78	int tailroom = 320; /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info))*/;
79	struct bpf_object *obj;	79	struct bpf_object *obj;
@@ -135,7 +135,7 @@ static void test_xdp_adjust_tail_grow2(void) · 55 unmodified lines
135		135
136	static void test_xdp_adjust_frags_tail_shrink(void)	136	static void test_xdp_adjust_frags_tail_shrink(void)
137	{	137	{
138	const char *file = "./test_xdp_adjust_tail_shrink.o";	138	const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
139	__u32 exp_size;	139	__u32 exp_size;
140	struct bpf_program *prog;	140	struct bpf_program *prog;
141	struct bpf_object *obj;	141	struct bpf_object *obj;
@@ -202,7 +202,7 @@ out: · 60 unmodified lines
202		202
203	static void test_xdp_adjust_frags_tail_grow(void)	203	static void test_xdp_adjust_frags_tail_grow(void)
204	{	204	{
205	const char *file = "./test_xdp_adjust_tail_grow.o";	205	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
206	__u32 exp_size;	206	__u32 exp_size;
207	struct bpf_program *prog;	207	struct bpf_program *prog;
208	struct bpf_object *obj;	208	struct bpf_object *obj;

tools/testing/selftests/bpf/prog_tests/xdp_attach.c+1−1

@@ -8,7 +8,7 @@ void serial_test_xdp_attach(void) · 7 unmodified lines
8	{	8	{
9	__u32 duration = 0, id1, id2, id0 = 0, len;	9	__u32 duration = 0, id1, id2, id0 = 0, len;
10	struct bpf_object obj1, obj2, *obj3;	10	struct bpf_object obj1, obj2, *obj3;
11	const char *file = "./test_xdp.o";	11	const char *file = "./test_xdp.bpf.o";
12	struct bpf_prog_info info = {};	12	struct bpf_prog_info info = {};
13	int err, fd1, fd2, fd3;	13	int err, fd1, fd2, fd3;
14	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);	14	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);

tools/testing/selftests/bpf/prog_tests/xdp_info.c+1−1

@@ -7,7 +7,7 @@ · 6 unmodified lines
7	void serial_test_xdp_info(void)	7	void serial_test_xdp_info(void)
8	{	8	{
9	__u32 len = sizeof(struct bpf_prog_info), duration = 0, prog_id;	9	__u32 len = sizeof(struct bpf_prog_info), duration = 0, prog_id;
10	const char *file = "./xdp_dummy.o";	10	const char *file = "./xdp_dummy.bpf.o";
11	struct bpf_prog_info info = {};	11	struct bpf_prog_info info = {};
12	struct bpf_object *obj;	12	struct bpf_object *obj;
13	int err, prog_fd;	13	int err, prog_fd;

tools/testing/selftests/bpf/prog_tests/xdp_perf.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3		3
4	void test_xdp_perf(void)	4	void test_xdp_perf(void)
5	{	5	{
6	const char *file = "./xdp_dummy.o";	6	const char *file = "./xdp_dummy.bpf.o";
7	struct bpf_object *obj;	7	struct bpf_object *obj;
8	char in[128], out[128];	8	char in[128], out[128];
9	int err, prog_fd;	9	int err, prog_fd;

tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c+1−1

@@ -82,7 +82,7 @@ static void test_synproxy(bool xdp) · 81 unmodified lines
82	SYS("ethtool -K tmp0 tx off");	82	SYS("ethtool -K tmp0 tx off");
83	if (xdp)	83	if (xdp)
84	/* Workaround required for veth. */	84	/* Workaround required for veth. */
85	SYS("ip link set tmp0 xdp object xdp_dummy.o section xdp 2> /dev/null");	85	SYS("ip link set tmp0 xdp object xdp_dummy.bpf.o section xdp 2> /dev/null");
86		86
87	ns = open_netns("synproxy");	87	ns = open_netns("synproxy");
88	if (!ASSERT_OK_PTR(ns, "setns"))	88	if (!ASSERT_OK_PTR(ns, "setns"))

tools/testing/selftests/bpf/progs/bind4_prog.c+0−2

@@ -6,8 +6,6 @@ · 5 unmodified lines
6	#include <linux/bpf.h>	6	#include <linux/bpf.h>
7	#include <linux/in.h>	7	#include <linux/in.h>
8	#include <linux/in6.h>	8	#include <linux/in6.h>
9	#include <sys/socket.h>
10	#include <netinet/tcp.h>
11	#include <linux/if.h>	9	#include <linux/if.h>
12	#include <errno.h>	10	#include <errno.h>
13		11

tools/testing/selftests/bpf/progs/bind6_prog.c+0−2

@@ -6,8 +6,6 @@ · 5 unmodified lines
6	#include <linux/bpf.h>	6	#include <linux/bpf.h>
7	#include <linux/in.h>	7	#include <linux/in.h>
8	#include <linux/in6.h>	8	#include <linux/in6.h>
9	#include <sys/socket.h>
10	#include <netinet/tcp.h>
11	#include <linux/if.h>	9	#include <linux/if.h>
12	#include <errno.h>	10	#include <errno.h>
13		11

tools/testing/selftests/bpf/progs/bpf_flow.c+15−0

@@ -22,6 +22,8 @@ · 21 unmodified lines
22	#define PROG(F) PROG_(F, _##F)	22	#define PROG(F) PROG_(F, _##F)
23	#define PROG_(NUM, NAME) SEC("flow_dissector") int flow_dissector_##NUM	23	#define PROG_(NUM, NAME) SEC("flow_dissector") int flow_dissector_##NUM
24		24
		25	#define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */
		26
25	/* These are the identifiers of the BPF programs that will be used in tail	27	/* These are the identifiers of the BPF programs that will be used in tail
26	* calls. Name is limited to 16 characters, with the terminating character and	28	* calls. Name is limited to 16 characters, with the terminating character and
27	* bpf_func_ above, we have only 6 to work with, anything after will be cropped.	29	* bpf_func_ above, we have only 6 to work with, anything after will be cropped.
@@ -143,6 +145,19 @@ int _dissect(struct __sk_buff *skb) · 115 unmodified lines
143	{	145	{
144	struct bpf_flow_keys *keys = skb->flow_keys;	146	struct bpf_flow_keys *keys = skb->flow_keys;
145		147
		148	if (keys->n_proto == bpf_htons(ETH_P_IP)) {
		149	/* IP traffic from FLOW_CONTINUE_SADDR falls-back to
		150	* standard dissector
		151	*/
		152	struct iphdr *iph, _iph;
		153
		154	iph = bpf_flow_dissect_get_header(skb, sizeof(*iph), &_iph);
		155	if (iph && iph->ihl == 5 &&
		156	iph->saddr == bpf_htonl(FLOW_CONTINUE_SADDR)) {
		157	return BPF_FLOW_DISSECTOR_CONTINUE;
		158	}
		159	}
		160
146	return parse_eth_proto(skb, keys->n_proto);	161	return parse_eth_proto(skb, keys->n_proto);
147	}	162	}
148		163

tools/testing/selftests/bpf/progs/bpf_iter.h+7−0

 #define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used
 #define bpf_iter__sockmap bpf_iter__sockmap___not_used
 #define bpf_iter__bpf_link bpf_iter__bpf_link___not_used
+#define bpf_iter__cgroup bpf_iter__cgroup___not_used
 #define btf_ptr btf_ptr___not_used
 #define BTF_F_COMPACT BTF_F_COMPACT___not_used
 #define BTF_F_NONAME BTF_F_NONAME___not_used
 #undef bpf_iter__bpf_sk_storage_map
 #undef bpf_iter__sockmap
 #undef bpf_iter__bpf_link
+#undef bpf_iter__cgroup
 #undef btf_ptr
 #undef BTF_F_COMPACT
 #undef BTF_F_NONAME
 	struct bpf_link *link;
 };
+struct bpf_iter__cgroup {
+	struct bpf_iter_meta *meta;
+	struct cgroup *cgroup;
+} __attribute__((preserve_access_index));
 struct btf_ptr {
 	void *ptr;
 	__u32 type_id;

tools/testing/selftests/bpf/progs/bpf_tracing_net.h+31−1

@@ -6,13 +6,41 @@ · 5 unmodified lines
6	#define AF_INET6 10	6	#define AF_INET6 10
7		7
8	#define SOL_SOCKET 1	8	#define SOL_SOCKET 1
		9	#define SO_REUSEADDR 2
9	#define SO_SNDBUF 7	10	#define SO_SNDBUF 7
10	#define __SO_ACCEPTCON (1 << 16)	11	#define SO_RCVBUF 8
		12	#define SO_KEEPALIVE 9
11	#define SO_PRIORITY 12	13	#define SO_PRIORITY 12
		14	#define SO_REUSEPORT 15
		15	#define SO_RCVLOWAT 18
		16	#define SO_BINDTODEVICE 25
		17	#define SO_MARK 36
		18	#define SO_MAX_PACING_RATE 47
		19	#define SO_BINDTOIFINDEX 62
		20	#define SO_TXREHASH 74
		21	#define __SO_ACCEPTCON (1 << 16)
		22
		23	#define IP_TOS 1
		24
		25	#define IPV6_TCLASS 67
		26	#define IPV6_AUTOFLOWLABEL 70
12		27
13	#define SOL_TCP 6	28	#define SOL_TCP 6
		29	#define TCP_NODELAY 1
		30	#define TCP_MAXSEG 2
		31	#define TCP_KEEPIDLE 4
		32	#define TCP_KEEPINTVL 5
		33	#define TCP_KEEPCNT 6
		34	#define TCP_SYNCNT 7
		35	#define TCP_WINDOW_CLAMP 10
14	#define TCP_CONGESTION 13	36	#define TCP_CONGESTION 13
		37	#define TCP_THIN_LINEAR_TIMEOUTS 16
		38	#define TCP_USER_TIMEOUT 18
		39	#define TCP_NOTSENT_LOWAT 25
		40	#define TCP_SAVE_SYN 27
		41	#define TCP_SAVED_SYN 28
15	#define TCP_CA_NAME_MAX 16	42	#define TCP_CA_NAME_MAX 16
		43	#define TCP_NAGLE_OFF 1
16		44
17	#define ICSK_TIME_RETRANS 1	45	#define ICSK_TIME_RETRANS 1
18	#define ICSK_TIME_PROBE0 3	46	#define ICSK_TIME_PROBE0 3
@@ -49,6 +77,8 @@ · 30 unmodified lines
49	#define sk_state __sk_common.skc_state	77	#define sk_state __sk_common.skc_state
50	#define sk_v6_daddr __sk_common.skc_v6_daddr	78	#define sk_v6_daddr __sk_common.skc_v6_daddr
51	#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr	79	#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
		80	#define sk_flags __sk_common.skc_flags
		81	#define sk_reuse __sk_common.skc_reuse
52		82
53	#define s6_addr32 in6_u.u6_addr32	83	#define s6_addr32 in6_u.u6_addr32
54		84

tools/testing/selftests/bpf/progs/cb_refs.c+116−0

@@ -0,0 +1,116 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <vmlinux.h>
	3	#include <bpf/bpf_tracing.h>
	4	#include <bpf/bpf_helpers.h>
	5
	6	struct map_value {
	7	struct prog_test_ref_kfunc __kptr_ref *ptr;
	8	};
	9
	10	struct {
	11	__uint(type, BPF_MAP_TYPE_ARRAY);
	12	__type(key, int);
	13	__type(value, struct map_value);
	14	__uint(max_entries, 16);
	15	} array_map SEC(".maps");
	16
	17	extern struct prog_test_ref_kfunc bpf_kfunc_call_test_acquire(unsigned long sp) __ksym;
	18	extern void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym;
	19
	20	static __noinline int cb1(void map, void key, void value, void ctx)
	21	{
	22	void p = (void **)ctx;
	23	bpf_kfunc_call_test_release(p);
	24	/* Without the fix this would cause underflow */
	25	return 0;
	26	}
	27
	28	SEC("?tc")
	29	int underflow_prog(void *ctx)
	30	{
	31	struct prog_test_ref_kfunc *p;
	32	unsigned long sl = 0;
	33
	34	p = bpf_kfunc_call_test_acquire(&sl);
	35	if (!p)
	36	return 0;
	37	bpf_for_each_map_elem(&array_map, cb1, &p, 0);
	38	return 0;
	39	}
	40
	41	static __always_inline int cb2(void map, void key, void value, void ctx)
	42	{
	43	unsigned long sl = 0;
	44
	45	(void *)ctx = bpf_kfunc_call_test_acquire(&sl);
	46	/* Without the fix this would leak memory */
	47	return 0;
	48	}
	49
	50	SEC("?tc")
	51	int leak_prog(void *ctx)
	52	{
	53	struct prog_test_ref_kfunc *p;
	54	struct map_value *v;
	55	unsigned long sl;
	56
	57	v = bpf_map_lookup_elem(&array_map, &(int){0});
	58	if (!v)
	59	return 0;
	60
	61	p = NULL;
	62	bpf_for_each_map_elem(&array_map, cb2, &p, 0);
	63	p = bpf_kptr_xchg(&v->ptr, p);
	64	if (p)
	65	bpf_kfunc_call_test_release(p);
	66	return 0;
	67	}
	68
	69	static __always_inline int cb(void map, void key, void value, void ctx)
	70	{
	71	return 0;
	72	}
	73
	74	static __always_inline int cb3(void map, void key, void value, void ctx)
	75	{
	76	unsigned long sl = 0;
	77	void *p;
	78
	79	bpf_kfunc_call_test_acquire(&sl);
	80	bpf_for_each_map_elem(&array_map, cb, &p, 0);
	81	/* It should only complain here, not in cb. This is why we need
	82	* callback_ref to be set to frameno.
	83	*/
	84	return 0;
	85	}
	86
	87	SEC("?tc")
	88	int nested_cb(void *ctx)
	89	{
	90	struct prog_test_ref_kfunc *p;
	91	unsigned long sl = 0;
	92	int sp = 0;
	93
	94	p = bpf_kfunc_call_test_acquire(&sl);
	95	if (!p)
	96	return 0;
	97	bpf_for_each_map_elem(&array_map, cb3, &sp, 0);
	98	bpf_kfunc_call_test_release(p);
	99	return 0;
	100	}
	101
	102	SEC("?tc")
	103	int non_cb_transfer_ref(void *ctx)
	104	{
	105	struct prog_test_ref_kfunc *p;
	106	unsigned long sl = 0;
	107
	108	p = bpf_kfunc_call_test_acquire(&sl);
	109	if (!p)
	110	return 0;
	111	cb1(NULL, NULL, NULL, &p);
	112	bpf_kfunc_call_test_acquire(&sl);
	113	return 0;
	114	}
	115
	116	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.c+16−0

@@ -0,0 +1,16 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5
	6	#define BPF_RETVAL_HOOK(name, section, ctx, expected_err) \
	7	__attribute__((__section__("?" section))) \
	8	int name(struct ctx *_ctx) \
	9	{ \
	10	bpf_set_retval(bpf_get_retval()); \
	11	return 1; \
	12	}
	13
	14	#include "cgroup_getset_retval_hooks.h"
	15
	16	#undef BPF_RETVAL_HOOK

tools/testing/selftests/bpf/progs/cgroup_hierarchical_stats.c+226−0

@@ -0,0 +1,226 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Functions to manage eBPF programs attached to cgroup subsystems
	4	*
	5	* Copyright 2022 Google LLC.
	6	*/
	7	#include "vmlinux.h"
	8	#include <bpf/bpf_helpers.h>
	9	#include <bpf/bpf_tracing.h>
	10	#include <bpf/bpf_core_read.h>
	11
	12	char _license[] SEC("license") = "GPL";
	13
	14	/*
	15	* Start times are stored per-task, not per-cgroup, as multiple tasks in one
	16	* cgroup can perform reclaim concurrently.
	17	*/
	18	struct {
	19	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
	20	__uint(map_flags, BPF_F_NO_PREALLOC);
	21	__type(key, int);
	22	__type(value, __u64);
	23	} vmscan_start_time SEC(".maps");
	24
	25	struct vmscan_percpu {
	26	/* Previous percpu state, to figure out if we have new updates */
	27	__u64 prev;
	28	/* Current percpu state */
	29	__u64 state;
	30	};
	31
	32	struct vmscan {
	33	/* State propagated through children, pending aggregation */
	34	__u64 pending;
	35	/* Total state, including all cpus and all children */
	36	__u64 state;
	37	};
	38
	39	struct {
	40	__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
	41	__uint(max_entries, 100);
	42	__type(key, __u64);
	43	__type(value, struct vmscan_percpu);
	44	} pcpu_cgroup_vmscan_elapsed SEC(".maps");
	45
	46	struct {
	47	__uint(type, BPF_MAP_TYPE_HASH);
	48	__uint(max_entries, 100);
	49	__type(key, __u64);
	50	__type(value, struct vmscan);
	51	} cgroup_vmscan_elapsed SEC(".maps");
	52
	53	extern void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) __ksym;
	54	extern void cgroup_rstat_flush(struct cgroup *cgrp) __ksym;
	55
	56	static struct cgroup task_memcg(struct task_struct task)
	57	{
	58	int cgrp_id;
	59
	60	#if __has_builtin(__builtin_preserve_enum_value)
	61	cgrp_id = bpf_core_enum_value(enum cgroup_subsys_id, memory_cgrp_id);
	62	#else
	63	cgrp_id = memory_cgrp_id;
	64	#endif
	65	return task->cgroups->subsys[cgrp_id]->cgroup;
	66	}
	67
	68	static uint64_t cgroup_id(struct cgroup *cgrp)
	69	{
	70	return cgrp->kn->id;
	71	}
	72
	73	static int create_vmscan_percpu_elem(__u64 cg_id, __u64 state)
	74	{
	75	struct vmscan_percpu pcpu_init = {.state = state, .prev = 0};
	76
	77	return bpf_map_update_elem(&pcpu_cgroup_vmscan_elapsed, &cg_id,
	78	&pcpu_init, BPF_NOEXIST);
	79	}
	80
	81	static int create_vmscan_elem(__u64 cg_id, __u64 state, __u64 pending)
	82	{
	83	struct vmscan init = {.state = state, .pending = pending};
	84
	85	return bpf_map_update_elem(&cgroup_vmscan_elapsed, &cg_id,
	86	&init, BPF_NOEXIST);
	87	}
	88
	89	SEC("tp_btf/mm_vmscan_memcg_reclaim_begin")
	90	int BPF_PROG(vmscan_start, int order, gfp_t gfp_flags)
	91	{
	92	struct task_struct *task = bpf_get_current_task_btf();
	93	__u64 *start_time_ptr;
	94
	95	start_time_ptr = bpf_task_storage_get(&vmscan_start_time, task, 0,
	96	BPF_LOCAL_STORAGE_GET_F_CREATE);
	97	if (start_time_ptr)
	98	*start_time_ptr = bpf_ktime_get_ns();
	99	return 0;
	100	}
	101
	102	SEC("tp_btf/mm_vmscan_memcg_reclaim_end")
	103	int BPF_PROG(vmscan_end, unsigned long nr_reclaimed)
	104	{
	105	struct vmscan_percpu *pcpu_stat;
	106	struct task_struct *current = bpf_get_current_task_btf();
	107	struct cgroup *cgrp;
	108	__u64 *start_time_ptr;
	109	__u64 current_elapsed, cg_id;
	110	__u64 end_time = bpf_ktime_get_ns();
	111
	112	/*
	113	* cgrp is the first parent cgroup of current that has memcg enabled in
	114	* its subtree_control, or NULL if memcg is disabled in the entire tree.
	115	* In a cgroup hierarchy like this:
	116	* a
	117	* / \
	118	* b c
	119	* If "a" has memcg enabled, while "b" doesn't, then processes in "b"
	120	* will accumulate their stats directly to "a". This makes sure that no
	121	* stats are lost from processes in leaf cgroups that don't have memcg
	122	* enabled, but only exposes stats for cgroups that have memcg enabled.
	123	*/
	124	cgrp = task_memcg(current);
	125	if (!cgrp)
	126	return 0;
	127
	128	cg_id = cgroup_id(cgrp);
	129	start_time_ptr = bpf_task_storage_get(&vmscan_start_time, current, 0,
	130	BPF_LOCAL_STORAGE_GET_F_CREATE);
	131	if (!start_time_ptr)
	132	return 0;
	133
	134	current_elapsed = end_time - *start_time_ptr;
	135	pcpu_stat = bpf_map_lookup_elem(&pcpu_cgroup_vmscan_elapsed,
	136	&cg_id);
	137	if (pcpu_stat)
	138	pcpu_stat->state += current_elapsed;
	139	else if (create_vmscan_percpu_elem(cg_id, current_elapsed))
	140	return 0;
	141
	142	cgroup_rstat_updated(cgrp, bpf_get_smp_processor_id());
	143	return 0;
	144	}
	145
	146	SEC("fentry/bpf_rstat_flush")
	147	int BPF_PROG(vmscan_flush, struct cgroup cgrp, struct cgroup parent, int cpu)
	148	{
	149	struct vmscan_percpu *pcpu_stat;
	150	struct vmscan total_stat, parent_stat;
	151	__u64 cg_id = cgroup_id(cgrp);
	152	__u64 parent_cg_id = parent ? cgroup_id(parent) : 0;
	153	__u64 *pcpu_vmscan;
	154	__u64 state;
	155	__u64 delta = 0;
	156
	157	/* Add CPU changes on this level since the last flush */
	158	pcpu_stat = bpf_map_lookup_percpu_elem(&pcpu_cgroup_vmscan_elapsed,
	159	&cg_id, cpu);
	160	if (pcpu_stat) {
	161	state = pcpu_stat->state;
	162	delta += state - pcpu_stat->prev;
	163	pcpu_stat->prev = state;
	164	}
	165
	166	total_stat = bpf_map_lookup_elem(&cgroup_vmscan_elapsed, &cg_id);
	167	if (!total_stat) {
	168	if (create_vmscan_elem(cg_id, delta, 0))
	169	return 0;
	170
	171	goto update_parent;
	172	}
	173
	174	/* Collect pending stats from subtree */
	175	if (total_stat->pending) {
	176	delta += total_stat->pending;
	177	total_stat->pending = 0;
	178	}
	179
	180	/* Propagate changes to this cgroup's total */
	181	total_stat->state += delta;
	182
	183	update_parent:
	184	/* Skip if there are no changes to propagate, or no parent */
	185	if (!delta \|\| !parent_cg_id)
	186	return 0;
	187
	188	/* Propagate changes to cgroup's parent */
	189	parent_stat = bpf_map_lookup_elem(&cgroup_vmscan_elapsed,
	190	&parent_cg_id);
	191	if (parent_stat)
	192	parent_stat->pending += delta;
	193	else
	194	create_vmscan_elem(parent_cg_id, 0, delta);
	195	return 0;
	196	}
	197
	198	SEC("iter.s/cgroup")
	199	int BPF_PROG(dump_vmscan, struct bpf_iter_meta meta, struct cgroup cgrp)
	200	{
	201	struct seq_file *seq = meta->seq;
	202	struct vmscan *total_stat;
	203	__u64 cg_id = cgrp ? cgroup_id(cgrp) : 0;
	204
	205	/* Do nothing for the terminal call */
	206	if (!cg_id)
	207	return 1;
	208
	209	/* Flush the stats to make sure we get the most updated numbers */
	210	cgroup_rstat_flush(cgrp);
	211
	212	total_stat = bpf_map_lookup_elem(&cgroup_vmscan_elapsed, &cg_id);
	213	if (!total_stat) {
	214	BPF_SEQ_PRINTF(seq, "cg_id: %llu, total_vmscan_delay: 0\n",
	215	cg_id);
	216	} else {
	217	BPF_SEQ_PRINTF(seq, "cg_id: %llu, total_vmscan_delay: %llu\n",
	218	cg_id, total_stat->state);
	219	}
	220
	221	/*
	222	* We only dump stats for one cgroup here, so return 1 to stop
	223	* iteration after the first cgroup.
	224	*/
	225	return 1;
	226	}

tools/testing/selftests/bpf/progs/cgroup_iter.c+39−0

@@ -0,0 +1,39 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2022 Google */
	3
	4	#include "bpf_iter.h"
	5	#include <bpf/bpf_helpers.h>
	6	#include <bpf/bpf_tracing.h>
	7
	8	char _license[] SEC("license") = "GPL";
	9	int terminate_early = 0;
	10	u64 terminal_cgroup = 0;
	11
	12	static inline u64 cgroup_id(struct cgroup *cgrp)
	13	{
	14	return cgrp->kn->id;
	15	}
	16
	17	SEC("iter/cgroup")
	18	int cgroup_id_printer(struct bpf_iter__cgroup *ctx)
	19	{
	20	struct seq_file *seq = ctx->meta->seq;
	21	struct cgroup *cgrp = ctx->cgroup;
	22
	23	/* epilogue */
	24	if (cgrp == NULL) {
	25	BPF_SEQ_PRINTF(seq, "epilogue\n");
	26	return 0;
	27	}
	28
	29	/* prologue */
	30	if (ctx->meta->seq_num == 0)
	31	BPF_SEQ_PRINTF(seq, "prologue\n");
	32
	33	BPF_SEQ_PRINTF(seq, "%8llu\n", cgroup_id(cgrp));
	34
	35	if (terminal_cgroup == cgroup_id(cgrp))
	36	return 1;
	37
	38	return terminate_early ? 1 : 0;
	39	}

tools/testing/selftests/bpf/progs/connect4_prog.c+3−2

 #include <linux/bpf.h>
 #include <linux/in.h>
 #include <linux/in6.h>
-#include <sys/socket.h>
+#include <linux/tcp.h>
-#include <netinet/tcp.h>
 #include <linux/if.h>
 #include <errno.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_endian.h>
+#include "bpf_tcp_helpers.h"
 #define SRC_REWRITE_IP4		0x7f000004U
 #define DST_REWRITE_IP4		0x7f000001U
 #define DST_REWRITE_PORT4	4444

tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c+4−4

@@ -73,10 +73,10 @@ int test_subprog2(struct args_subprog2 *ctx) · 72 unmodified lines
73	__builtin_preserve_access_index(&skb->len));	73	__builtin_preserve_access_index(&skb->len));
74		74
75	ret = ctx->ret;	75	ret = ctx->ret;
76	/* bpf_prog_test_load() loads "test_pkt_access.o" with BPF_F_TEST_RND_HI32	76	/* bpf_prog_test_load() loads "test_pkt_access.bpf.o" with
77	* which randomizes upper 32 bits after BPF_ALU32 insns.	77	* BPF_F_TEST_RND_HI32 which randomizes upper 32 bits after BPF_ALU32
78	* Hence after 'w0 <<= 1' upper bits of $rax are random.	78	* insns. Hence after 'w0 <<= 1' upper bits of $rax are random. That is
79	* That is expected and correct. Trim them.	79	* expected and correct. Trim them.
80	*/	80	*/
81	ret = (__u32) ret;	81	ret = (__u32) ret;
82	if (len != 74 \|\| ret != 148)	82	if (len != 74 \|\| ret != 148)

tools/testing/selftests/bpf/progs/htab_update.c+29−0

@@ -0,0 +1,29 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include <bpf/bpf_tracing.h>
	6
	7	char _license[] SEC("license") = "GPL";
	8
	9	struct {
	10	__uint(type, BPF_MAP_TYPE_HASH);
	11	__uint(max_entries, 1);
	12	__uint(key_size, sizeof(__u32));
	13	__uint(value_size, sizeof(__u32));
	14	} htab SEC(".maps");
	15
	16	int pid = 0;
	17	int update_err = 0;
	18
	19	SEC("?fentry/lookup_elem_raw")
	20	int lookup_elem_raw(void *ctx)
	21	{
	22	__u32 key = 0, value = 1;
	23
	24	if ((bpf_get_current_pid_tgid() >> 32) != pid)
	25	return 0;
	26
	27	update_err = bpf_map_update_elem(&htab, &key, &value, 0);
	28	return 0;
	29	}

tools/testing/selftests/bpf/progs/read_bpf_task_storage_busy.c+39−0

@@ -0,0 +1,39 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
	3	#include "vmlinux.h"
	4	#include <bpf/bpf_helpers.h>
	5	#include <bpf/bpf_tracing.h>
	6
	7	extern bool CONFIG_PREEMPT __kconfig __weak;
	8	extern const int bpf_task_storage_busy __ksym;
	9
	10	char _license[] SEC("license") = "GPL";
	11
	12	int pid = 0;
	13	int busy = 0;
	14
	15	struct {
	16	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
	17	__uint(map_flags, BPF_F_NO_PREALLOC);
	18	__type(key, int);
	19	__type(value, long);
	20	} task SEC(".maps");
	21
	22	SEC("raw_tp/sys_enter")
	23	int BPF_PROG(read_bpf_task_storage_busy)
	24	{
	25	int *value;
	26	int key;
	27
	28	if (!CONFIG_PREEMPT)
	29	return 0;
	30
	31	if (bpf_get_current_pid_tgid() >> 32 != pid)
	32	return 0;
	33
	34	value = bpf_this_cpu_ptr(&bpf_task_storage_busy);
	35	if (value)
	36	busy = *value;
	37
	38	return 0;
	39	}

tools/testing/selftests/bpf/progs/setget_sockopt.c+395−0

@@ -0,0 +1,395 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) Meta Platforms, Inc. and affiliates. */
	3
	4	#include "vmlinux.h"
	5	#include "bpf_tracing_net.h"
	6	#include <bpf/bpf_core_read.h>
	7	#include <bpf/bpf_helpers.h>
	8	#include <bpf/bpf_tracing.h>
	9
	10	#ifndef ARRAY_SIZE
	11	#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
	12	#endif
	13
	14	extern unsigned long CONFIG_HZ __kconfig;
	15
	16	const volatile char veth[IFNAMSIZ];
	17	const volatile int veth_ifindex;
	18
	19	int nr_listen;
	20	int nr_passive;
	21	int nr_active;
	22	int nr_connect;
	23	int nr_binddev;
	24	int nr_socket_post_create;
	25
	26	struct sockopt_test {
	27	int opt;
	28	int new;
	29	int restore;
	30	int expected;
	31	int tcp_expected;
	32	unsigned int flip:1;
	33	};
	34
	35	static const char not_exist_cc[] = "not_exist";
	36	static const char cubic_cc[] = "cubic";
	37	static const char reno_cc[] = "reno";
	38
	39	static const struct sockopt_test sol_socket_tests[] = {
	40	{ .opt = SO_REUSEADDR, .flip = 1, },
	41	{ .opt = SO_SNDBUF, .new = 8123, .expected = 8123 * 2, },
	42	{ .opt = SO_RCVBUF, .new = 8123, .expected = 8123 * 2, },
	43	{ .opt = SO_KEEPALIVE, .flip = 1, },
	44	{ .opt = SO_PRIORITY, .new = 0xeb9f, .expected = 0xeb9f, },
	45	{ .opt = SO_REUSEPORT, .flip = 1, },
	46	{ .opt = SO_RCVLOWAT, .new = 8123, .expected = 8123, },
	47	{ .opt = SO_MARK, .new = 0xeb9f, .expected = 0xeb9f, },
	48	{ .opt = SO_MAX_PACING_RATE, .new = 0xeb9f, .expected = 0xeb9f, },
	49	{ .opt = SO_TXREHASH, .flip = 1, },
	50	{ .opt = 0, },
	51	};
	52
	53	static const struct sockopt_test sol_tcp_tests[] = {
	54	{ .opt = TCP_NODELAY, .flip = 1, },
	55	{ .opt = TCP_KEEPIDLE, .new = 123, .expected = 123, .restore = 321, },
	56	{ .opt = TCP_KEEPINTVL, .new = 123, .expected = 123, .restore = 321, },
	57	{ .opt = TCP_KEEPCNT, .new = 123, .expected = 123, .restore = 124, },
	58	{ .opt = TCP_SYNCNT, .new = 123, .expected = 123, .restore = 124, },
	59	{ .opt = TCP_WINDOW_CLAMP, .new = 8123, .expected = 8123, .restore = 8124, },
	60	{ .opt = TCP_CONGESTION, },
	61	{ .opt = TCP_THIN_LINEAR_TIMEOUTS, .flip = 1, },
	62	{ .opt = TCP_USER_TIMEOUT, .new = 123400, .expected = 123400, },
	63	{ .opt = TCP_NOTSENT_LOWAT, .new = 1314, .expected = 1314, },
	64	{ .opt = 0, },
	65	};
	66
	67	static const struct sockopt_test sol_ip_tests[] = {
	68	{ .opt = IP_TOS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, },
	69	{ .opt = 0, },
	70	};
	71
	72	static const struct sockopt_test sol_ipv6_tests[] = {
	73	{ .opt = IPV6_TCLASS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, },
	74	{ .opt = IPV6_AUTOFLOWLABEL, .flip = 1, },
	75	{ .opt = 0, },
	76	};
	77
	78	struct loop_ctx {
	79	void *ctx;
	80	struct sock *sk;
	81	};
	82
	83	static int bpf_test_sockopt_flip(void ctx, struct sock sk,
	84	const struct sockopt_test *t,
	85	int level)
	86	{
	87	int old, tmp, new, opt = t->opt;
	88
	89	opt = t->opt;
	90
	91	if (bpf_getsockopt(ctx, level, opt, &old, sizeof(old)))
	92	return 1;
	93	/* kernel initialized txrehash to 255 */
	94	if (level == SOL_SOCKET && opt == SO_TXREHASH && old != 0 && old != 1)
	95	old = 1;
	96
	97	new = !old;
	98	if (bpf_setsockopt(ctx, level, opt, &new, sizeof(new)))
	99	return 1;
	100	if (bpf_getsockopt(ctx, level, opt, &tmp, sizeof(tmp)) \|\|
	101	tmp != new)
	102	return 1;
	103
	104	if (bpf_setsockopt(ctx, level, opt, &old, sizeof(old)))
	105	return 1;
	106
	107	return 0;
	108	}
	109
	110	static int bpf_test_sockopt_int(void ctx, struct sock sk,
	111	const struct sockopt_test *t,
	112	int level)
	113	{
	114	int old, tmp, new, expected, opt;
	115
	116	opt = t->opt;
	117	new = t->new;
	118	if (sk->sk_type == SOCK_STREAM && t->tcp_expected)
	119	expected = t->tcp_expected;
	120	else
	121	expected = t->expected;
	122
	123	if (bpf_getsockopt(ctx, level, opt, &old, sizeof(old)) \|\|
	124	old == new)
	125	return 1;
	126
	127	if (bpf_setsockopt(ctx, level, opt, &new, sizeof(new)))
	128	return 1;
	129	if (bpf_getsockopt(ctx, level, opt, &tmp, sizeof(tmp)) \|\|
	130	tmp != expected)
	131	return 1;
	132
	133	if (t->restore)
	134	old = t->restore;
	135	if (bpf_setsockopt(ctx, level, opt, &old, sizeof(old)))
	136	return 1;
	137
	138	return 0;
	139	}
	140
	141	static int bpf_test_socket_sockopt(__u32 i, struct loop_ctx *lc)
	142	{
	143	const struct sockopt_test *t;
	144
	145	if (i >= ARRAY_SIZE(sol_socket_tests))
	146	return 1;
	147
	148	t = &sol_socket_tests[i];
	149	if (!t->opt)
	150	return 1;
	151
	152	if (t->flip)
	153	return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, SOL_SOCKET);
	154
	155	return bpf_test_sockopt_int(lc->ctx, lc->sk, t, SOL_SOCKET);
	156	}
	157
	158	static int bpf_test_ip_sockopt(__u32 i, struct loop_ctx *lc)
	159	{
	160	const struct sockopt_test *t;
	161
	162	if (i >= ARRAY_SIZE(sol_ip_tests))
	163	return 1;
	164
	165	t = &sol_ip_tests[i];
	166	if (!t->opt)
	167	return 1;
	168
	169	if (t->flip)
	170	return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, IPPROTO_IP);
	171
	172	return bpf_test_sockopt_int(lc->ctx, lc->sk, t, IPPROTO_IP);
	173	}
	174
	175	static int bpf_test_ipv6_sockopt(__u32 i, struct loop_ctx *lc)
	176	{
	177	const struct sockopt_test *t;
	178
	179	if (i >= ARRAY_SIZE(sol_ipv6_tests))
	180	return 1;
	181
	182	t = &sol_ipv6_tests[i];
	183	if (!t->opt)
	184	return 1;
	185
	186	if (t->flip)
	187	return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, IPPROTO_IPV6);
	188
	189	return bpf_test_sockopt_int(lc->ctx, lc->sk, t, IPPROTO_IPV6);
	190	}
	191
	192	static int bpf_test_tcp_sockopt(__u32 i, struct loop_ctx *lc)
	193	{
	194	const struct sockopt_test *t;
	195	struct sock *sk;
	196	void *ctx;
	197
	198	if (i >= ARRAY_SIZE(sol_tcp_tests))
	199	return 1;
	200
	201	t = &sol_tcp_tests[i];
	202	if (!t->opt)
	203	return 1;
	204
	205	ctx = lc->ctx;
	206	sk = lc->sk;
	207
	208	if (t->opt == TCP_CONGESTION) {
	209	char old_cc[16], tmp_cc[16];
	210	const char *new_cc;
	211	int new_cc_len;
	212
	213	if (!bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION,
	214	(void *)not_exist_cc, sizeof(not_exist_cc)))
	215	return 1;
	216	if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, old_cc, sizeof(old_cc)))
	217	return 1;
	218	if (!bpf_strncmp(old_cc, sizeof(old_cc), cubic_cc)) {
	219	new_cc = reno_cc;
	220	new_cc_len = sizeof(reno_cc);
	221	} else {
	222	new_cc = cubic_cc;
	223	new_cc_len = sizeof(cubic_cc);
	224	}
	225	if (bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, (void *)new_cc,
	226	new_cc_len))
	227	return 1;
	228	if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, tmp_cc, sizeof(tmp_cc)))
	229	return 1;
	230	if (bpf_strncmp(tmp_cc, sizeof(tmp_cc), new_cc))
	231	return 1;
	232	if (bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, old_cc, sizeof(old_cc)))
	233	return 1;
	234	return 0;
	235	}
	236
	237	if (t->flip)
	238	return bpf_test_sockopt_flip(ctx, sk, t, IPPROTO_TCP);
	239
	240	return bpf_test_sockopt_int(ctx, sk, t, IPPROTO_TCP);
	241	}
	242
	243	static int bpf_test_sockopt(void ctx, struct sock sk)
	244	{
	245	struct loop_ctx lc = { .ctx = ctx, .sk = sk, };
	246	__u16 family, proto;
	247	int n;
	248
	249	family = sk->sk_family;
	250	proto = sk->sk_protocol;
	251
	252	n = bpf_loop(ARRAY_SIZE(sol_socket_tests), bpf_test_socket_sockopt, &lc, 0);
	253	if (n != ARRAY_SIZE(sol_socket_tests))
	254	return -1;
	255
	256	if (proto == IPPROTO_TCP) {
	257	n = bpf_loop(ARRAY_SIZE(sol_tcp_tests), bpf_test_tcp_sockopt, &lc, 0);
	258	if (n != ARRAY_SIZE(sol_tcp_tests))
	259	return -1;
	260	}
	261
	262	if (family == AF_INET) {
	263	n = bpf_loop(ARRAY_SIZE(sol_ip_tests), bpf_test_ip_sockopt, &lc, 0);
	264	if (n != ARRAY_SIZE(sol_ip_tests))
	265	return -1;
	266	} else {
	267	n = bpf_loop(ARRAY_SIZE(sol_ipv6_tests), bpf_test_ipv6_sockopt, &lc, 0);
	268	if (n != ARRAY_SIZE(sol_ipv6_tests))
	269	return -1;
	270	}
	271
	272	return 0;
	273	}
	274
	275	static int binddev_test(void *ctx)
	276	{
	277	const char empty_ifname[] = "";
	278	int ifindex, zero = 0;
	279
	280	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
	281	(void *)veth, sizeof(veth)))
	282	return -1;
	283	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
	284	&ifindex, sizeof(int)) \|\|
	285	ifindex != veth_ifindex)
	286	return -1;
	287
	288	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
	289	(void *)empty_ifname, sizeof(empty_ifname)))
	290	return -1;
	291	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
	292	&ifindex, sizeof(int)) \|\|
	293	ifindex != 0)
	294	return -1;
	295
	296	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
	297	(void *)&veth_ifindex, sizeof(int)))
	298	return -1;
	299	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
	300	&ifindex, sizeof(int)) \|\|
	301	ifindex != veth_ifindex)
	302	return -1;
	303
	304	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
	305	&zero, sizeof(int)))
	306	return -1;
	307	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
	308	&ifindex, sizeof(int)) \|\|
	309	ifindex != 0)
	310	return -1;
	311
	312	return 0;
	313	}
	314
	315	static int test_tcp_maxseg(void ctx, struct sock sk)
	316	{
	317	int val = 1314, tmp;
	318
	319	if (sk->sk_state != TCP_ESTABLISHED)
	320	return bpf_setsockopt(ctx, IPPROTO_TCP, TCP_MAXSEG,
	321	&val, sizeof(val));
	322
	323	if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_MAXSEG, &tmp, sizeof(tmp)) \|\|
	324	tmp > val)
	325	return -1;
	326
	327	return 0;
	328	}
	329
	330	static int test_tcp_saved_syn(void ctx, struct sock sk)
	331	{
	332	__u8 saved_syn[20];
	333	int one = 1;
	334
	335	if (sk->sk_state == TCP_LISTEN)
	336	return bpf_setsockopt(ctx, IPPROTO_TCP, TCP_SAVE_SYN,
	337	&one, sizeof(one));
	338
	339	return bpf_getsockopt(ctx, IPPROTO_TCP, TCP_SAVED_SYN,
	340	saved_syn, sizeof(saved_syn));
	341	}
	342
	343	SEC("lsm_cgroup/socket_post_create")
	344	int BPF_PROG(socket_post_create, struct socket *sock, int family,
	345	int type, int protocol, int kern)
	346	{
	347	struct sock *sk = sock->sk;
	348
	349	if (!sk)
	350	return 1;
	351
	352	nr_socket_post_create += !bpf_test_sockopt(sk, sk);
	353	nr_binddev += !binddev_test(sk);
	354
	355	return 1;
	356	}
	357
	358	SEC("sockops")
	359	int skops_sockopt(struct bpf_sock_ops *skops)
	360	{
	361	struct bpf_sock *bpf_sk = skops->sk;
	362	struct sock *sk;
	363
	364	if (!bpf_sk)
	365	return 1;
	366
	367	sk = (struct sock *)bpf_skc_to_tcp_sock(bpf_sk);
	368	if (!sk)
	369	return 1;
	370
	371	switch (skops->op) {
	372	case BPF_SOCK_OPS_TCP_LISTEN_CB:
	373	nr_listen += !(bpf_test_sockopt(skops, sk) \|\|
	374	test_tcp_maxseg(skops, sk) \|\|
	375	test_tcp_saved_syn(skops, sk));
	376	break;
	377	case BPF_SOCK_OPS_TCP_CONNECT_CB:
	378	nr_connect += !(bpf_test_sockopt(skops, sk) \|\|
	379	test_tcp_maxseg(skops, sk));
	380	break;
	381	case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:
	382	nr_active += !(bpf_test_sockopt(skops, sk) \|\|
	383	test_tcp_maxseg(skops, sk));
	384	break;
	385	case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
	386	nr_passive += !(bpf_test_sockopt(skops, sk) \|\|
	387	test_tcp_maxseg(skops, sk) \|\|
	388	test_tcp_saved_syn(skops, sk));
	389	break;
	390	}
	391
	392	return 1;
	393	}
	394
	395	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c+1−1

@@ -19,7 +19,7 @@ struct { · 18 unmodified lines
19	int count = 0;	19	int count = 0;
20	int noise = 0;	20	int noise = 0;
21		21
22	__always_inline int subprog_noise(void)	22	static __always_inline int subprog_noise(void)
23	{	23	{
24	__u32 key = 0;	24	__u32 key = 0;
25		25

tools/testing/selftests/bpf/progs/test_tc_dtime.c+0−1

@@ -15,7 +15,6 @@ · 14 unmodified lines
15	#include <linux/udp.h>	15	#include <linux/udp.h>
16	#include <bpf/bpf_helpers.h>	16	#include <bpf/bpf_helpers.h>
17	#include <bpf/bpf_endian.h>	17	#include <bpf/bpf_endian.h>
18	#include <sys/socket.h>
19		18
20	/* veth_src --- veth_src_fwd --- veth_det_fwd --- veth_dst	19	/* veth_src --- veth_src_fwd --- veth_det_fwd --- veth_dst
21	* \| \|	20	* \| \|

tools/testing/selftests/bpf/progs/test_tunnel_kern.c+16−8

 #include <linux/bpf.h>
 #include <linux/if_ether.h>
 #include <linux/if_packet.h>
+#include <linux/if_tunnel.h>
 #include <linux/ip.h>
 #include <linux/ipv6.h>
 #include <linux/icmp.h>
 	__u32 orig_daddr;
 	__u32 index = 0;
-	ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0);
+	ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key),
+				     BPF_F_TUNINFO_FLAGS);
 	if (ret < 0) {
 		log_err(ret);
 		return TC_ACT_SHOT;
 		return TC_ACT_SHOT;
 	}
-	if (key.local_ipv4 != ASSIGNED_ADDR_VETH1 || md.gbp != 0x800FF) {
+	if (key.local_ipv4 != ASSIGNED_ADDR_VETH1 || md.gbp != 0x800FF ||
-		bpf_printk("vxlan key %d local ip 0x%x remote ip 0x%x gbp 0x%x\n",
+	    !(key.tunnel_flags & TUNNEL_KEY) ||
+	    (key.tunnel_flags & TUNNEL_CSUM)) {
+		bpf_printk("vxlan key %d local ip 0x%x remote ip 0x%x gbp 0x%x flags 0x%x\n",
 			   key.tunnel_id, key.local_ipv4,
-			   key.remote_ipv4, md.gbp);
+			   key.remote_ipv4, md.gbp,
+			   bpf_ntohs(key.tunnel_flags));
 		log_err(ret);
 		return TC_ACT_SHOT;
 	}
 	}
 	ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key),
-				     BPF_F_TUNINFO_IPV6);
+				     BPF_F_TUNINFO_IPV6 | BPF_F_TUNINFO_FLAGS);
 	if (ret < 0) {
 		log_err(ret);
 		return TC_ACT_SHOT;
 	}
-	if (bpf_ntohl(key.local_ipv6[3]) != *local_ip) {
+	if (bpf_ntohl(key.local_ipv6[3]) != *local_ip ||
-		bpf_printk("ip6vxlan key %d local ip6 ::%x remote ip6 ::%x label 0x%x\n",
+	    !(key.tunnel_flags & TUNNEL_KEY) ||
+	    !(key.tunnel_flags & TUNNEL_CSUM)) {
+		bpf_printk("ip6vxlan key %d local ip6 ::%x remote ip6 ::%x label 0x%x flags 0x%x\n",
 			   key.tunnel_id, bpf_ntohl(key.local_ipv6[3]),
-			   bpf_ntohl(key.remote_ipv6[3]), key.tunnel_label);
+			   bpf_ntohl(key.remote_ipv6[3]), key.tunnel_label,
+			   bpf_ntohs(key.tunnel_flags));
 		bpf_printk("local_ip 0x%x\n", *local_ip);
 		log_err(ret);
 		return TC_ACT_SHOT;

tools/testing/selftests/bpf/progs/timer.c+0−11

@@ -208,17 +208,6 @@ static int timer_cb2(void map, int key, struct hmap_elem *val) · 207 unmodified lines
208	*/	208	*/
209	bpf_map_delete_elem(map, key);	209	bpf_map_delete_elem(map, key);
210		210
211	/* in non-preallocated hashmap both 'key' and 'val' are RCU
212	* protected and still valid though this element was deleted
213	* from the map. Arm this timer for ~35 seconds. When callback
214	* finishes the call_rcu will invoke:
215	* htab_elem_free_rcu
216	* check_and_free_timer
217	* bpf_timer_cancel_and_free
218	* to cancel this 35 second sleep and delete the timer for real.
219	*/
220	if (bpf_timer_start(&val->timer, 1ull << 35, 0) != 0)
221	err \|= 256;
222	ok \|= 4;	211	ok \|= 4;
223	}	212	}
224	return 0;	213	return 0;

tools/testing/selftests/bpf/task_local_storage_helpers.h+18−0

@@ -0,0 +1,18 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __TASK_LOCAL_STORAGE_HELPER_H
	3	#define __TASK_LOCAL_STORAGE_HELPER_H
	4
	5	#include <unistd.h>
	6	#include <sys/syscall.h>
	7	#include <sys/types.h>
	8
	9	#ifndef __NR_pidfd_open
	10	#define __NR_pidfd_open 434
	11	#endif
	12
	13	static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
	14	{
	15	return syscall(__NR_pidfd_open, pid, flags);
	16	}
	17
	18	#endif

tools/testing/selftests/bpf/test_dev_cgroup.c+1−1

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	#include "cgroup_helpers.h"	16	#include "cgroup_helpers.h"
17	#include "testing_helpers.h"	17	#include "testing_helpers.h"
18		18
19	#define DEV_CGROUP_PROG "./dev_cgroup.o"	19	#define DEV_CGROUP_PROG "./dev_cgroup.bpf.o"
20		20
21	#define TEST_CGROUP "/test-bpf-based-device-cgroup/"	21	#define TEST_CGROUP "/test-bpf-based-device-cgroup/"
22		22

tools/testing/selftests/bpf/test_flow_dissector.sh+8−0

@@ -115,6 +115,14 @@ tc filter add dev lo parent ffff: protocol ip pref 1337 flower ip_proto \ · 114 unmodified lines
115	# Send 10 IPv4/UDP packets from port 10. Filter should not drop any.	115	# Send 10 IPv4/UDP packets from port 10. Filter should not drop any.
116	./test_flow_dissector -i 4 -f 10	116	./test_flow_dissector -i 4 -f 10
117		117
		118	echo "Testing IPv4 from 127.0.0.127 (fallback to generic dissector)..."
		119	# Send 10 IPv4/UDP packets from port 8. Filter should not drop any.
		120	./test_flow_dissector -i 4 -S 127.0.0.127 -f 8
		121	# Send 10 IPv4/UDP packets from port 9. Filter should drop all.
		122	./test_flow_dissector -i 4 -S 127.0.0.127 -f 9 -F
		123	# Send 10 IPv4/UDP packets from port 10. Filter should not drop any.
		124	./test_flow_dissector -i 4 -S 127.0.0.127 -f 10
		125
118	echo "Testing IPIP..."	126	echo "Testing IPIP..."
119	# Send 10 IPv4/IPv4/UDP packets from port 8. Filter should not drop any.	127	# Send 10 IPv4/IPv4/UDP packets from port 8. Filter should not drop any.
120	./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e bare -i 4 \	128	./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e bare -i 4 \

tools/testing/selftests/bpf/test_lirc_mode2_user.c+1−1

@@ -59,7 +59,7 @@ int main(int argc, char **argv) · 58 unmodified lines
59	return 2;	59	return 2;
60	}	60	}
61		61
62	ret = bpf_prog_test_load("test_lirc_mode2_kern.o",	62	ret = bpf_prog_test_load("test_lirc_mode2_kern.bpf.o",
63	BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd);	63	BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd);
64	if (ret) {	64	if (ret) {
65	printf("Failed to load bpf program\n");	65	printf("Failed to load bpf program\n");

tools/testing/selftests/bpf/test_maps.c+29−19

 	close(fd);
 }
+#define VALUE_SIZE 3
 static int helper_fill_hashmap(int max_entries)
 {
 	int i, fd, ret;
-	long long key, value;
+	long long key, value[VALUE_SIZE] = {};
 	fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),
 			    max_entries, &map_opts);
 	      "err: %s, flags: 0x%x\n", strerror(errno), map_opts.map_flags);
 	for (i = 0; i < max_entries; i++) {
-		key = i; value = key;
+		key = i; value[0] = key;
-		ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST);
+		ret = bpf_map_update_elem(fd, &key, value, BPF_NOEXIST);
 		CHECK(ret != 0,
 		      "can't update hashmap",
 		      "err: %s\n", strerror(ret));
 static void test_hashmap_walk(unsigned int task, void *data)
 {
-	int fd, i, max_entries = 1000;
+	int fd, i, max_entries = 10000;
-	long long key, value, next_key;
+	long long key, value[VALUE_SIZE], next_key;
 	bool next_key_valid = true;
 	fd = helper_fill_hashmap(max_entries);
 	for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,
 					 &next_key) == 0; i++) {
 		key = next_key;
-		assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
+		assert(bpf_map_lookup_elem(fd, &key, value) == 0);
 	}
 	assert(i == max_entries);
 	assert(bpf_map_get_next_key(fd, NULL, &key) == 0);
 	for (i = 0; next_key_valid; i++) {
 		next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0;
-		assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
+		assert(bpf_map_lookup_elem(fd, &key, value) == 0);
-		value++;
+		value[0]++;
-		assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0);
+		assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0);
 		key = next_key;
 	}
 	for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,
 					 &next_key) == 0; i++) {
 		key = next_key;
-		assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
+		assert(bpf_map_lookup_elem(fd, &key, value) == 0);
-		assert(value - 1 == key);
+		assert(value[0] - 1 == key);
 	}
 	assert(i == max_entries);
 #include <arpa/inet.h>
 #include <sys/select.h>
 #include <linux/err.h>
-#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.o"
+#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.bpf.o"
-#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o"
+#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.bpf.o"
-#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.o"
+#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.bpf.o"
 static void test_sockmap(unsigned int tasks, void *data)
 {
 	struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break;
 	exit(1);
 }
-#define MAPINMAP_PROG "./test_map_in_map.o"
+#define MAPINMAP_PROG "./test_map_in_map.bpf.o"
-#define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.o"
+#define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.bpf.o"
 static void test_map_in_map(void)
 {
 	struct bpf_object *obj;
 static void test_map_stress(void)
 {
+	run_parallel(100, test_hashmap_walk, NULL);
 	run_parallel(100, test_hashmap, NULL);
 	run_parallel(100, test_hashmap_percpu, NULL);
 	run_parallel(100, test_hashmap_sizes, NULL);
-	run_parallel(100, test_hashmap_walk, NULL);
 	run_parallel(100, test_arraymap, NULL);
 	run_parallel(100, test_arraymap_percpu, NULL);
 }
-#define TASKS 1024
+#define TASKS 100
 #define DO_UPDATE 1
 #define DO_DELETE 0
 	int fd = ((int *)data)[0];
 	int i, key, value, err;
+	if (fn & 1)
+		test_hashmap_walk(fn, NULL);
 	for (i = fn; i < MAP_SIZE; i += TASKS) {
 		key = value = i;
 static void test_map_parallel(void)
 {
-	int i, fd, key = 0, value = 0;
+	int i, fd, key = 0, value = 0, j = 0;
 	int data[2];
 	fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),
 		exit(1);
 	}
+again:
 	/* Use the same fd in children to add elements to this map:
 	 * child_0 adds key=0, key=1024, key=2048, ...
 	 * child_1 adds key=1, key=1025, key=2049, ...
 	key = -1;
 	assert(bpf_map_get_next_key(fd, NULL, &key) < 0 && errno == ENOENT);
 	assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT);
+	key = 0;
+	bpf_map_delete_elem(fd, &key);
+	if (j++ < 5)
+		goto again;
+	close(fd);
 }
 static void test_map_rdonly(void)

tools/testing/selftests/bpf/test_offload.py+11−11

@@ -782,7 +782,7 @@ if out.find("/sys/kernel/debug type debugfs") == -1: · 781 unmodified lines
782	cmd("mount -t debugfs none /sys/kernel/debug")	782	cmd("mount -t debugfs none /sys/kernel/debug")
783		783
784	# Check samples are compiled	784	# Check samples are compiled
785	samples = ["sample_ret0.o", "sample_map_ret0.o"]	785	samples = ["sample_ret0.bpf.o", "sample_map_ret0.bpf.o"]
786	for s in samples:	786	for s in samples:
787	ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)	787	ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)
788	skip(ret != 0, "sample %s/%s not found, please compile it" %	788	skip(ret != 0, "sample %s/%s not found, please compile it" %
@@ -803,7 +803,7 @@ cmd("ip netns delete %s" % (ns)) · 14 unmodified lines
803	netns = []	803	netns = []
804		804
805	try:	805	try:
806	obj = bpf_obj("sample_ret0.o")	806	obj = bpf_obj("sample_ret0.bpf.o")
807	bytecode = bpf_bytecode("1,6 0 0 4294967295,")	807	bytecode = bpf_bytecode("1,6 0 0 4294967295,")
808		808
809	start_test("Test destruction of generic XDP...")	809	start_test("Test destruction of generic XDP...")
@@ -1023,7 +1023,7 @@ try: · 213 unmodified lines
1023		1023
1024	sim.wait_for_flush()	1024	sim.wait_for_flush()
1025	start_test("Test non-offload XDP attaching to HW...")	1025	start_test("Test non-offload XDP attaching to HW...")
1026	bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload")	1026	bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/nooffload")
1027	nooffload = bpf_pinned("/sys/fs/bpf/nooffload")	1027	nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
1028	ret, _, err = sim.set_xdp(nooffload, "offload",	1028	ret, _, err = sim.set_xdp(nooffload, "offload",
1029	fail=False, include_stderr=True)	1029	fail=False, include_stderr=True)
@@ -1032,7 +1032,7 @@ try: · 2 unmodified lines
1032	rm("/sys/fs/bpf/nooffload")	1032	rm("/sys/fs/bpf/nooffload")
1033		1033
1034	start_test("Test offload XDP attaching to drv...")	1034	start_test("Test offload XDP attaching to drv...")
1035	bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",	1035	bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload",
1036	dev=sim['ifname'])	1036	dev=sim['ifname'])
1037	offload = bpf_pinned("/sys/fs/bpf/offload")	1037	offload = bpf_pinned("/sys/fs/bpf/offload")
1038	ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)	1038	ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
@@ -1043,7 +1043,7 @@ try: · 4 unmodified lines
1043		1043
1044	start_test("Test XDP load failure...")	1044	start_test("Test XDP load failure...")
1045	sim.dfs["dev/bpf_bind_verifier_accept"] = 0	1045	sim.dfs["dev/bpf_bind_verifier_accept"] = 0
1046	ret, _, err = bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",	1046	ret, _, err = bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload",
1047	dev=sim['ifname'], fail=False, include_stderr=True)	1047	dev=sim['ifname'], fail=False, include_stderr=True)
1048	fail(ret == 0, "verifier should fail on load")	1048	fail(ret == 0, "verifier should fail on load")
1049	check_verifier_log(err, "[netdevsim] Hello from netdevsim!")	1049	check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
@@ -1169,7 +1169,7 @@ try: · 119 unmodified lines
1169		1169
1170	simdev = NetdevSimDev()	1170	simdev = NetdevSimDev()
1171	sim, = simdev.nsims	1171	sim, = simdev.nsims
1172	map_obj = bpf_obj("sample_map_ret0.o")	1172	map_obj = bpf_obj("sample_map_ret0.bpf.o")
1173	start_test("Test loading program with maps...")	1173	start_test("Test loading program with maps...")
1174	sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON	1174	sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
1175		1175
@@ -1307,10 +1307,10 @@ try: · 131 unmodified lines
1307	sims = (simA, simB1, simB2, simB3)	1307	sims = (simA, simB1, simB2, simB3)
1308	simB = (simB1, simB2, simB3)	1308	simB = (simB1, simB2, simB3)
1309		1309
1310	bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA",	1310	bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA",
1311	dev=simA['ifname'])	1311	dev=simA['ifname'])
1312	progA = bpf_pinned("/sys/fs/bpf/nsimA")	1312	progA = bpf_pinned("/sys/fs/bpf/nsimA")
1313	bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB",	1313	bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB",
1314	dev=simB1['ifname'])	1314	dev=simB1['ifname'])
1315	progB = bpf_pinned("/sys/fs/bpf/nsimB")	1315	progB = bpf_pinned("/sys/fs/bpf/nsimB")
1316		1316
@@ -1344,14 +1344,14 @@ try: · 27 unmodified lines
1344	mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]	1344	mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
1345	mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]	1345	mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
1346		1346
1347	ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",	1347	ret, _ = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_",
1348	dev=simB3['ifname'],	1348	dev=simB3['ifname'],
1349	maps=["idx 0 id %d" % (mapB)],	1349	maps=["idx 0 id %d" % (mapB)],
1350	fail=False)	1350	fail=False)
1351	fail(ret != 0, "couldn't reuse a map on the same ASIC")	1351	fail(ret != 0, "couldn't reuse a map on the same ASIC")
1352	rm("/sys/fs/bpf/nsimB_")	1352	rm("/sys/fs/bpf/nsimB_")
1353		1353
1354	ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_",	1354	ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA_",
1355	dev=simA['ifname'],	1355	dev=simA['ifname'],
1356	maps=["idx 0 id %d" % (mapB)],	1356	maps=["idx 0 id %d" % (mapB)],
1357	fail=False, include_stderr=True)	1357	fail=False, include_stderr=True)
@@ -1359,7 +1359,7 @@ try: · 1 unmodified line
1359	fail(err.count("offload device mismatch between prog and map") == 0,	1359	fail(err.count("offload device mismatch between prog and map") == 0,
1360	"error message missing for cross-ASIC map")	1360	"error message missing for cross-ASIC map")
1361		1361
1362	ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",	1362	ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_",
1363	dev=simB1['ifname'],	1363	dev=simB1['ifname'],
1364	maps=["idx 0 id %d" % (mapA)],	1364	maps=["idx 0 id %d" % (mapA)],
1365	fail=False, include_stderr=True)	1365	fail=False, include_stderr=True)

tools/testing/selftests/bpf/test_skb_cgroup_id.sh+1−1

@@ -54,7 +54,7 @@ DIR=$(dirname $0) · 53 unmodified lines
54	TEST_IF="test_cgid_1"	54	TEST_IF="test_cgid_1"
55	TEST_IF_PEER="test_cgid_2"	55	TEST_IF_PEER="test_cgid_2"
56	MAX_PING_TRIES=5	56	MAX_PING_TRIES=5
57	BPF_PROG_OBJ="${DIR}/test_skb_cgroup_id_kern.o"	57	BPF_PROG_OBJ="${DIR}/test_skb_cgroup_id_kern.bpf.o"
58	BPF_PROG_SECTION="cgroup_id_logger"	58	BPF_PROG_SECTION="cgroup_id_logger"
59	BPF_PROG_ID=0	59	BPF_PROG_ID=0
60	PROG="${DIR}/test_skb_cgroup_id_user"	60	PROG="${DIR}/test_skb_cgroup_id_user"

tools/testing/selftests/bpf/test_sock_addr.c+8−8

@@ -26,14 +26,14 @@ · 25 unmodified lines
26	#endif	26	#endif
27		27
28	#define CG_PATH "/foo"	28	#define CG_PATH "/foo"
29	#define CONNECT4_PROG_PATH "./connect4_prog.o"	29	#define CONNECT4_PROG_PATH "./connect4_prog.bpf.o"
30	#define CONNECT6_PROG_PATH "./connect6_prog.o"	30	#define CONNECT6_PROG_PATH "./connect6_prog.bpf.o"
31	#define SENDMSG4_PROG_PATH "./sendmsg4_prog.o"	31	#define SENDMSG4_PROG_PATH "./sendmsg4_prog.bpf.o"
32	#define SENDMSG6_PROG_PATH "./sendmsg6_prog.o"	32	#define SENDMSG6_PROG_PATH "./sendmsg6_prog.bpf.o"
33	#define RECVMSG4_PROG_PATH "./recvmsg4_prog.o"	33	#define RECVMSG4_PROG_PATH "./recvmsg4_prog.bpf.o"
34	#define RECVMSG6_PROG_PATH "./recvmsg6_prog.o"	34	#define RECVMSG6_PROG_PATH "./recvmsg6_prog.bpf.o"
35	#define BIND4_PROG_PATH "./bind4_prog.o"	35	#define BIND4_PROG_PATH "./bind4_prog.bpf.o"
36	#define BIND6_PROG_PATH "./bind6_prog.o"	36	#define BIND6_PROG_PATH "./bind6_prog.bpf.o"
37		37
38	#define SERV4_IP "192.168.1.254"	38	#define SERV4_IP "192.168.1.254"
39	#define SERV4_REWRITE_IP "127.0.0.1"	39	#define SERV4_REWRITE_IP "127.0.0.1"

tools/testing/selftests/bpf/test_sockmap.c+2−2

@@ -52,8 +52,8 @@ static void running_handler(int a); · 51 unmodified lines
52	#define S1_PORT 10000	52	#define S1_PORT 10000
53	#define S2_PORT 10001	53	#define S2_PORT 10001
54		54
55	#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"	55	#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.bpf.o"
56	#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"	56	#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
57	#define CG_PATH "/sockmap"	57	#define CG_PATH "/sockmap"
58		58
59	/* global sockets */	59	/* global sockets */

tools/testing/selftests/bpf/test_sysctl.c+3−3

@@ -1372,7 +1372,7 @@ static struct sysctl_test tests[] = { · 1371 unmodified lines
1372	},	1372	},
1373	{	1373	{
1374	"C prog: deny all writes",	1374	"C prog: deny all writes",
1375	.prog_file = "./test_sysctl_prog.o",	1375	.prog_file = "./test_sysctl_prog.bpf.o",
1376	.attach_type = BPF_CGROUP_SYSCTL,	1376	.attach_type = BPF_CGROUP_SYSCTL,
1377	.sysctl = "net/ipv4/tcp_mem",	1377	.sysctl = "net/ipv4/tcp_mem",
1378	.open_flags = O_WRONLY,	1378	.open_flags = O_WRONLY,
@@ -1381,7 +1381,7 @@ static struct sysctl_test tests[] = { · 2 unmodified lines
1381	},	1381	},
1382	{	1382	{
1383	"C prog: deny access by name",	1383	"C prog: deny access by name",
1384	.prog_file = "./test_sysctl_prog.o",	1384	.prog_file = "./test_sysctl_prog.bpf.o",
1385	.attach_type = BPF_CGROUP_SYSCTL,	1385	.attach_type = BPF_CGROUP_SYSCTL,
1386	.sysctl = "net/ipv4/route/mtu_expires",	1386	.sysctl = "net/ipv4/route/mtu_expires",
1387	.open_flags = O_RDONLY,	1387	.open_flags = O_RDONLY,
@@ -1389,7 +1389,7 @@ static struct sysctl_test tests[] = { · 1 unmodified line
1389	},	1389	},
1390	{	1390	{
1391	"C prog: read tcp_mem",	1391	"C prog: read tcp_mem",
1392	.prog_file = "./test_sysctl_prog.o",	1392	.prog_file = "./test_sysctl_prog.bpf.o",
1393	.attach_type = BPF_CGROUP_SYSCTL,	1393	.attach_type = BPF_CGROUP_SYSCTL,
1394	.sysctl = "net/ipv4/tcp_mem",	1394	.sysctl = "net/ipv4/tcp_mem",
1395	.open_flags = O_RDONLY,	1395	.open_flags = O_RDONLY,

tools/testing/selftests/bpf/test_tcp_check_syncookie.sh+1−1

@@ -76,7 +76,7 @@ main() · 75 unmodified lines
76	DIR=$(dirname $0)	76	DIR=$(dirname $0)
77	TEST_IF=lo	77	TEST_IF=lo
78	MAX_PING_TRIES=5	78	MAX_PING_TRIES=5
79	BPF_PROG_OBJ="${DIR}/test_tcp_check_syncookie_kern.o"	79	BPF_PROG_OBJ="${DIR}/test_tcp_check_syncookie_kern.bpf.o"
80	CLSACT_SECTION="tc"	80	CLSACT_SECTION="tc"
81	XDP_SECTION="xdp"	81	XDP_SECTION="xdp"
82	BPF_PROG_ID=0	82	BPF_PROG_ID=0

tools/testing/selftests/bpf/test_tcpnotify_user.c+1−1

@@ -69,7 +69,7 @@ int verify_result(const struct tcpnotify_globals *result) · 68 unmodified lines
69		69
70	int main(int argc, char **argv)	70	int main(int argc, char **argv)
71	{	71	{
72	const char *file = "test_tcpnotify_kern.o";	72	const char *file = "test_tcpnotify_kern.bpf.o";
73	struct bpf_map perf_map, global_map;	73	struct bpf_map perf_map, global_map;
74	struct tcpnotify_globals g = {0};	74	struct tcpnotify_globals g = {0};
75	struct perf_buffer *pb = NULL;	75	struct perf_buffer *pb = NULL;

tools/testing/selftests/bpf/test_xdp_redirect.sh+4−4

@@ -54,10 +54,10 @@ test_xdp_redirect() · 53 unmodified lines
54	return 0	54	return 0
55	fi	55	fi
56		56
57	ip -n ${NS1} link set veth11 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null	57	ip -n ${NS1} link set veth11 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null
58	ip -n ${NS2} link set veth22 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null	58	ip -n ${NS2} link set veth22 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null
59	ip link set dev veth1 $xdpmode obj test_xdp_redirect.o sec redirect_to_222 &> /dev/null	59	ip link set dev veth1 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_222 &> /dev/null
60	ip link set dev veth2 $xdpmode obj test_xdp_redirect.o sec redirect_to_111 &> /dev/null	60	ip link set dev veth2 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_111 &> /dev/null
61		61
62	if ip netns exec ${NS1} ping -c 1 10.1.1.22 &> /dev/null &&	62	if ip netns exec ${NS1} ping -c 1 10.1.1.22 &> /dev/null &&
63	ip netns exec ${NS2} ping -c 1 10.1.1.11 &> /dev/null; then	63	ip netns exec ${NS2} ping -c 1 10.1.1.11 &> /dev/null; then

tools/testing/selftests/bpf/test_xdp_redirect_multi.sh+1−1

@@ -94,7 +94,7 @@ setup_ns() · 93 unmodified lines
94	# Add a neigh entry for IPv4 ping test	94	# Add a neigh entry for IPv4 ping test
95	ip -n ${NS[$i]} neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0	95	ip -n ${NS[$i]} neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0
96	ip -n ${NS[$i]} link set veth0 $mode obj \	96	ip -n ${NS[$i]} link set veth0 $mode obj \
97	xdp_dummy.o sec xdp &> /dev/null \|\| \	97	xdp_dummy.bpf.o sec xdp &> /dev/null \|\| \
98	{ test_fail "Unable to load dummy xdp" && exit 1; }	98	{ test_fail "Unable to load dummy xdp" && exit 1; }
99	IFACES="$IFACES veth$i"	99	IFACES="$IFACES veth$i"
100	veth_mac[$i]=$(ip -n ${NS[0]} link show veth$i \| awk '/link\/ether/ {print $2}')	100	veth_mac[$i]=$(ip -n ${NS[0]} link show veth$i \| awk '/link\/ether/ {print $2}')

tools/testing/selftests/bpf/test_xdp_veth.sh+4−4

@@ -101,7 +101,7 @@ ip -n ${NS3} link set dev veth33 up · 100 unmodified lines
101		101
102	mkdir $BPF_DIR	102	mkdir $BPF_DIR
103	bpftool prog loadall \	103	bpftool prog loadall \
104	xdp_redirect_map.o $BPF_DIR/progs type xdp \	104	xdp_redirect_map.bpf.o $BPF_DIR/progs type xdp \
105	pinmaps $BPF_DIR/maps	105	pinmaps $BPF_DIR/maps
106	bpftool map update pinned $BPF_DIR/maps/tx_port key 0 0 0 0 value 122 0 0 0	106	bpftool map update pinned $BPF_DIR/maps/tx_port key 0 0 0 0 value 122 0 0 0
107	bpftool map update pinned $BPF_DIR/maps/tx_port key 1 0 0 0 value 133 0 0 0	107	bpftool map update pinned $BPF_DIR/maps/tx_port key 1 0 0 0 value 133 0 0 0
@@ -110,9 +110,9 @@ ip link set dev veth1 xdp pinned $BPF_DIR/progs/xdp_redirect_map_0 · 2 unmodified lines
110	ip link set dev veth2 xdp pinned $BPF_DIR/progs/xdp_redirect_map_1	110	ip link set dev veth2 xdp pinned $BPF_DIR/progs/xdp_redirect_map_1
111	ip link set dev veth3 xdp pinned $BPF_DIR/progs/xdp_redirect_map_2	111	ip link set dev veth3 xdp pinned $BPF_DIR/progs/xdp_redirect_map_2
112		112
113	ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.o sec xdp	113	ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.bpf.o sec xdp
114	ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.o sec xdp	114	ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.bpf.o sec xdp
115	ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.o sec xdp	115	ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.bpf.o sec xdp
116		116
117	trap cleanup EXIT	117	trap cleanup EXIT
118		118

tools/testing/selftests/bpf/test_xsk.sh+36−16

@@ -73,14 +73,20 @@ · 72 unmodified lines
73	#	73	#
74	# Run and dump packet contents:	74	# Run and dump packet contents:
75	# sudo ./test_xsk.sh -D	75	# sudo ./test_xsk.sh -D
		76	#
		77	# Run test suite for physical device in loopback mode
		78	# sudo ./test_xsk.sh -i IFACE
76		79
77	. xsk_prereqs.sh	80	. xsk_prereqs.sh
78		81
79	while getopts "vD" flag	82	ETH=""
		83
		84	while getopts "vDi:" flag
80	do	85	do
81	case "${flag}" in	86	case "${flag}" in
82	v) verbose=1;;	87	v) verbose=1;;
83	D) dump_pkts=1;;	88	D) dump_pkts=1;;
		89	i) ETH=${OPTARG};;
84	esac	90	esac
85	done	91	done
86		92
@@ -132,18 +138,25 @@ setup_vethPairs() { · 45 unmodified lines
132	ip link set ${VETH0} up	138	ip link set ${VETH0} up
133	}	139	}
134		140
135	validate_root_exec	141	if [ ! -z $ETH ]; then
136	validate_veth_support ${VETH0}	142	VETH0=${ETH}
137	validate_ip_utility	143	VETH1=${ETH}
138	setup_vethPairs	144	NS1=""
139		145	else
140	retval=$?	146	validate_root_exec
141	if [ $retval -ne 0 ]; then	147	validate_veth_support ${VETH0}
142	test_status $retval "${TEST_NAME}"	148	validate_ip_utility
143	cleanup_exit ${VETH0} ${VETH1} ${NS1}	149	setup_vethPairs
144	exit $retval	150
		151	retval=$?
		152	if [ $retval -ne 0 ]; then
		153	test_status $retval "${TEST_NAME}"
		154	cleanup_exit ${VETH0} ${VETH1} ${NS1}
		155	exit $retval
		156	fi
145	fi	157	fi
146		158
		159
147	if [[ $verbose -eq 1 ]]; then	160	if [[ $verbose -eq 1 ]]; then
148	ARGS+="-v "	161	ARGS+="-v "
149	fi	162	fi
@@ -152,26 +165,33 @@ if [[ $dump_pkts -eq 1 ]]; then · 2 unmodified lines
152	ARGS="-D "	165	ARGS="-D "
153	fi	166	fi
154		167
		168	retval=$?
155	test_status $retval "${TEST_NAME}"	169	test_status $retval "${TEST_NAME}"
156		170
157	## START TESTS	171	## START TESTS
158		172
159	statusList=()	173	statusList=()
160		174
161	TEST_NAME="XSK_SELFTESTS_SOFTIRQ"	175	TEST_NAME="XSK_SELFTESTS_${VETH0}_SOFTIRQ"
162		176
163	exec_xskxceiver	177	exec_xskxceiver
164		178
165	cleanup_exit ${VETH0} ${VETH1} ${NS1}	179	if [ -z $ETH ]; then
166	TEST_NAME="XSK_SELFTESTS_BUSY_POLL"	180	cleanup_exit ${VETH0} ${VETH1} ${NS1}
		181	fi
		182	TEST_NAME="XSK_SELFTESTS_${VETH0}_BUSY_POLL"
167	busy_poll=1	183	busy_poll=1
168		184
169	setup_vethPairs	185	if [ -z $ETH ]; then
		186	setup_vethPairs
		187	fi
170	exec_xskxceiver	188	exec_xskxceiver
171		189
172	## END TESTS	190	## END TESTS
173		191
174	cleanup_exit ${VETH0} ${VETH1} ${NS1}	192	if [ -z $ETH ]; then
		193	cleanup_exit ${VETH0} ${VETH1} ${NS1}
		194	fi
175		195
176	failures=0	196	failures=0
177	echo -e "\nSummary:"	197	echo -e "\nSummary:"

tools/testing/selftests/bpf/xdp_redirect_multi.c+1−1

@@ -142,7 +142,7 @@ int main(int argc, char **argv) · 141 unmodified lines
142	}	142	}
143	printf("\n");	143	printf("\n");
144		144
145	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);	145	snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]);
146	obj = bpf_object__open_file(filename, NULL);	146	obj = bpf_object__open_file(filename, NULL);
147	err = libbpf_get_error(obj);	147	err = libbpf_get_error(obj);
148	if (err)	148	if (err)

tools/testing/selftests/bpf/xdp_synproxy.c+1−1

@@ -193,7 +193,7 @@ static int syncookie_attach(const char *argv0, unsigned int ifindex, bool tc) · 192 unmodified lines
193	int prog_fd;	193	int prog_fd;
194	int err;	194	int err;
195		195
196	snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv0);	196	snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.bpf.o", argv0);
197	obj = bpf_object__open_file(xdp_filename, NULL);	197	obj = bpf_object__open_file(xdp_filename, NULL);
198	err = libbpf_get_error(obj);	198	err = libbpf_get_error(obj);
199	if (err < 0) {	199	if (err < 0) {

tools/testing/selftests/bpf/xdping.c+1−1

@@ -168,7 +168,7 @@ int main(int argc, char **argv) · 167 unmodified lines
168	/* Use libbpf 1.0 API mode */	168	/* Use libbpf 1.0 API mode */
169	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);	169	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
170		170
171	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);	171	snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]);
172		172
173	if (bpf_prog_test_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd)) {	173	if (bpf_prog_test_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd)) {
174	fprintf(stderr, "load of %s failed\n", filename);	174	fprintf(stderr, "load of %s failed\n", filename);

tools/testing/selftests/bpf/xsk.c+3−3

 	ctx = xsk->ctx;
 	umem = ctx->umem;
-	xsk_put_ctx(ctx, true);
+	if (ctx->refcount == 1) {
-	if (!ctx->refcount) {
 		xsk_delete_bpf_maps(xsk);
 		close(ctx->prog_fd);
 		if (ctx->has_bpf_link)
 			close(ctx->link_fd);
 	}
+	xsk_put_ctx(ctx, true);
 	err = xsk_get_mmap_offsets(xsk->fd, &off);
 	if (!err) {
 		if (xsk->rx) {

tools/testing/selftests/bpf/xskxceiver.c+296−102

@@ -99,6 +99,8 @@ · 98 unmodified lines
99	#include <stdatomic.h>	99	#include <stdatomic.h>
100	#include "xsk.h"	100	#include "xsk.h"
101	#include "xskxceiver.h"	101	#include "xskxceiver.h"
		102	#include <bpf/bpf.h>
		103	#include <linux/filter.h>
102	#include "../kselftest.h"	104	#include "../kselftest.h"
103		105
104	/* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf.	106	/* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf.
@@ -122,9 +124,20 @@ static void __exit_with_error(int error, const char file, const char func, int · 17 unmodified lines
122	}	124	}
123		125
124	#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)	126	#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
125
126	#define mode_string(test) (test)->ifobj_tx->xdp_flags & XDP_FLAGS_SKB_MODE ? "SKB" : "DRV"
127	#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""	127	#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
		128	static char mode_string(struct test_spec test)
		129	{
		130	switch (test->mode) {
		131	case TEST_MODE_SKB:
		132	return "SKB";
		133	case TEST_MODE_DRV:
		134	return "DRV";
		135	case TEST_MODE_ZC:
		136	return "ZC";
		137	default:
		138	return "BOGUS";
		139	}
		140	}
128		141
129	static void report_failure(struct test_spec *test)	142	static void report_failure(struct test_spec *test)
130	{	143	{
@@ -299,8 +312,8 @@ static void enable_busy_poll(struct xsk_socket_info *xsk) · 168 unmodified lines
299	exit_with_error(errno);	312	exit_with_error(errno);
300	}	313	}
301		314
302	static int xsk_configure_socket(struct xsk_socket_info xsk, struct xsk_umem_info umem,	315	static int __xsk_configure_socket(struct xsk_socket_info xsk, struct xsk_umem_info umem,
303	struct ifobject *ifobject, bool shared)	316	struct ifobject *ifobject, bool shared)
304	{	317	{
305	struct xsk_socket_config cfg = {};	318	struct xsk_socket_config cfg = {};
306	struct xsk_ring_cons *rxr;	319	struct xsk_ring_cons *rxr;
@@ -320,6 +333,51 @@ static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_inf · 13 unmodified lines
320	return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);	333	return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);
321	}	334	}
322		335
		336	static bool ifobj_zc_avail(struct ifobject *ifobject)
		337	{
		338	size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
		339	int mmap_flags = MAP_PRIVATE \| MAP_ANONYMOUS \| MAP_NORESERVE;
		340	struct xsk_socket_info *xsk;
		341	struct xsk_umem_info *umem;
		342	bool zc_avail = false;
		343	void *bufs;
		344	int ret;
		345
		346	bufs = mmap(NULL, umem_sz, PROT_READ \| PROT_WRITE, mmap_flags, -1, 0);
		347	if (bufs == MAP_FAILED)
		348	exit_with_error(errno);
		349
		350	umem = calloc(1, sizeof(struct xsk_umem_info));
		351	if (!umem) {
		352	munmap(bufs, umem_sz);
		353	exit_with_error(-ENOMEM);
		354	}
		355	umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
		356	ret = xsk_configure_umem(umem, bufs, umem_sz);
		357	if (ret)
		358	exit_with_error(-ret);
		359
		360	xsk = calloc(1, sizeof(struct xsk_socket_info));
		361	if (!xsk)
		362	goto out;
		363	ifobject->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
		364	ifobject->xdp_flags \|= XDP_FLAGS_DRV_MODE;
		365	ifobject->bind_flags = XDP_USE_NEED_WAKEUP \| XDP_ZEROCOPY;
		366	ifobject->rx_on = true;
		367	xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
		368	ret = __xsk_configure_socket(xsk, umem, ifobject, false);
		369	if (!ret)
		370	zc_avail = true;
		371
		372	xsk_socket__delete(xsk->xsk);
		373	free(xsk);
		374	out:
		375	munmap(umem->buffer, umem_sz);
		376	xsk_umem__delete(umem->umem);
		377	free(umem);
		378	return zc_avail;
		379	}
		380
323	static struct option long_options[] = {	381	static struct option long_options[] = {
324	{"interface", required_argument, 0, 'i'},	382	{"interface", required_argument, 0, 'i'},
325	{"busy-poll", no_argument, 0, 'b'},	383	{"busy-poll", no_argument, 0, 'b'},
@@ -431,20 +489,24 @@ static void __test_spec_init(struct test_spec test, struct ifobject ifobj_tx, · 105 unmodified lines
431	ifobj->use_poll = false;	489	ifobj->use_poll = false;
432	ifobj->use_fill_ring = true;	490	ifobj->use_fill_ring = true;
433	ifobj->release_rx = true;	491	ifobj->release_rx = true;
434	ifobj->pkt_stream = test->pkt_stream_default;
435	ifobj->validation_func = NULL;	492	ifobj->validation_func = NULL;
436		493
437	if (i == 0) {	494	if (i == 0) {
438	ifobj->rx_on = false;	495	ifobj->rx_on = false;
439	ifobj->tx_on = true;	496	ifobj->tx_on = true;
		497	ifobj->pkt_stream = test->tx_pkt_stream_default;
440	} else {	498	} else {
441	ifobj->rx_on = true;	499	ifobj->rx_on = true;
442	ifobj->tx_on = false;	500	ifobj->tx_on = false;
		501	ifobj->pkt_stream = test->rx_pkt_stream_default;
443	}	502	}
444		503
445	memset(ifobj->umem, 0, sizeof(*ifobj->umem));	504	memset(ifobj->umem, 0, sizeof(*ifobj->umem));
446	ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;	505	ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
447	ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;	506	ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
		507	if (ifobj->shared_umem && ifobj->rx_on)
		508	ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS *
		509	XSK_UMEM__DEFAULT_FRAME_SIZE;
448		510
449	for (j = 0; j < MAX_SOCKETS; j++) {	511	for (j = 0; j < MAX_SOCKETS; j++) {
450	memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));	512	memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
@@ -463,12 +525,15 @@ static void __test_spec_init(struct test_spec test, struct ifobject ifobj_tx, · 12 unmodified lines
463	static void test_spec_init(struct test_spec test, struct ifobject ifobj_tx,	525	static void test_spec_init(struct test_spec test, struct ifobject ifobj_tx,
464	struct ifobject *ifobj_rx, enum test_mode mode)	526	struct ifobject *ifobj_rx, enum test_mode mode)
465	{	527	{
466	struct pkt_stream *pkt_stream;	528	struct pkt_stream *tx_pkt_stream;
		529	struct pkt_stream *rx_pkt_stream;
467	u32 i;	530	u32 i;
468		531
469	pkt_stream = test->pkt_stream_default;	532	tx_pkt_stream = test->tx_pkt_stream_default;
		533	rx_pkt_stream = test->rx_pkt_stream_default;
470	memset(test, 0, sizeof(*test));	534	memset(test, 0, sizeof(*test));
471	test->pkt_stream_default = pkt_stream;	535	test->tx_pkt_stream_default = tx_pkt_stream;
		536	test->rx_pkt_stream_default = rx_pkt_stream;
472		537
473	for (i = 0; i < MAX_INTERFACES; i++) {	538	for (i = 0; i < MAX_INTERFACES; i++) {
474	struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;	539	struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
@@ -479,9 +544,14 @@ static void test_spec_init(struct test_spec test, struct ifobject ifobj_tx, · 4 unmodified lines
479	else	544	else
480	ifobj->xdp_flags \|= XDP_FLAGS_DRV_MODE;	545	ifobj->xdp_flags \|= XDP_FLAGS_DRV_MODE;
481		546
482	ifobj->bind_flags = XDP_USE_NEED_WAKEUP \| XDP_COPY;	547	ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
		548	if (mode == TEST_MODE_ZC)
		549	ifobj->bind_flags \|= XDP_ZEROCOPY;
		550	else
		551	ifobj->bind_flags \|= XDP_COPY;
483	}	552	}
484		553
		554	test->mode = mode;
485	__test_spec_init(test, ifobj_tx, ifobj_rx);	555	__test_spec_init(test, ifobj_tx, ifobj_rx);
486	}	556	}
487		557
@@ -529,16 +599,17 @@ static void pkt_stream_delete(struct pkt_stream *pkt_stream) · 41 unmodified lines
529	static void pkt_stream_restore_default(struct test_spec *test)	599	static void pkt_stream_restore_default(struct test_spec *test)
530	{	600	{
531	struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;	601	struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
		602	struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
532		603
533	if (tx_pkt_stream != test->pkt_stream_default) {	604	if (tx_pkt_stream != test->tx_pkt_stream_default) {
534	pkt_stream_delete(test->ifobj_tx->pkt_stream);	605	pkt_stream_delete(test->ifobj_tx->pkt_stream);
535	test->ifobj_tx->pkt_stream = test->pkt_stream_default;	606	test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
536	}	607	}
537		608
538	if (test->ifobj_rx->pkt_stream != test->pkt_stream_default &&	609	if (rx_pkt_stream != test->rx_pkt_stream_default) {
539	test->ifobj_rx->pkt_stream != tx_pkt_stream)
540	pkt_stream_delete(test->ifobj_rx->pkt_stream);	610	pkt_stream_delete(test->ifobj_rx->pkt_stream);
541	test->ifobj_rx->pkt_stream = test->pkt_stream_default;	611	test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
		612	}
542	}	613	}
543		614
544	static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)	615	static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
@@ -561,7 +632,7 @@ static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) · 16 unmodified lines
561		632
562	static void pkt_set(struct xsk_umem_info umem, struct pkt pkt, u64 addr, u32 len)	633	static void pkt_set(struct xsk_umem_info umem, struct pkt pkt, u64 addr, u32 len)
563	{	634	{
564	pkt->addr = addr;	635	pkt->addr = addr + umem->base_addr;
565	pkt->len = len;	636	pkt->len = len;
566	if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)	637	if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)
567	pkt->valid = false;	638	pkt->valid = false;
@@ -600,22 +671,29 @@ static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) · 32 unmodified lines
600		671
601	pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);	672	pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
602	test->ifobj_tx->pkt_stream = pkt_stream;	673	test->ifobj_tx->pkt_stream = pkt_stream;
		674	pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
603	test->ifobj_rx->pkt_stream = pkt_stream;	675	test->ifobj_rx->pkt_stream = pkt_stream;
604	}	676	}
605		677
606	static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)	678	static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
		679	int offset)
607	{	680	{
608	struct xsk_umem_info *umem = test->ifobj_tx->umem;	681	struct xsk_umem_info *umem = ifobj->umem;
609	struct pkt_stream *pkt_stream;	682	struct pkt_stream *pkt_stream;
610	u32 i;	683	u32 i;
611		684
612	pkt_stream = pkt_stream_clone(umem, test->pkt_stream_default);	685	pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
613	for (i = 1; i < test->pkt_stream_default->nb_pkts; i += 2)	686	for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
614	pkt_set(umem, &pkt_stream->pkts[i],	687	pkt_set(umem, &pkt_stream->pkts[i],
615	(i % umem->num_frames) * umem->frame_size + offset, pkt_len);	688	(i % umem->num_frames) * umem->frame_size + offset, pkt_len);
616		689
617	test->ifobj_tx->pkt_stream = pkt_stream;	690	ifobj->pkt_stream = pkt_stream;
618	test->ifobj_rx->pkt_stream = pkt_stream;	691	}
		692
		693	static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
		694	{
		695	__pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
		696	__pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
619	}	697	}
620		698
621	static void pkt_stream_receive_half(struct test_spec *test)	699	static void pkt_stream_receive_half(struct test_spec *test)
@@ -657,7 +735,8 @@ static struct pkt pkt_generate(struct ifobject ifobject, u32 pkt_nb) · 35 unmodified lines
657	return pkt;	735	return pkt;
658	}	736	}
659		737
660	static void pkt_stream_generate_custom(struct test_spec test, struct pkt pkts, u32 nb_pkts)	738	static void __pkt_stream_generate_custom(struct ifobject *ifobj,
		739	struct pkt *pkts, u32 nb_pkts)
661	{	740	{
662	struct pkt_stream *pkt_stream;	741	struct pkt_stream *pkt_stream;
663	u32 i;	742	u32 i;
@@ -666,15 +745,20 @@ static void pkt_stream_generate_custom(struct test_spec test, struct pkt pkts, · 2 unmodified lines
666	if (!pkt_stream)	745	if (!pkt_stream)
667	exit_with_error(ENOMEM);	746	exit_with_error(ENOMEM);
668		747
669	test->ifobj_tx->pkt_stream = pkt_stream;
670	test->ifobj_rx->pkt_stream = pkt_stream;
671
672	for (i = 0; i < nb_pkts; i++) {	748	for (i = 0; i < nb_pkts; i++) {
673	pkt_stream->pkts[i].addr = pkts[i].addr;	749	pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr;
674	pkt_stream->pkts[i].len = pkts[i].len;	750	pkt_stream->pkts[i].len = pkts[i].len;
675	pkt_stream->pkts[i].payload = i;	751	pkt_stream->pkts[i].payload = i;
676	pkt_stream->pkts[i].valid = pkts[i].valid;	752	pkt_stream->pkts[i].valid = pkts[i].valid;
677	}	753	}
		754
		755	ifobj->pkt_stream = pkt_stream;
		756	}
		757
		758	static void pkt_stream_generate_custom(struct test_spec test, struct pkt pkts, u32 nb_pkts)
		759	{
		760	__pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts);
		761	__pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts);
678	}	762	}
679		763
680	static void pkt_dump(void *pkt, u32 len)	764	static void pkt_dump(void *pkt, u32 len)
@@ -1126,6 +1210,70 @@ static int validate_tx_invalid_descs(struct ifobject *ifobject) · 445 unmodified lines
1126	return TEST_PASS;	1210	return TEST_PASS;
1127	}	1211	}
1128		1212
		1213	static void xsk_configure_socket(struct test_spec test, struct ifobject ifobject,
		1214	struct xsk_umem_info *umem, bool tx)
		1215	{
		1216	int i, ret;
		1217
		1218	for (i = 0; i < test->nb_sockets; i++) {
		1219	bool shared = (ifobject->shared_umem && tx) ? true : !!i;
		1220	u32 ctr = 0;
		1221
		1222	while (ctr++ < SOCK_RECONF_CTR) {
		1223	ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
		1224	ifobject, shared);
		1225	if (!ret)
		1226	break;
		1227
		1228	/* Retry if it fails as xsk_socket__create() is asynchronous */
		1229	if (ctr >= SOCK_RECONF_CTR)
		1230	exit_with_error(-ret);
		1231	usleep(USLEEP_MAX);
		1232	}
		1233	if (ifobject->busy_poll)
		1234	enable_busy_poll(&ifobject->xsk_arr[i]);
		1235	}
		1236	}
		1237
		1238	static void thread_common_ops_tx(struct test_spec test, struct ifobject ifobject)
		1239	{
		1240	xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
		1241	ifobject->xsk = &ifobject->xsk_arr[0];
		1242	ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd;
		1243	memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
		1244	}
		1245
		1246	static void xsk_populate_fill_ring(struct xsk_umem_info umem, struct pkt_stream pkt_stream)
		1247	{
		1248	u32 idx = 0, i, buffers_to_fill;
		1249	int ret;
		1250
		1251	if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
		1252	buffers_to_fill = umem->num_frames;
		1253	else
		1254	buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
		1255
		1256	ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
		1257	if (ret != buffers_to_fill)
		1258	exit_with_error(ENOSPC);
		1259	for (i = 0; i < buffers_to_fill; i++) {
		1260	u64 addr;
		1261
		1262	if (pkt_stream->use_addr_for_fill) {
		1263	struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
		1264
		1265	if (!pkt)
		1266	break;
		1267	addr = pkt->addr;
		1268	} else {
		1269	addr = i * umem->frame_size;
		1270	}
		1271
		1272	*xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
		1273	}
		1274	xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
		1275	}
		1276
1129	static void thread_common_ops(struct test_spec test, struct ifobject ifobject)	1277	static void thread_common_ops(struct test_spec test, struct ifobject ifobject)
1130	{	1278	{
1131	u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;	1279	u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
@@ -1133,13 +1281,15 @@ static void thread_common_ops(struct test_spec test, struct ifobject ifobject) · 1 unmodified line
1133	LIBBPF_OPTS(bpf_xdp_query_opts, opts);	1281	LIBBPF_OPTS(bpf_xdp_query_opts, opts);
1134	int ret, ifindex;	1282	int ret, ifindex;
1135	void *bufs;	1283	void *bufs;
1136	u32 i;
1137		1284
1138	ifobject->ns_fd = switch_namespace(ifobject->nsname);	1285	ifobject->ns_fd = switch_namespace(ifobject->nsname);
1139		1286
1140	if (ifobject->umem->unaligned_mode)	1287	if (ifobject->umem->unaligned_mode)
1141	mmap_flags \|= MAP_HUGETLB;	1288	mmap_flags \|= MAP_HUGETLB;
1142		1289
		1290	if (ifobject->shared_umem)
		1291	umem_sz *= 2;
		1292
1143	bufs = mmap(NULL, umem_sz, PROT_READ \| PROT_WRITE, mmap_flags, -1, 0);	1293	bufs = mmap(NULL, umem_sz, PROT_READ \| PROT_WRITE, mmap_flags, -1, 0);
1144	if (bufs == MAP_FAILED)	1294	if (bufs == MAP_FAILED)
1145	exit_with_error(errno);	1295	exit_with_error(errno);
@@ -1148,24 +1298,9 @@ static void thread_common_ops(struct test_spec test, struct ifobject ifobject) · 2 unmodified lines
1148	if (ret)	1298	if (ret)
1149	exit_with_error(-ret);	1299	exit_with_error(-ret);
1150		1300
1151	for (i = 0; i < test->nb_sockets; i++) {	1301	xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
1152	u32 ctr = 0;
1153
1154	while (ctr++ < SOCK_RECONF_CTR) {
1155	ret = xsk_configure_socket(&ifobject->xsk_arr[i], ifobject->umem,
1156	ifobject, !!i);
1157	if (!ret)
1158	break;
1159
1160	/* Retry if it fails as xsk_socket__create() is asynchronous */
1161	if (ctr >= SOCK_RECONF_CTR)
1162	exit_with_error(-ret);
1163	usleep(USLEEP_MAX);
1164	}
1165		1302
1166	if (ifobject->busy_poll)	1303	xsk_configure_socket(test, ifobject, ifobject->umem, false);
1167	enable_busy_poll(&ifobject->xsk_arr[i]);
1168	}
1169		1304
1170	ifobject->xsk = &ifobject->xsk_arr[0];	1305	ifobject->xsk = &ifobject->xsk_arr[0];
1171		1306
@@ -1201,22 +1336,18 @@ static void thread_common_ops(struct test_spec test, struct ifobject ifobject) · 29 unmodified lines
1201	exit_with_error(-ret);	1336	exit_with_error(-ret);
1202	}	1337	}
1203		1338
1204	static void testapp_cleanup_xsk_res(struct ifobject *ifobj)
1205	{
1206	print_verbose("Destroying socket\n");
1207	xsk_socket__delete(ifobj->xsk->xsk);
1208	munmap(ifobj->umem->buffer, ifobj->umem->num_frames * ifobj->umem->frame_size);
1209	xsk_umem__delete(ifobj->umem->umem);
1210	}
1211
1212	static void worker_testapp_validate_tx(void arg)	1339	static void worker_testapp_validate_tx(void arg)
1213	{	1340	{
1214	struct test_spec test = (struct test_spec )arg;	1341	struct test_spec test = (struct test_spec )arg;
1215	struct ifobject *ifobject = test->ifobj_tx;	1342	struct ifobject *ifobject = test->ifobj_tx;
1216	int err;	1343	int err;
1217		1344
1218	if (test->current_step == 1)	1345	if (test->current_step == 1) {
1219	thread_common_ops(test, ifobject);	1346	if (!ifobject->shared_umem)
		1347	thread_common_ops(test, ifobject);
		1348	else
		1349	thread_common_ops_tx(test, ifobject);
		1350	}
1220		1351
1221	print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,	1352	print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
1222	ifobject->ifname);	1353	ifobject->ifname);
@@ -1227,53 +1358,23 @@ static void worker_testapp_validate_tx(void arg) · 4 unmodified lines
1227	if (err)	1358	if (err)
1228	report_failure(test);	1359	report_failure(test);
1229		1360
1230	if (test->total_steps == test->current_step \|\| err)
1231	testapp_cleanup_xsk_res(ifobject);
1232	pthread_exit(NULL);	1361	pthread_exit(NULL);
1233	}	1362	}
1234		1363
1235	static void xsk_populate_fill_ring(struct xsk_umem_info umem, struct pkt_stream pkt_stream)
1236	{
1237	u32 idx = 0, i, buffers_to_fill;
1238	int ret;
1239
1240	if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1241	buffers_to_fill = umem->num_frames;
1242	else
1243	buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
1244
1245	ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1246	if (ret != buffers_to_fill)
1247	exit_with_error(ENOSPC);
1248	for (i = 0; i < buffers_to_fill; i++) {
1249	u64 addr;
1250
1251	if (pkt_stream->use_addr_for_fill) {
1252	struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
1253
1254	if (!pkt)
1255	break;
1256	addr = pkt->addr;
1257	} else {
1258	addr = i * umem->frame_size;
1259	}
1260
1261	*xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1262	}
1263	xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
1264	}
1265
1266	static void worker_testapp_validate_rx(void arg)	1364	static void worker_testapp_validate_rx(void arg)
1267	{	1365	{
1268	struct test_spec test = (struct test_spec )arg;	1366	struct test_spec test = (struct test_spec )arg;
1269	struct ifobject *ifobject = test->ifobj_rx;	1367	struct ifobject *ifobject = test->ifobj_rx;
1270	struct pollfd fds = { };	1368	struct pollfd fds = { };
		1369	int id = 0;
1271	int err;	1370	int err;
1272		1371
1273	if (test->current_step == 1)	1372	if (test->current_step == 1) {
1274	thread_common_ops(test, ifobject);	1373	thread_common_ops(test, ifobject);
1275		1374	} else {
1276	xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);	1375	bpf_map_delete_elem(ifobject->xsk_map_fd, &id);
		1376	xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
		1377	}
1277		1378
1278	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);	1379	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
1279	fds.events = POLLIN;	1380	fds.events = POLLIN;
@@ -1291,25 +1392,44 @@ static void worker_testapp_validate_rx(void arg) · 11 unmodified lines
1291	pthread_mutex_unlock(&pacing_mutex);	1392	pthread_mutex_unlock(&pacing_mutex);
1292	}	1393	}
1293		1394
1294	if (test->total_steps == test->current_step \|\| err)	1395	pthread_exit(NULL);
1295	testapp_cleanup_xsk_res(ifobject);	1396	}
		1397
		1398	static void testapp_clean_xsk_umem(struct ifobject *ifobj)
		1399	{
		1400	u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
		1401
		1402	if (ifobj->shared_umem)
		1403	umem_sz *= 2;
		1404
		1405	xsk_umem__delete(ifobj->umem->umem);
		1406	munmap(ifobj->umem->buffer, umem_sz);
		1407	}
		1408
		1409	static void handler(int signum)
		1410	{
1296	pthread_exit(NULL);	1411	pthread_exit(NULL);
1297	}	1412	}
1298		1413
1299	static int testapp_validate_traffic_single_thread(struct test_spec test, struct ifobject ifobj,	1414	static int testapp_validate_traffic_single_thread(struct test_spec test, struct ifobject ifobj,
1300	enum test_type type)	1415	enum test_type type)
1301	{	1416	{
		1417	bool old_shared_umem = ifobj->shared_umem;
1302	pthread_t t0;	1418	pthread_t t0;
1303		1419
1304	if (pthread_barrier_init(&barr, NULL, 2))	1420	if (pthread_barrier_init(&barr, NULL, 2))
1305	exit_with_error(errno);	1421	exit_with_error(errno);
1306		1422
1307	test->current_step++;	1423	test->current_step++;
1308	if (type == TEST_TYPE_POLL_RXQ_TMOUT)	1424	if (type == TEST_TYPE_POLL_RXQ_TMOUT)
1309	pkt_stream_reset(ifobj->pkt_stream);	1425	pkt_stream_reset(ifobj->pkt_stream);
1310	pkts_in_flight = 0;	1426	pkts_in_flight = 0;
1311		1427
1312	/Spawn thread /	1428	test->ifobj_rx->shared_umem = false;
		1429	test->ifobj_tx->shared_umem = false;
		1430
		1431	signal(SIGUSR1, handler);
		1432	/* Spawn thread */
1313	pthread_create(&t0, NULL, ifobj->func_ptr, test);	1433	pthread_create(&t0, NULL, ifobj->func_ptr, test);
1314		1434
1315	if (type != TEST_TYPE_POLL_TXQ_TMOUT)	1435	if (type != TEST_TYPE_POLL_TXQ_TMOUT)
@@ -1318,8 +1438,17 @@ static int testapp_validate_traffic_single_thread(struct test_spec *test, struct · 2 unmodified lines
1318	if (pthread_barrier_destroy(&barr))	1438	if (pthread_barrier_destroy(&barr))
1319	exit_with_error(errno);	1439	exit_with_error(errno);
1320		1440
		1441	pthread_kill(t0, SIGUSR1);
1321	pthread_join(t0, NULL);	1442	pthread_join(t0, NULL);
1322		1443
		1444	if (test->total_steps == test->current_step \|\| test->fail) {
		1445	xsk_socket__delete(ifobj->xsk->xsk);
		1446	testapp_clean_xsk_umem(ifobj);
		1447	}
		1448
		1449	test->ifobj_rx->shared_umem = old_shared_umem;
		1450	test->ifobj_tx->shared_umem = old_shared_umem;
		1451
1323	return !!test->fail;	1452	return !!test->fail;
1324	}	1453	}
1325		1454
@@ -1349,6 +1478,14 @@ static int testapp_validate_traffic(struct test_spec *test) · 23 unmodified lines
1349	pthread_join(t1, NULL);	1478	pthread_join(t1, NULL);
1350	pthread_join(t0, NULL);	1479	pthread_join(t0, NULL);
1351		1480
		1481	if (test->total_steps == test->current_step \|\| test->fail) {
		1482	xsk_socket__delete(ifobj_tx->xsk->xsk);
		1483	xsk_socket__delete(ifobj_rx->xsk->xsk);
		1484	testapp_clean_xsk_umem(ifobj_rx);
		1485	if (!ifobj_tx->shared_umem)
		1486	testapp_clean_xsk_umem(ifobj_tx);
		1487	}
		1488
1352	return !!test->fail;	1489	return !!test->fail;
1353	}	1490	}
1354		1491
@@ -1428,9 +1565,9 @@ static void testapp_headroom(struct test_spec *test) · 73 unmodified lines
1428	static void testapp_stats_rx_dropped(struct test_spec *test)	1565	static void testapp_stats_rx_dropped(struct test_spec *test)
1429	{	1566	{
1430	test_spec_set_name(test, "STAT_RX_DROPPED");	1567	test_spec_set_name(test, "STAT_RX_DROPPED");
		1568	pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1431	test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -	1569	test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
1432	XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;	1570	XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
1433	pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
1434	pkt_stream_receive_half(test);	1571	pkt_stream_receive_half(test);
1435	test->ifobj_rx->validation_func = validate_rx_dropped;	1572	test->ifobj_rx->validation_func = validate_rx_dropped;
1436	testapp_validate_traffic(test);	1573	testapp_validate_traffic(test);
@@ -1553,6 +1690,11 @@ static void testapp_invalid_desc(struct test_spec *test) · 116 unmodified lines
1553	pkts[7].valid = false;	1690	pkts[7].valid = false;
1554	}	1691	}
1555		1692
		1693	if (test->ifobj_tx->shared_umem) {
		1694	pkts[4].addr += UMEM_SIZE;
		1695	pkts[5].addr += UMEM_SIZE;
		1696	}
		1697
1556	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));	1698	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
1557	testapp_validate_traffic(test);	1699	testapp_validate_traffic(test);
1558	pkt_stream_restore_default(test);	1700	pkt_stream_restore_default(test);
@@ -1583,6 +1725,10 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_ · 24 unmodified lines
1583	{	1725	{
1584	switch (type) {	1726	switch (type) {
1585	case TEST_TYPE_STATS_RX_DROPPED:	1727	case TEST_TYPE_STATS_RX_DROPPED:
		1728	if (mode == TEST_MODE_ZC) {
		1729	ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
		1730	return;
		1731	}
1586	testapp_stats_rx_dropped(test);	1732	testapp_stats_rx_dropped(test);
1587	break;	1733	break;
1588	case TEST_TYPE_STATS_TX_INVALID_DESCS:	1734	case TEST_TYPE_STATS_TX_INVALID_DESCS:
@@ -1712,12 +1858,44 @@ static void ifobject_delete(struct ifobject *ifobj) · 123 unmodified lines
1712	free(ifobj);	1858	free(ifobj);
1713	}	1859	}
1714		1860
		1861	static bool is_xdp_supported(struct ifobject *ifobject)
		1862	{
		1863	int flags = XDP_FLAGS_DRV_MODE;
		1864
		1865	LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
		1866	struct bpf_insn insns[2] = {
		1867	BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
		1868	BPF_EXIT_INSN()
		1869	};
		1870	int ifindex = if_nametoindex(ifobject->ifname);
		1871	int prog_fd, insn_cnt = ARRAY_SIZE(insns);
		1872	int err;
		1873
		1874	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
		1875	if (prog_fd < 0)
		1876	return false;
		1877
		1878	err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
		1879	if (err) {
		1880	close(prog_fd);
		1881	return false;
		1882	}
		1883
		1884	bpf_xdp_detach(ifindex, flags, NULL);
		1885	close(prog_fd);
		1886
		1887	return true;
		1888	}
		1889
1715	int main(int argc, char **argv)	1890	int main(int argc, char **argv)
1716	{	1891	{
1717	struct pkt_stream *pkt_stream_default;	1892	struct pkt_stream *rx_pkt_stream_default;
		1893	struct pkt_stream *tx_pkt_stream_default;
1718	struct ifobject ifobj_tx, ifobj_rx;	1894	struct ifobject ifobj_tx, ifobj_rx;
		1895	int modes = TEST_MODE_SKB + 1;
1719	u32 i, j, failed_tests = 0;	1896	u32 i, j, failed_tests = 0;
1720	struct test_spec test;	1897	struct test_spec test;
		1898	bool shared_umem;
1721		1899
1722	/* Use libbpf 1.0 API mode */	1900	/* Use libbpf 1.0 API mode */
1723	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);	1901	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
@@ -1732,6 +1910,10 @@ int main(int argc, char **argv) · 8 unmodified lines
1732	setlocale(LC_ALL, "");	1910	setlocale(LC_ALL, "");
1733		1911
1734	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);	1912	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
		1913	shared_umem = !strcmp(ifobj_tx->ifname, ifobj_rx->ifname);
		1914
		1915	ifobj_tx->shared_umem = shared_umem;
		1916	ifobj_rx->shared_umem = shared_umem;
1735		1917
1736	if (!validate_interface(ifobj_tx) \|\| !validate_interface(ifobj_rx)) {	1918	if (!validate_interface(ifobj_tx) \|\| !validate_interface(ifobj_rx)) {
1737	usage(basename(argv[0]));	1919	usage(basename(argv[0]));
@@ -1743,15 +1925,23 @@ int main(int argc, char **argv) · 5 unmodified lines
1743	init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,	1925	init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
1744	worker_testapp_validate_rx);	1926	worker_testapp_validate_rx);
1745		1927
		1928	if (is_xdp_supported(ifobj_tx)) {
		1929	modes++;
		1930	if (ifobj_zc_avail(ifobj_tx))
		1931	modes++;
		1932	}
		1933
1746	test_spec_init(&test, ifobj_tx, ifobj_rx, 0);	1934	test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
1747	pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);	1935	tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
1748	if (!pkt_stream_default)	1936	rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
		1937	if (!tx_pkt_stream_default \|\| !rx_pkt_stream_default)
1749	exit_with_error(ENOMEM);	1938	exit_with_error(ENOMEM);
1750	test.pkt_stream_default = pkt_stream_default;	1939	test.tx_pkt_stream_default = tx_pkt_stream_default;
		1940	test.rx_pkt_stream_default = rx_pkt_stream_default;
1751		1941
1752	ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);	1942	ksft_set_plan(modes * TEST_TYPE_MAX);
1753		1943
1754	for (i = 0; i < TEST_MODE_MAX; i++)	1944	for (i = 0; i < modes; i++)
1755	for (j = 0; j < TEST_TYPE_MAX; j++) {	1945	for (j = 0; j < TEST_TYPE_MAX; j++) {
1756	test_spec_init(&test, ifobj_tx, ifobj_rx, i);	1946	test_spec_init(&test, ifobj_tx, ifobj_rx, i);
1757	run_pkt_test(&test, i, j);	1947	run_pkt_test(&test, i, j);
@@ -1761,7 +1951,11 @@ int main(int argc, char **argv) · 3 unmodified lines
1761	failed_tests++;	1951	failed_tests++;
1762	}	1952	}
1763		1953
1764	pkt_stream_delete(pkt_stream_default);	1954	pkt_stream_delete(tx_pkt_stream_default);
		1955	pkt_stream_delete(rx_pkt_stream_default);
		1956	free(ifobj_rx->umem);
		1957	if (!ifobj_tx->shared_umem)
		1958	free(ifobj_tx->umem);
1765	ifobject_delete(ifobj_tx);	1959	ifobject_delete(ifobj_tx);
1766	ifobject_delete(ifobj_rx);	1960	ifobject_delete(ifobj_rx);
1767		1961

tools/testing/selftests/bpf/xskxceiver.h+8−3

 #define TEST_FAILURE -1
 #define TEST_CONTINUE 1
 #define MAX_INTERFACES 2
-#define MAX_INTERFACE_NAME_CHARS 7
+#define MAX_INTERFACE_NAME_CHARS 16
-#define MAX_INTERFACES_NAMESPACE_CHARS 10
+#define MAX_INTERFACES_NAMESPACE_CHARS 16
 #define MAX_SOCKETS 2
 #define MAX_TEST_NAME_SIZE 32
 #define MAX_TEARDOWN_ITER 10
 enum test_mode {
 	TEST_MODE_SKB,
 	TEST_MODE_DRV,
+	TEST_MODE_ZC,
 	TEST_MODE_MAX
 };
 	u32 frame_headroom;
 	void *buffer;
 	u32 frame_size;
+	u32 base_addr;
 	bool unaligned_mode;
 };
 	bool busy_poll;
 	bool use_fill_ring;
 	bool release_rx;
+	bool shared_umem;
 	u8 dst_mac[ETH_ALEN];
 	u8 src_mac[ETH_ALEN];
 };
 struct test_spec {
 	struct ifobject *ifobj_tx;
 	struct ifobject *ifobj_rx;
-	struct pkt_stream *pkt_stream_default;
+	struct pkt_stream *tx_pkt_stream_default;
+	struct pkt_stream *rx_pkt_stream_default;
 	u16 total_steps;
 	u16 current_step;
 	u16 nb_sockets;
 	bool fail;
+	enum test_mode mode;
 	char name[MAX_TEST_NAME_SIZE];
 };