Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Alexei Starovoitov says: ==================== pull-request: bpf-next 2019-06-19 The following pull-request contains BPF updates for your *net-next* tree. The main changes are: 1) new SO_REUSEPORT_DETACH_BPF setsocktopt, from Martin. 2) BTF based map definition, from Andrii. 3) support bpf_map_lookup_elem for xskmap, from Jonathan. 4) bounded loops and scalar precision logic in the verifier, from Alexei. ====================

David S. Miller committed 7 years ago

Commit dca73a62 parents 497ad9f 94079b6101 files · +4048 −860Browse files

Unified · Split

101 files

arch/alpha/include/uapi/asm/socket.h +2 −0
arch/mips/include/uapi/asm/socket.h +2 −0
arch/parisc/include/uapi/asm/socket.h +2 −0
arch/sparc/include/uapi/asm/socket.h +2 −0
include/linux/bpf.h +25 −0
include/linux/bpf_verifier.h +68 −1
include/net/sock_reuseport.h +2 −0
include/net/xdp_sock.h +2 −2
include/uapi/asm-generic/socket.h +2 −0
include/uapi/linux/bpf.h +6 −0
kernel/bpf/Makefile +1 −0
kernel/bpf/devmap.c +1 −1
kernel/bpf/verifier.c +724 −69
kernel/bpf/xskmap.c +8 −1
net/core/filter.c +86 −0
net/core/sock.c +4 −0
net/core/sock_reuseport.c +24 −0
samples/bpf/Makefile +13 −10
samples/bpf/fds_example.c +1 −1
samples/bpf/hbm.c +3 −3
samples/bpf/ibumad_user.c +1 −1
samples/bpf/sockex1_user.c +1 −1
samples/bpf/sockex2_user.c +1 −1
samples/bpf/xdp1_user.c +2 −2
samples/bpf/xdp_adjust_tail_user.c +2 −2
samples/bpf/xdp_fwd_user.c +1 −1
samples/bpf/xdp_redirect_cpu_user.c +1 −1
samples/bpf/xdp_redirect_map_user.c +1 −1
samples/bpf/xdp_redirect_user.c +1 −1
samples/bpf/xdp_router_ipv4_user.c +1 −1
samples/bpf/xdp_rxq_info_user.c +2 −2
samples/bpf/xdp_tx_iptunnel_user.c +1 −1
samples/bpf/xdpsock_user.c +2 −2
tools/bpf/bpftool/common.c +5 −48
tools/include/uapi/asm-generic/socket.h +147 −0
tools/include/uapi/linux/bpf.h +6 −0
tools/lib/bpf/bpf.c +2 −5
tools/lib/bpf/bpf_prog_linfo.c +1 −4
tools/lib/bpf/btf.c +0 −3
tools/lib/bpf/btf.h +1 −0
tools/lib/bpf/btf_dump.c +0 −3
tools/lib/bpf/libbpf.c +689 −238
tools/lib/bpf/libbpf.h +47 −31
tools/lib/bpf/libbpf.map +1 −0
tools/lib/bpf/libbpf_internal.h +7 −0
tools/lib/bpf/xsk.c +28 −75
tools/testing/selftests/bpf/Makefile +2 −1
tools/testing/selftests/bpf/bpf_endian.h +1 −0
tools/testing/selftests/bpf/bpf_helpers.h +2 −2
tools/testing/selftests/bpf/bpf_util.h +5 −32
tools/testing/selftests/bpf/cgroup_helpers.c +1 −1
tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c +56 −11
tools/testing/selftests/bpf/progs/bpf_flow.c +13 −5
tools/testing/selftests/bpf/progs/loop1.c +28 −0
tools/testing/selftests/bpf/progs/loop2.c +28 −0
tools/testing/selftests/bpf/progs/loop3.c +22 −0
tools/testing/selftests/bpf/progs/netcnt_prog.c +10 −12
tools/testing/selftests/bpf/progs/pyperf.h +5 −1
tools/testing/selftests/bpf/progs/pyperf600.c +9 −0
tools/testing/selftests/bpf/progs/pyperf600_nounroll.c +8 −0
tools/testing/selftests/bpf/progs/socket_cookie_prog.c +31 −18
tools/testing/selftests/bpf/progs/sockmap_parse_prog.c +0 −1
tools/testing/selftests/bpf/progs/sockmap_tcp_msg_prog.c +1 −1
tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c +0 −1
tools/testing/selftests/bpf/progs/strobemeta.c +10 −0
tools/testing/selftests/bpf/progs/strobemeta.h +528 −0
tools/testing/selftests/bpf/progs/strobemeta_nounroll1.c +9 −0
tools/testing/selftests/bpf/progs/strobemeta_nounroll2.c +9 −0
tools/testing/selftests/bpf/progs/test_btf_newkv.c +73 −0
tools/testing/selftests/bpf/progs/test_get_stack_rawtp.c +19 −8
tools/testing/selftests/bpf/progs/test_global_data.c +18 −9
tools/testing/selftests/bpf/progs/test_l4lb.c +30 −15
tools/testing/selftests/bpf/progs/test_l4lb_noinline.c +30 −15
tools/testing/selftests/bpf/progs/test_map_lock.c +12 −10
tools/testing/selftests/bpf/progs/test_seg6_loop.c +261 −0
tools/testing/selftests/bpf/progs/test_select_reuseport_kern.c +31 −14
tools/testing/selftests/bpf/progs/test_send_signal_kern.c +12 −10
tools/testing/selftests/bpf/progs/test_sock_fields_kern.c +36 −24
tools/testing/selftests/bpf/progs/test_spin_lock.c +17 −16
tools/testing/selftests/bpf/progs/test_stacktrace_build_id.c +30 −14
tools/testing/selftests/bpf/progs/test_stacktrace_map.c +28 −12
tools/testing/selftests/bpf/progs/test_sysctl_loop1.c +71 −0
tools/testing/selftests/bpf/progs/test_sysctl_loop2.c +72 −0
tools/testing/selftests/bpf/progs/test_sysctl_prog.c +4 −1
tools/testing/selftests/bpf/progs/test_tcp_estats.c +6 −3
tools/testing/selftests/bpf/progs/test_tcpbpf_kern.c +12 −6
tools/testing/selftests/bpf/progs/test_tcpnotify_kern.c +13 −5
tools/testing/selftests/bpf/progs/test_xdp.c +12 −6
tools/testing/selftests/bpf/progs/test_xdp_loop.c +231 −0
tools/testing/selftests/bpf/progs/test_xdp_noinline.c +37 −23
tools/testing/selftests/bpf/test_btf.c +3 −7
tools/testing/selftests/bpf/test_select_reuseport.c +54 −0
tools/testing/selftests/bpf/test_socket_cookie.c +9 −15
tools/testing/selftests/bpf/test_verifier.c +4 −7
tools/testing/selftests/bpf/verifier/calls.c +13 −9
tools/testing/selftests/bpf/verifier/cfg.c +7 −4
tools/testing/selftests/bpf/verifier/direct_packet_access.c +2 −1
tools/testing/selftests/bpf/verifier/helper_access_var_len.c +15 −13
tools/testing/selftests/bpf/verifier/loops1.c +161 −0
tools/testing/selftests/bpf/verifier/prevent_map_lookup.c +0 −15
tools/testing/selftests/bpf/verifier/sock.c +18 −0

arch/alpha/include/uapi/asm/socket.h+2−0

@@ -122,6 +122,8 @@ · 121 unmodified lines
122	122	#define SO_RCVTIMEO_NEW 66
123	123	#define SO_SNDTIMEO_NEW 67
124	124
	125	+#define SO_DETACH_REUSEPORT_BPF 68
	126	+
125	127	#if !defined(__KERNEL__)
126	128
127	129	#if __BITS_PER_LONG == 64

arch/mips/include/uapi/asm/socket.h+2−0

@@ -133,6 +133,8 @@ · 132 unmodified lines
133	133	#define SO_RCVTIMEO_NEW 66
134	134	#define SO_SNDTIMEO_NEW 67
135	135
	136	+#define SO_DETACH_REUSEPORT_BPF 68
	137	+
136	138	#if !defined(__KERNEL__)
137	139
138	140	#if __BITS_PER_LONG == 64

arch/parisc/include/uapi/asm/socket.h+2−0

@@ -114,6 +114,8 @@ · 113 unmodified lines
114	114	#define SO_RCVTIMEO_NEW 0x4040
115	115	#define SO_SNDTIMEO_NEW 0x4041
116	116
	117	+#define SO_DETACH_REUSEPORT_BPF 0x4042
	118	+
117	119	#if !defined(__KERNEL__)
118	120
119	121	#if __BITS_PER_LONG == 64

arch/sparc/include/uapi/asm/socket.h+2−0

@@ -115,6 +115,8 @@ · 114 unmodified lines
115	115	#define SO_RCVTIMEO_NEW 0x0044
116	116	#define SO_SNDTIMEO_NEW 0x0045
117	117
	118	+#define SO_DETACH_REUSEPORT_BPF 0x0047
	119	+
118	120	#if !defined(__KERNEL__)
119	121
120	122

include/linux/bpf.h+25−0

@@ -277,6 +277,7 @@ enum bpf_reg_type { · 276 unmodified lines
277	277	PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */
278	278	PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
279	279	PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */
	280	+ PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */
280	281	};
281	282
282	283	/* The information passed from prog-specific *_is_valid_access
@@ -1098,6 +1099,15 @@ u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, · 815 unmodified lines
1098	1099	struct bpf_insn *insn_buf,
1099	1100	struct bpf_prog *prog,
1100	1101	u32 *target_size);
	1102	+
	1103	+bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
	1104	+ struct bpf_insn_access_aux *info);
	1105	+
	1106	+u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
	1107	+ const struct bpf_insn *si,
	1108	+ struct bpf_insn *insn_buf,
	1109	+ struct bpf_prog *prog,
	1110	+ u32 *target_size);
1101	1111	#else
1102	1112	static inline bool bpf_tcp_sock_is_valid_access(int off, int size,
1103	1113	enum bpf_access_type type,
@@ -1114,6 +1124,21 @@ static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, · 10 unmodified lines
1114	1124	{
1115	1125	return 0;
1116	1126	}
	1127	+static inline bool bpf_xdp_sock_is_valid_access(int off, int size,
	1128	+ enum bpf_access_type type,
	1129	+ struct bpf_insn_access_aux *info)
	1130	+{
	1131	+ return false;
	1132	+}
	1133	+
	1134	+static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
	1135	+ const struct bpf_insn *si,
	1136	+ struct bpf_insn *insn_buf,
	1137	+ struct bpf_prog *prog,
	1138	+ u32 *target_size)
	1139	+{
	1140	+ return 0;
	1141	+}
1117	1142	#endif /* CONFIG_INET */
1118	1143
1119	1144	#endif /* _LINUX_BPF_H */

include/linux/bpf_verifier.h+68−1

@@ -136,6 +136,8 @@ struct bpf_reg_state { · 135 unmodified lines
136	136	*/
137	137	s32 subreg_def;
138	138	enum bpf_reg_liveness live;
	139	+ /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */
	140	+ bool precise;
139	141	};
140	142
141	143	enum bpf_stack_slot_type {
@@ -187,14 +189,77 @@ struct bpf_func_state { · 45 unmodified lines
187	189	struct bpf_stack_state *stack;
188	190	};
189	191
	192	+struct bpf_idx_pair {
	193	+ u32 prev_idx;
	194	+ u32 idx;
	195	+};
	196	+
190	197	#define MAX_CALL_FRAMES 8
191	198	struct bpf_verifier_state {
192	199	/* call stack tracking */
193	200	struct bpf_func_state *frame[MAX_CALL_FRAMES];
	201	+ struct bpf_verifier_state *parent;
	202	+ /*
	203	+ * 'branches' field is the number of branches left to explore:
	204	+ * 0 - all possible paths from this state reached bpf_exit or
	205	+ * were safely pruned
	206	+ * 1 - at least one path is being explored.
	207	+ * This state hasn't reached bpf_exit
	208	+ * 2 - at least two paths are being explored.
	209	+ * This state is an immediate parent of two children.
	210	+ * One is fallthrough branch with branches==1 and another
	211	+ * state is pushed into stack (to be explored later) also with
	212	+ * branches==1. The parent of this state has branches==1.
	213	+ * The verifier state tree connected via 'parent' pointer looks like:
	214	+ * 1
	215	+ * 1
	216	+ * 2 -> 1 (first 'if' pushed into stack)
	217	+ * 1
	218	+ * 2 -> 1 (second 'if' pushed into stack)
	219	+ * 1
	220	+ * 1
	221	+ * 1 bpf_exit.
	222	+ *
	223	+ * Once do_check() reaches bpf_exit, it calls update_branch_counts()
	224	+ * and the verifier state tree will look:
	225	+ * 1
	226	+ * 1
	227	+ * 2 -> 1 (first 'if' pushed into stack)
	228	+ * 1
	229	+ * 1 -> 1 (second 'if' pushed into stack)
	230	+ * 0
	231	+ * 0
	232	+ * 0 bpf_exit.
	233	+ * After pop_stack() the do_check() will resume at second 'if'.
	234	+ *
	235	+ * If is_state_visited() sees a state with branches > 0 it means
	236	+ * there is a loop. If such state is exactly equal to the current state
	237	+ * it's an infinite loop. Note states_equal() checks for states
	238	+ * equvalency, so two states being 'states_equal' does not mean
	239	+ * infinite loop. The exact comparison is provided by
	240	+ * states_maybe_looping() function. It's a stronger pre-check and
	241	+ * much faster than states_equal().
	242	+ *
	243	+ * This algorithm may not find all possible infinite loops or
	244	+ * loop iteration count may be too high.
	245	+ * In such cases BPF_COMPLEXITY_LIMIT_INSNS limit kicks in.
	246	+ */
	247	+ u32 branches;
194	248	u32 insn_idx;
195	249	u32 curframe;
196	250	u32 active_spin_lock;
197	251	bool speculative;
	252	+
	253	+ /* first and last insn idx of this verifier state */
	254	+ u32 first_insn_idx;
	255	+ u32 last_insn_idx;
	256	+ /* jmp history recorded from first to last.
	257	+ * backtracking is using it to go from last to first.
	258	+ * For most states jmp_history_cnt is [0-3].
	259	+ * For loops can go up to ~40.
	260	+ */
	261	+ struct bpf_idx_pair *jmp_history;
	262	+ u32 jmp_history_cnt;
198	263	};
199	264
200	265	#define bpf_get_spilled_reg(slot, frame) \
@@ -309,7 +374,9 @@ struct bpf_verifier_env { · 108 unmodified lines
309	374	} cfg;
310	375	u32 subprog_cnt;
311	376	/* number of instructions analyzed by the verifier */
312		− u32 insn_processed;
	377	+ u32 prev_insn_processed, insn_processed;
	378	+ /* number of jmps, calls, exits analyzed so far */
	379	+ u32 prev_jmps_processed, jmps_processed;
313	380	/* total verification time */
314	381	u64 verification_time;
315	382	/* maximum number of verifier states kept in 'branching' instructions */

include/net/sock_reuseport.h+2−0

@@ -35,6 +35,8 @@ extern struct sock reuseport_select_sock(struct sock sk, · 34 unmodified lines
35	35	struct sk_buff *skb,
36	36	int hdr_len);
37	37	extern int reuseport_attach_prog(struct sock sk, struct bpf_prog prog);
	38	+extern int reuseport_detach_prog(struct sock *sk);
	39	+
38	40	int reuseport_get_id(struct sock_reuseport *reuse);
39	41
40	42	#endif /* _SOCK_REUSEPORT_H */

include/net/xdp_sock.h+2−2

@@ -58,11 +58,11 @@ struct xdp_sock { · 57 unmodified lines
58	58	struct xdp_umem *umem;
59	59	struct list_head flush_node;
60	60	u16 queue_id;
61		− struct xsk_queue *tx ____cacheline_aligned_in_smp;
62		− struct list_head list;
63	61	bool zc;
64	62	/* Protects multiple processes in the control path */
65	63	struct mutex mutex;
	64	+ struct xsk_queue *tx ____cacheline_aligned_in_smp;
	65	+ struct list_head list;
66	66	/* Mutual exclusion of NAPI TX thread and sendmsg error paths
67	67	* in the SKB destructor callback.
68	68	*/

include/uapi/asm-generic/socket.h+2−0

@@ -117,6 +117,8 @@ · 116 unmodified lines
117	117	#define SO_RCVTIMEO_NEW 66
118	118	#define SO_SNDTIMEO_NEW 67
119	119
	120	+#define SO_DETACH_REUSEPORT_BPF 68
	121	+
120	122	#if !defined(__KERNEL__)
121	123
122	124	#if __BITS_PER_LONG == 64 \|\| (defined(__x86_64__) && defined(__ILP32__))

include/uapi/linux/bpf.h+6−0

@@ -3085,6 +3085,10 @@ struct bpf_sock_tuple { · 3084 unmodified lines
3085	3085	};
3086	3086	};
3087	3087
	3088	+struct bpf_xdp_sock {
	3089	+ __u32 queue_id;
	3090	+};
	3091	+
3088	3092	#define XDP_PACKET_HEADROOM 256
3089	3093
3090	3094	/* User return codes for XDP prog type.
@@ -3245,6 +3249,7 @@ struct bpf_sock_addr { · 154 unmodified lines
3245	3249	__u32 msg_src_ip6[4]; /* Allows 1,2,4-byte read an 4-byte write.
3246	3250	* Stored in network byte order.
3247	3251	*/
	3252	+ __bpf_md_ptr(struct bpf_sock *, sk);
3248	3253	};
3249	3254
3250	3255	/* User bpf_sock_ops struct to access socket values and specify request ops
@@ -3296,6 +3301,7 @@ struct bpf_sock_ops { · 45 unmodified lines
3296	3301	__u32 sk_txhash;
3297	3302	__u64 bytes_received;
3298	3303	__u64 bytes_acked;
	3304	+ __bpf_md_ptr(struct bpf_sock *, sk);
3299	3305	};
3300	3306
3301	3307	/* Definitions for bpf_sock_ops_cb_flags */

kernel/bpf/Makefile+1−0

@@ -1,5 +1,6 @@
1	1	# SPDX-License-Identifier: GPL-2.0
2	2	obj-y := core.o
	3	+CFLAGS_core.o += $(call cc-disable-warning, override-init)
3	4
4	5	obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o
5	6	obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o

kernel/bpf/devmap.c+1−1

@@ -80,8 +80,8 @@ static u64 dev_map_bitmap_size(const union bpf_attr *attr) · 79 unmodified lines
80	80	static struct bpf_map dev_map_alloc(union bpf_attr attr)
81	81	{
82	82	struct bpf_dtab *dtab;
83		− int err = -EINVAL;
84	83	u64 cost;
	84	+ int err;
85	85
86	86	if (!capable(CAP_NET_ADMIN))
87	87	return ERR_PTR(-EPERM);

kernel/bpf/verifier.c+724−69

@@ -326,7 +326,8 @@ static bool type_is_sk_pointer(enum bpf_reg_type type) · 325 unmodified lines
326	326	{
327	327	return type == PTR_TO_SOCKET \|\|
328	328	type == PTR_TO_SOCK_COMMON \|\|
329		− type == PTR_TO_TCP_SOCK;
	329	+ type == PTR_TO_TCP_SOCK \|\|
	330	+ type == PTR_TO_XDP_SOCK;
330	331	}
331	332
332	333	static bool reg_type_may_be_null(enum bpf_reg_type type)
@@ -398,6 +399,7 @@ static const char * const reg_type_str[] = { · 65 unmodified lines
398	399	[PTR_TO_TCP_SOCK] = "tcp_sock",
399	400	[PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
400	401	[PTR_TO_TP_BUFFER] = "tp_buffer",
	402	+ [PTR_TO_XDP_SOCK] = "xdp_sock",
401	403	};
402	404
403	405	static char slot_type_char[] = {
@@ -445,12 +447,12 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 41 unmodified lines
445	447	verbose(env, " R%d", i);
446	448	print_liveness(env, reg->live);
447	449	verbose(env, "=%s", reg_type_str[t]);
	450	+ if (t == SCALAR_VALUE && reg->precise)
	451	+ verbose(env, "P");
448	452	if ((t == SCALAR_VALUE \|\| t == PTR_TO_STACK) &&
449	453	tnum_is_const(reg->var_off)) {
450	454	/* reg->off should be 0 for SCALAR_VALUE */
451	455	verbose(env, "%lld", reg->var_off.value + reg->off);
452		− if (t == PTR_TO_STACK)
453		− verbose(env, ",call_%d", func(env, reg)->callsite);
454	456	} else {
455	457	verbose(env, "(id=%d", reg->id);
456	458	if (reg_type_may_be_refcounted_or_null(t))
@@ -512,11 +514,17 @@ static void print_verifier_state(struct bpf_verifier_env *env, · 55 unmodified lines
512	514	continue;
513	515	verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
514	516	print_liveness(env, state->stack[i].spilled_ptr.live);
515		− if (state->stack[i].slot_type[0] == STACK_SPILL)
516		− verbose(env, "=%s",
517		− reg_type_str[state->stack[i].spilled_ptr.type]);
518		− else
	517	+ if (state->stack[i].slot_type[0] == STACK_SPILL) {
	518	+ reg = &state->stack[i].spilled_ptr;
	519	+ t = reg->type;
	520	+ verbose(env, "=%s", reg_type_str[t]);
	521	+ if (t == SCALAR_VALUE && reg->precise)
	522	+ verbose(env, "P");
	523	+ if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
	524	+ verbose(env, "%lld", reg->var_off.value + reg->off);
	525	+ } else {
519	526	verbose(env, "=%s", types_buf);
	527	+ }
520	528	}
521	529	if (state->acquired_refs && state->refs[0].id) {
522	530	verbose(env, " refs=%d", state->refs[0].id);
@@ -665,6 +673,13 @@ static void free_func_state(struct bpf_func_state *state) · 142 unmodified lines
665	673	kfree(state);
666	674	}
667	675
	676	+static void clear_jmp_history(struct bpf_verifier_state *state)
	677	+{
	678	+ kfree(state->jmp_history);
	679	+ state->jmp_history = NULL;
	680	+ state->jmp_history_cnt = 0;
	681	+}
	682	+
668	683	static void free_verifier_state(struct bpf_verifier_state *state,
669	684	bool free_self)
670	685	{
@@ -674,6 +689,7 @@ static void free_verifier_state(struct bpf_verifier_state *state, · 3 unmodified lines
674	689	free_func_state(state->frame[i]);
675	690	state->frame[i] = NULL;
676	691	}
	692	+ clear_jmp_history(state);
677	693	if (free_self)
678	694	kfree(state);
679	695	}
@@ -701,8 +717,18 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 21 unmodified lines
701	717	const struct bpf_verifier_state *src)
702	718	{
703	719	struct bpf_func_state *dst;
	720	+ u32 jmp_sz = sizeof(struct bpf_idx_pair) * src->jmp_history_cnt;
704	721	int i, err;
705	722
	723	+ if (dst_state->jmp_history_cnt < src->jmp_history_cnt) {
	724	+ kfree(dst_state->jmp_history);
	725	+ dst_state->jmp_history = kmalloc(jmp_sz, GFP_USER);
	726	+ if (!dst_state->jmp_history)
	727	+ return -ENOMEM;
	728	+ }
	729	+ memcpy(dst_state->jmp_history, src->jmp_history, jmp_sz);
	730	+ dst_state->jmp_history_cnt = src->jmp_history_cnt;
	731	+
706	732	/* if dst has more stack frames then src frame, free them */
707	733	for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
708	734	free_func_state(dst_state->frame[i]);
@@ -711,6 +737,10 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 2 unmodified lines
711	737	dst_state->speculative = src->speculative;
712	738	dst_state->curframe = src->curframe;
713	739	dst_state->active_spin_lock = src->active_spin_lock;
	740	+ dst_state->branches = src->branches;
	741	+ dst_state->parent = src->parent;
	742	+ dst_state->first_insn_idx = src->first_insn_idx;
	743	+ dst_state->last_insn_idx = src->last_insn_idx;
714	744	for (i = 0; i <= src->curframe; i++) {
715	745	dst = dst_state->frame[i];
716	746	if (!dst) {
@@ -726,6 +756,23 @@ static int copy_verifier_state(struct bpf_verifier_state *dst_state, · 9 unmodified lines
726	756	return 0;
727	757	}
728	758
	759	+static void update_branch_counts(struct bpf_verifier_env env, struct bpf_verifier_state st)
	760	+{
	761	+ while (st) {
	762	+ u32 br = --st->branches;
	763	+
	764	+ /* WARN_ON(br > 1) technically makes sense here,
	765	+ * but see comment in push_stack(), hence:
	766	+ */
	767	+ WARN_ONCE((int)br < 0,
	768	+ "BUG update_branch_counts:branches_to_explore=%d\n",
	769	+ br);
	770	+ if (br)
	771	+ break;
	772	+ st = st->parent;
	773	+ }
	774	+}
	775	+
729	776	static int pop_stack(struct bpf_verifier_env env, int prev_insn_idx,
730	777	int *insn_idx)
731	778	{
@@ -779,6 +826,18 @@ static struct bpf_verifier_state push_stack(struct bpf_verifier_env env, · 47 unmodified lines
779	826	env->stack_size);
780	827	goto err;
781	828	}
	829	+ if (elem->st.parent) {
	830	+ ++elem->st.parent->branches;
	831	+ /* WARN_ON(branches > 2) technically makes sense here,
	832	+ * but
	833	+ * 1. speculative states will bump 'branches' for non-branch
	834	+ * instructions
	835	+ * 2. is_state_visited() heuristics may decide not to create
	836	+ * a new state for a sequence of branches and all such current
	837	+ * and cloned states will be pointing to a single parent state
	838	+ * which might have large 'branches' count.
	839	+ */
	840	+ }
782	841	return &elem->st;
783	842	err:
784	843	free_verifier_state(env->cur_state, true);
@@ -926,6 +985,9 @@ static void __mark_reg_unbounded(struct bpf_reg_state *reg) · 141 unmodified lines
926	985	reg->smax_value = S64_MAX;
927	986	reg->umin_value = 0;
928	987	reg->umax_value = U64_MAX;
	988	+
	989	+ /* constant backtracking is enabled for root only for now */
	990	+ reg->precise = capable(CAP_SYS_ADMIN) ? false : true;
929	991	}
930	992
931	993	/* Mark a register as having a completely unknown (scalar) value. */
@@ -1337,6 +1399,389 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, · 405 unmodified lines
1337	1399	return 0;
1338	1400	}
1339	1401
	1402	+/* for any branch, call, exit record the history of jmps in the given state */
	1403	+static int push_jmp_history(struct bpf_verifier_env *env,
	1404	+ struct bpf_verifier_state *cur)
	1405	+{
	1406	+ u32 cnt = cur->jmp_history_cnt;
	1407	+ struct bpf_idx_pair *p;
	1408	+
	1409	+ cnt++;
	1410	+ p = krealloc(cur->jmp_history, cnt * sizeof(*p), GFP_USER);
	1411	+ if (!p)
	1412	+ return -ENOMEM;
	1413	+ p[cnt - 1].idx = env->insn_idx;
	1414	+ p[cnt - 1].prev_idx = env->prev_insn_idx;
	1415	+ cur->jmp_history = p;
	1416	+ cur->jmp_history_cnt = cnt;
	1417	+ return 0;
	1418	+}
	1419	+
	1420	+/* Backtrack one insn at a time. If idx is not at the top of recorded
	1421	+ * history then previous instruction came from straight line execution.
	1422	+ */
	1423	+static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
	1424	+ u32 *history)
	1425	+{
	1426	+ u32 cnt = *history;
	1427	+
	1428	+ if (cnt && st->jmp_history[cnt - 1].idx == i) {
	1429	+ i = st->jmp_history[cnt - 1].prev_idx;
	1430	+ (*history)--;
	1431	+ } else {
	1432	+ i--;
	1433	+ }
	1434	+ return i;
	1435	+}
	1436	+
	1437	+/* For given verifier state backtrack_insn() is called from the last insn to
	1438	+ * the first insn. Its purpose is to compute a bitmask of registers and
	1439	+ * stack slots that needs precision in the parent verifier state.
	1440	+ */
	1441	+static int backtrack_insn(struct bpf_verifier_env *env, int idx,
	1442	+ u32 reg_mask, u64 stack_mask)
	1443	+{
	1444	+ const struct bpf_insn_cbs cbs = {
	1445	+ .cb_print = verbose,
	1446	+ .private_data = env,
	1447	+ };
	1448	+ struct bpf_insn *insn = env->prog->insnsi + idx;
	1449	+ u8 class = BPF_CLASS(insn->code);
	1450	+ u8 opcode = BPF_OP(insn->code);
	1451	+ u8 mode = BPF_MODE(insn->code);
	1452	+ u32 dreg = 1u << insn->dst_reg;
	1453	+ u32 sreg = 1u << insn->src_reg;
	1454	+ u32 spi;
	1455	+
	1456	+ if (insn->code == 0)
	1457	+ return 0;
	1458	+ if (env->log.level & BPF_LOG_LEVEL) {
	1459	+ verbose(env, "regs=%x stack=%llx before ", reg_mask, stack_mask);
	1460	+ verbose(env, "%d: ", idx);
	1461	+ print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
	1462	+ }
	1463	+
	1464	+ if (class == BPF_ALU \|\| class == BPF_ALU64) {
	1465	+ if (!(*reg_mask & dreg))
	1466	+ return 0;
	1467	+ if (opcode == BPF_MOV) {
	1468	+ if (BPF_SRC(insn->code) == BPF_X) {
	1469	+ /* dreg = sreg
	1470	+ * dreg needs precision after this insn
	1471	+ * sreg needs precision before this insn
	1472	+ */
	1473	+ *reg_mask &= ~dreg;
	1474	+ *reg_mask \|= sreg;
	1475	+ } else {
	1476	+ /* dreg = K
	1477	+ * dreg needs precision after this insn.
	1478	+ * Corresponding register is already marked
	1479	+ * as precise=true in this verifier state.
	1480	+ * No further markings in parent are necessary
	1481	+ */
	1482	+ *reg_mask &= ~dreg;
	1483	+ }
	1484	+ } else {
	1485	+ if (BPF_SRC(insn->code) == BPF_X) {
	1486	+ /* dreg += sreg
	1487	+ * both dreg and sreg need precision
	1488	+ * before this insn
	1489	+ */
	1490	+ *reg_mask \|= sreg;
	1491	+ } /* else dreg += K
	1492	+ * dreg still needs precision before this insn
	1493	+ */
	1494	+ }
	1495	+ } else if (class == BPF_LDX) {
	1496	+ if (!(*reg_mask & dreg))
	1497	+ return 0;
	1498	+ *reg_mask &= ~dreg;
	1499	+
	1500	+ /* scalars can only be spilled into stack w/o losing precision.
	1501	+ * Load from any other memory can be zero extended.
	1502	+ * The desire to keep that precision is already indicated
	1503	+ * by 'precise' mark in corresponding register of this state.
	1504	+ * No further tracking necessary.
	1505	+ */
	1506	+ if (insn->src_reg != BPF_REG_FP)
	1507	+ return 0;
	1508	+ if (BPF_SIZE(insn->code) != BPF_DW)
	1509	+ return 0;
	1510	+
	1511	+ /* dreg = (u64 )[fp - off] was a fill from the stack.
	1512	+ * that [fp - off] slot contains scalar that needs to be
	1513	+ * tracked with precision
	1514	+ */
	1515	+ spi = (-insn->off - 1) / BPF_REG_SIZE;
	1516	+ if (spi >= 64) {
	1517	+ verbose(env, "BUG spi %d\n", spi);
	1518	+ WARN_ONCE(1, "verifier backtracking bug");
	1519	+ return -EFAULT;
	1520	+ }
	1521	+ *stack_mask \|= 1ull << spi;
	1522	+ } else if (class == BPF_STX) {
	1523	+ if (*reg_mask & dreg)
	1524	+ /* stx shouldn't be using _scalar_ dst_reg
	1525	+ * to access memory. It means backtracking
	1526	+ * encountered a case of pointer subtraction.
	1527	+ */
	1528	+ return -ENOTSUPP;
	1529	+ /* scalars can only be spilled into stack */
	1530	+ if (insn->dst_reg != BPF_REG_FP)
	1531	+ return 0;
	1532	+ if (BPF_SIZE(insn->code) != BPF_DW)
	1533	+ return 0;
	1534	+ spi = (-insn->off - 1) / BPF_REG_SIZE;
	1535	+ if (spi >= 64) {
	1536	+ verbose(env, "BUG spi %d\n", spi);
	1537	+ WARN_ONCE(1, "verifier backtracking bug");
	1538	+ return -EFAULT;
	1539	+ }
	1540	+ if (!(*stack_mask & (1ull << spi)))
	1541	+ return 0;
	1542	+ *stack_mask &= ~(1ull << spi);
	1543	+ *reg_mask \|= sreg;
	1544	+ } else if (class == BPF_JMP \|\| class == BPF_JMP32) {
	1545	+ if (opcode == BPF_CALL) {
	1546	+ if (insn->src_reg == BPF_PSEUDO_CALL)
	1547	+ return -ENOTSUPP;
	1548	+ /* regular helper call sets R0 */
	1549	+ *reg_mask &= ~1;
	1550	+ if (*reg_mask & 0x3f) {
	1551	+ /* if backtracing was looking for registers R1-R5
	1552	+ * they should have been found already.
	1553	+ */
	1554	+ verbose(env, "BUG regs %x\n", *reg_mask);
	1555	+ WARN_ONCE(1, "verifier backtracking bug");
	1556	+ return -EFAULT;
	1557	+ }
	1558	+ } else if (opcode == BPF_EXIT) {
	1559	+ return -ENOTSUPP;
	1560	+ }
	1561	+ } else if (class == BPF_LD) {
	1562	+ if (!(*reg_mask & dreg))
	1563	+ return 0;
	1564	+ *reg_mask &= ~dreg;
	1565	+ /* It's ld_imm64 or ld_abs or ld_ind.
	1566	+ * For ld_imm64 no further tracking of precision
	1567	+ * into parent is necessary
	1568	+ */
	1569	+ if (mode == BPF_IND \|\| mode == BPF_ABS)
	1570	+ /* to be analyzed */
	1571	+ return -ENOTSUPP;
	1572	+ } else if (class == BPF_ST) {
	1573	+ if (*reg_mask & dreg)
	1574	+ /* likely pointer subtraction */
	1575	+ return -ENOTSUPP;
	1576	+ }
	1577	+ return 0;
	1578	+}
	1579	+
	1580	+/* the scalar precision tracking algorithm:
	1581	+ * . at the start all registers have precise=false.
	1582	+ * . scalar ranges are tracked as normal through alu and jmp insns.
	1583	+ * . once precise value of the scalar register is used in:
	1584	+ * . ptr + scalar alu
	1585	+ * . if (scalar cond K\|scalar)
	1586	+ * . helper_call(.., scalar, ...) where ARG_CONST is expected
	1587	+ * backtrack through the verifier states and mark all registers and
	1588	+ * stack slots with spilled constants that these scalar regisers
	1589	+ * should be precise.
	1590	+ * . during state pruning two registers (or spilled stack slots)
	1591	+ * are equivalent if both are not precise.
	1592	+ *
	1593	+ * Note the verifier cannot simply walk register parentage chain,
	1594	+ * since many different registers and stack slots could have been
	1595	+ * used to compute single precise scalar.
	1596	+ *
	1597	+ * The approach of starting with precise=true for all registers and then
	1598	+ * backtrack to mark a register as not precise when the verifier detects
	1599	+ * that program doesn't care about specific value (e.g., when helper
	1600	+ * takes register as ARG_ANYTHING parameter) is not safe.
	1601	+ *
	1602	+ * It's ok to walk single parentage chain of the verifier states.
	1603	+ * It's possible that this backtracking will go all the way till 1st insn.
	1604	+ * All other branches will be explored for needing precision later.
	1605	+ *
	1606	+ * The backtracking needs to deal with cases like:
	1607	+ * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
	1608	+ * r9 -= r8
	1609	+ * r5 = r9
	1610	+ * if r5 > 0x79f goto pc+7
	1611	+ * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
	1612	+ * r5 += 1
	1613	+ * ...
	1614	+ * call bpf_perf_event_output#25
	1615	+ * where .arg5_type = ARG_CONST_SIZE_OR_ZERO
	1616	+ *
	1617	+ * and this case:
	1618	+ * r6 = 1
	1619	+ * call foo // uses callee's r6 inside to compute r0
	1620	+ * r0 += r6
	1621	+ * if r0 == 0 goto
	1622	+ *
	1623	+ * to track above reg_mask/stack_mask needs to be independent for each frame.
	1624	+ *
	1625	+ * Also if parent's curframe > frame where backtracking started,
	1626	+ * the verifier need to mark registers in both frames, otherwise callees
	1627	+ * may incorrectly prune callers. This is similar to
	1628	+ * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
	1629	+ *
	1630	+ * For now backtracking falls back into conservative marking.
	1631	+ */
	1632	+static void mark_all_scalars_precise(struct bpf_verifier_env *env,
	1633	+ struct bpf_verifier_state *st)
	1634	+{
	1635	+ struct bpf_func_state *func;
	1636	+ struct bpf_reg_state *reg;
	1637	+ int i, j;
	1638	+
	1639	+ /* big hammer: mark all scalars precise in this path.
	1640	+ * pop_stack may still get !precise scalars.
	1641	+ */
	1642	+ for (; st; st = st->parent)
	1643	+ for (i = 0; i <= st->curframe; i++) {
	1644	+ func = st->frame[i];
	1645	+ for (j = 0; j < BPF_REG_FP; j++) {
	1646	+ reg = &func->regs[j];
	1647	+ if (reg->type != SCALAR_VALUE)
	1648	+ continue;
	1649	+ reg->precise = true;
	1650	+ }
	1651	+ for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
	1652	+ if (func->stack[j].slot_type[0] != STACK_SPILL)
	1653	+ continue;
	1654	+ reg = &func->stack[j].spilled_ptr;
	1655	+ if (reg->type != SCALAR_VALUE)
	1656	+ continue;
	1657	+ reg->precise = true;
	1658	+ }
	1659	+ }
	1660	+}
	1661	+
	1662	+static int mark_chain_precision(struct bpf_verifier_env *env, int regno)
	1663	+{
	1664	+ struct bpf_verifier_state *st = env->cur_state;
	1665	+ int first_idx = st->first_insn_idx;
	1666	+ int last_idx = env->insn_idx;
	1667	+ struct bpf_func_state *func;
	1668	+ struct bpf_reg_state *reg;
	1669	+ u32 reg_mask = 1u << regno;
	1670	+ u64 stack_mask = 0;
	1671	+ bool skip_first = true;
	1672	+ int i, err;
	1673	+
	1674	+ if (!env->allow_ptr_leaks)
	1675	+ /* backtracking is root only for now */
	1676	+ return 0;
	1677	+
	1678	+ func = st->frame[st->curframe];
	1679	+ reg = &func->regs[regno];
	1680	+ if (reg->type != SCALAR_VALUE) {
	1681	+ WARN_ONCE(1, "backtracing misuse");
	1682	+ return -EFAULT;
	1683	+ }
	1684	+ if (reg->precise)
	1685	+ return 0;
	1686	+ func->regs[regno].precise = true;
	1687	+
	1688	+ for (;;) {
	1689	+ DECLARE_BITMAP(mask, 64);
	1690	+ bool new_marks = false;
	1691	+ u32 history = st->jmp_history_cnt;
	1692	+
	1693	+ if (env->log.level & BPF_LOG_LEVEL)
	1694	+ verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
	1695	+ for (i = last_idx;;) {
	1696	+ if (skip_first) {
	1697	+ err = 0;
	1698	+ skip_first = false;
	1699	+ } else {
	1700	+ err = backtrack_insn(env, i, &reg_mask, &stack_mask);
	1701	+ }
	1702	+ if (err == -ENOTSUPP) {
	1703	+ mark_all_scalars_precise(env, st);
	1704	+ return 0;
	1705	+ } else if (err) {
	1706	+ return err;
	1707	+ }
	1708	+ if (!reg_mask && !stack_mask)
	1709	+ /* Found assignment(s) into tracked register in this state.
	1710	+ * Since this state is already marked, just return.
	1711	+ * Nothing to be tracked further in the parent state.
	1712	+ */
	1713	+ return 0;
	1714	+ if (i == first_idx)
	1715	+ break;
	1716	+ i = get_prev_insn_idx(st, i, &history);
	1717	+ if (i >= env->prog->len) {
	1718	+ /* This can happen if backtracking reached insn 0
	1719	+ * and there are still reg_mask or stack_mask
	1720	+ * to backtrack.
	1721	+ * It means the backtracking missed the spot where
	1722	+ * particular register was initialized with a constant.
	1723	+ */
	1724	+ verbose(env, "BUG backtracking idx %d\n", i);
	1725	+ WARN_ONCE(1, "verifier backtracking bug");
	1726	+ return -EFAULT;
	1727	+ }
	1728	+ }
	1729	+ st = st->parent;
	1730	+ if (!st)
	1731	+ break;
	1732	+
	1733	+ func = st->frame[st->curframe];
	1734	+ bitmap_from_u64(mask, reg_mask);
	1735	+ for_each_set_bit(i, mask, 32) {
	1736	+ reg = &func->regs[i];
	1737	+ if (reg->type != SCALAR_VALUE)
	1738	+ continue;
	1739	+ if (!reg->precise)
	1740	+ new_marks = true;
	1741	+ reg->precise = true;
	1742	+ }
	1743	+
	1744	+ bitmap_from_u64(mask, stack_mask);
	1745	+ for_each_set_bit(i, mask, 64) {
	1746	+ if (i >= func->allocated_stack / BPF_REG_SIZE) {
	1747	+ /* This can happen if backtracking
	1748	+ * is propagating stack precision where
	1749	+ * caller has larger stack frame
	1750	+ * than callee, but backtrack_insn() should
	1751	+ * have returned -ENOTSUPP.
	1752	+ */
	1753	+ verbose(env, "BUG spi %d stack_size %d\n",
	1754	+ i, func->allocated_stack);
	1755	+ WARN_ONCE(1, "verifier backtracking bug");
	1756	+ return -EFAULT;
	1757	+ }
	1758	+
	1759	+ if (func->stack[i].slot_type[0] != STACK_SPILL)
	1760	+ continue;
	1761	+ reg = &func->stack[i].spilled_ptr;
	1762	+ if (reg->type != SCALAR_VALUE)
	1763	+ continue;
	1764	+ if (!reg->precise)
	1765	+ new_marks = true;
	1766	+ reg->precise = true;
	1767	+ }
	1768	+ if (env->log.level & BPF_LOG_LEVEL) {
	1769	+ print_verifier_state(env, func);
	1770	+ verbose(env, "parent %s regs=%x stack=%llx marks\n",
	1771	+ new_marks ? "didn't have" : "already had",
	1772	+ reg_mask, stack_mask);
	1773	+ }
	1774	+
	1775	+ if (!new_marks)
	1776	+ break;
	1777	+
	1778	+ last_idx = st->last_insn_idx;
	1779	+ first_idx = st->first_insn_idx;
	1780	+ }
	1781	+ return 0;
	1782	+}
	1783	+
	1784	+
1340	1785	static bool is_spillable_regtype(enum bpf_reg_type type)
1341	1786	{
1342	1787	switch (type) {
@@ -1355,6 +1800,7 @@ static bool is_spillable_regtype(enum bpf_reg_type type) · 12 unmodified lines
1355	1800	case PTR_TO_SOCK_COMMON_OR_NULL:
1356	1801	case PTR_TO_TCP_SOCK:
1357	1802	case PTR_TO_TCP_SOCK_OR_NULL:
	1803	+ case PTR_TO_XDP_SOCK:
1358	1804	return true;
1359	1805	default:
1360	1806	return false;
@@ -1367,6 +1813,23 @@ static bool register_is_null(struct bpf_reg_state *reg) · 6 unmodified lines
1367	1813	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
1368	1814	}
1369	1815
	1816	+static bool register_is_const(struct bpf_reg_state *reg)
	1817	+{
	1818	+ return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
	1819	+}
	1820	+
	1821	+static void save_register_state(struct bpf_func_state *state,
	1822	+ int spi, struct bpf_reg_state *reg)
	1823	+{
	1824	+ int i;
	1825	+
	1826	+ state->stack[spi].spilled_ptr = *reg;
	1827	+ state->stack[spi].spilled_ptr.live \|= REG_LIVE_WRITTEN;
	1828	+
	1829	+ for (i = 0; i < BPF_REG_SIZE; i++)
	1830	+ state->stack[spi].slot_type[i] = STACK_SPILL;
	1831	+}
	1832	+
1370	1833	/* check_stack_read/write functions track spill/fill of registers,
1371	1834	* stack boundary and alignment are checked in check_mem_access()
1372	1835	*/
@@ -1376,7 +1839,8 @@ static int check_stack_write(struct bpf_verifier_env *env, · 3 unmodified lines
1376	1839	{
1377	1840	struct bpf_func_state cur; / state of the current function */
1378	1841	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
1379		− enum bpf_reg_type type;
	1842	+ u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg;
	1843	+ struct bpf_reg_state *reg = NULL;
1380	1844
1381	1845	err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE),
1382	1846	state->acquired_refs, true);
@@ -1393,27 +1857,48 @@ static int check_stack_write(struct bpf_verifier_env *env, · 10 unmodified lines
1393	1857	}
1394	1858
1395	1859	cur = env->cur_state->frame[env->cur_state->curframe];
1396		− if (value_regno >= 0 &&
1397		− is_spillable_regtype((type = cur->regs[value_regno].type))) {
1398		−
	1860	+ if (value_regno >= 0)
	1861	+ reg = &cur->regs[value_regno];
	1862	+
	1863	+ if (reg && size == BPF_REG_SIZE && register_is_const(reg) &&
	1864	+ !register_is_null(reg) && env->allow_ptr_leaks) {
	1865	+ if (dst_reg != BPF_REG_FP) {
	1866	+ /* The backtracking logic can only recognize explicit
	1867	+ * stack slot address like [fp - 8]. Other spill of
	1868	+ * scalar via different register has to be conervative.
	1869	+ * Backtrack from here and mark all registers as precise
	1870	+ * that contributed into 'reg' being a constant.
	1871	+ */
	1872	+ err = mark_chain_precision(env, value_regno);
	1873	+ if (err)
	1874	+ return err;
	1875	+ }
	1876	+ save_register_state(state, spi, reg);
	1877	+ } else if (reg && is_spillable_regtype(reg->type)) {
1399	1878	/* register containing pointer is being spilled into stack */
1400	1879	if (size != BPF_REG_SIZE) {
	1880	+ verbose_linfo(env, insn_idx, "; ");
1401	1881	verbose(env, "invalid size of register spill\n");
1402	1882	return -EACCES;
1403	1883	}
1404	1884
1405		− if (state != cur && type == PTR_TO_STACK) {
	1885	+ if (state != cur && reg->type == PTR_TO_STACK) {
1406	1886	verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
1407	1887	return -EINVAL;
1408	1888	}
1409	1889
1410		− /* save register state */
1411		− state->stack[spi].spilled_ptr = cur->regs[value_regno];
1412		− state->stack[spi].spilled_ptr.live \|= REG_LIVE_WRITTEN;
	1890	+ if (!env->allow_ptr_leaks) {
	1891	+ bool sanitize = false;
1413	1892
1414		− for (i = 0; i < BPF_REG_SIZE; i++) {
1415		− if (state->stack[spi].slot_type[i] == STACK_MISC &&
1416		− !env->allow_ptr_leaks) {
	1893	+ if (state->stack[spi].slot_type[0] == STACK_SPILL &&
	1894	+ register_is_const(&state->stack[spi].spilled_ptr))
	1895	+ sanitize = true;
	1896	+ for (i = 0; i < BPF_REG_SIZE; i++)
	1897	+ if (state->stack[spi].slot_type[i] == STACK_MISC) {
	1898	+ sanitize = true;
	1899	+ break;
	1900	+ }
	1901	+ if (sanitize) {
1417	1902	int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
1418	1903	int soff = (-spi - 1) * BPF_REG_SIZE;
1419	1904
@@ -1436,8 +1921,8 @@ static int check_stack_write(struct bpf_verifier_env *env, · 16 unmodified lines
1436	1921	}
1437	1922	*poff = soff;
1438	1923	}
1439		− state->stack[spi].slot_type[i] = STACK_SPILL;
1440	1924	}
	1925	+ save_register_state(state, spi, reg);
1441	1926	} else {
1442	1927	u8 type = STACK_MISC;
1443	1928
@@ -1460,9 +1945,13 @@ static int check_stack_write(struct bpf_verifier_env *env, · 16 unmodified lines
1460	1945	state->stack[spi].spilled_ptr.live \|= REG_LIVE_WRITTEN;
1461	1946
1462	1947	/* when we zero initialize stack slots mark them as such */
1463		− if (value_regno >= 0 &&
1464		− register_is_null(&cur->regs[value_regno]))
	1948	+ if (reg && register_is_null(reg)) {
	1949	+ /* backtracking doesn't work for STACK_ZERO yet. */
	1950	+ err = mark_chain_precision(env, value_regno);
	1951	+ if (err)
	1952	+ return err;
1465	1953	type = STACK_ZERO;
	1954	+ }
1466	1955
1467	1956	/* Mark slots affected by this stack write. */
1468	1957	for (i = 0; i < size; i++)
@@ -1479,6 +1968,7 @@ static int check_stack_read(struct bpf_verifier_env *env, · 10 unmodified lines
1479	1968	struct bpf_verifier_state *vstate = env->cur_state;
1480	1969	struct bpf_func_state *state = vstate->frame[vstate->curframe];
1481	1970	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
	1971	+ struct bpf_reg_state *reg;
1482	1972	u8 *stype;
1483	1973
1484	1974	if (reg_state->allocated_stack <= slot) {
@@ -1487,11 +1977,21 @@ static int check_stack_read(struct bpf_verifier_env *env, · 2 unmodified lines
1487	1977	return -EACCES;
1488	1978	}
1489	1979	stype = reg_state->stack[spi].slot_type;
	1980	+ reg = &reg_state->stack[spi].spilled_ptr;
1490	1981
1491	1982	if (stype[0] == STACK_SPILL) {
1492	1983	if (size != BPF_REG_SIZE) {
1493		− verbose(env, "invalid size of register spill\n");
1494		− return -EACCES;
	1984	+ if (reg->type != SCALAR_VALUE) {
	1985	+ verbose_linfo(env, env->insn_idx, "; ");
	1986	+ verbose(env, "invalid size of register fill\n");
	1987	+ return -EACCES;
	1988	+ }
	1989	+ if (value_regno >= 0) {
	1990	+ mark_reg_unknown(env, state->regs, value_regno);
	1991	+ state->regs[value_regno].live \|= REG_LIVE_WRITTEN;
	1992	+ }
	1993	+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
	1994	+ return 0;
1495	1995	}
1496	1996	for (i = 1; i < BPF_REG_SIZE; i++) {
1497	1997	if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) {
@@ -1502,17 +2002,14 @@ static int check_stack_read(struct bpf_verifier_env *env, · 4 unmodified lines
1502	2002
1503	2003	if (value_regno >= 0) {
1504	2004	/* restore register state from stack */
1505		− state->regs[value_regno] = reg_state->stack[spi].spilled_ptr;
	2005	+ state->regs[value_regno] = *reg;
1506	2006	/* mark reg as written since spilled pointer state likely
1507	2007	* has its liveness marks cleared by is_state_visited()
1508	2008	* which resets stack/reg liveness for state transitions
1509	2009	*/
1510	2010	state->regs[value_regno].live \|= REG_LIVE_WRITTEN;
1511	2011	}
1512		− mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
1513		− reg_state->stack[spi].spilled_ptr.parent,
1514		− REG_LIVE_READ64);
1515		− return 0;
	2012	+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
1516	2013	} else {
1517	2014	int zeros = 0;
1518	2015
@@ -1527,23 +2024,32 @@ static int check_stack_read(struct bpf_verifier_env *env, · 8 unmodified lines
1527	2024	off, i, size);
1528	2025	return -EACCES;
1529	2026	}
1530		− mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
1531		− reg_state->stack[spi].spilled_ptr.parent,
1532		− REG_LIVE_READ64);
	2027	+ mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
1533	2028	if (value_regno >= 0) {
1534	2029	if (zeros == size) {
1535	2030	/* any size read into register is zero extended,
1536	2031	* so the whole register == const_zero
1537	2032	*/
1538	2033	__mark_reg_const_zero(&state->regs[value_regno]);
	2034	+ /* backtracking doesn't support STACK_ZERO yet,
	2035	+ * so mark it precise here, so that later
	2036	+ * backtracking can stop here.
	2037	+ * Backtracking may not need this if this register
	2038	+ * doesn't participate in pointer adjustment.
	2039	+ * Forward propagation of precise flag is not
	2040	+ * necessary either. This mark is only to stop
	2041	+ * backtracking. Any register that contributed
	2042	+ * to const 0 was marked precise before spill.
	2043	+ */
	2044	+ state->regs[value_regno].precise = true;
1539	2045	} else {
1540	2046	/* have read misc data from the stack */
1541	2047	mark_reg_unknown(env, state->regs, value_regno);
1542	2048	}
1543	2049	state->regs[value_regno].live \|= REG_LIVE_WRITTEN;
1544	2050	}
1545		− return 0;
1546	2051	}
	2052	+ return 0;
1547	2053	}
1548	2054
1549	2055	static int check_stack_access(struct bpf_verifier_env *env,
@@ -1835,6 +2341,9 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, · 285 unmodified lines
1835	2341	case PTR_TO_TCP_SOCK:
1836	2342	valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
1837	2343	break;
	2344	+ case PTR_TO_XDP_SOCK:
	2345	+ valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
	2346	+ break;
1838	2347	default:
1839	2348	valid = false;
1840	2349	}
@@ -1999,6 +2508,9 @@ static int check_ptr_alignment(struct bpf_verifier_env *env, · 158 unmodified lines
1999	2508	case PTR_TO_TCP_SOCK:
2000	2509	pointer_desc = "tcp_sock ";
2001	2510	break;
	2511	+ case PTR_TO_XDP_SOCK:
	2512	+ pointer_desc = "xdp_sock ";
	2513	+ break;
2002	2514	default:
2003	2515	break;
2004	2516	}
@@ -2398,7 +2910,7 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 393 unmodified lines
2398	2910	{
2399	2911	struct bpf_reg_state *reg = reg_state(env, regno);
2400	2912	struct bpf_func_state *state = func(env, reg);
2401		− int err, min_off, max_off, i, slot, spi;
	2913	+ int err, min_off, max_off, i, j, slot, spi;
2402	2914
2403	2915	if (reg->type != PTR_TO_STACK) {
2404	2916	/* Allow zero-byte read from NULL, regardless of pointer type */
@@ -2486,6 +2998,14 @@ static int check_stack_boundary(struct bpf_verifier_env *env, int regno, · 81 unmodified lines
2486	2998	*stype = STACK_MISC;
2487	2999	goto mark;
2488	3000	}
	3001	+ if (state->stack[spi].slot_type[0] == STACK_SPILL &&
	3002	+ state->stack[spi].spilled_ptr.type == SCALAR_VALUE) {
	3003	+ __mark_reg_unknown(&state->stack[spi].spilled_ptr);
	3004	+ for (j = 0; j < BPF_REG_SIZE; j++)
	3005	+ state->stack[spi].slot_type[j] = STACK_MISC;
	3006	+ goto mark;
	3007	+ }
	3008	+
2489	3009	err:
2490	3010	if (tnum_is_const(reg->var_off)) {
2491	3011	verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
@@ -2837,6 +3357,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno, · 345 unmodified lines
2837	3357	err = check_helper_mem_access(env, regno - 1,
2838	3358	reg->umax_value,
2839	3359	zero_size_allowed, meta);
	3360	+ if (!err)
	3361	+ err = mark_chain_precision(env, regno);
2840	3362	} else if (arg_type_is_int_ptr(arg_type)) {
2841	3363	int size = int_ptr_type_to_size(arg_type);
2842	3364
@@ -2897,10 +3419,14 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env, · 54 unmodified lines
2897	3419	* appear.
2898	3420	*/
2899	3421	case BPF_MAP_TYPE_CPUMAP:
2900		− case BPF_MAP_TYPE_XSKMAP:
2901	3422	if (func_id != BPF_FUNC_redirect_map)
2902	3423	goto error;
2903	3424	break;
	3425	+ case BPF_MAP_TYPE_XSKMAP:
	3426	+ if (func_id != BPF_FUNC_redirect_map &&
	3427	+ func_id != BPF_FUNC_map_lookup_elem)
	3428	+ goto error;
	3429	+ break;
2904	3430	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
2905	3431	case BPF_MAP_TYPE_HASH_OF_MAPS:
2906	3432	if (func_id != BPF_FUNC_map_lookup_elem)
@@ -3791,6 +4317,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, · 884 unmodified lines
3791	4317	case PTR_TO_SOCK_COMMON_OR_NULL:
3792	4318	case PTR_TO_TCP_SOCK:
3793	4319	case PTR_TO_TCP_SOCK_OR_NULL:
	4320	+ case PTR_TO_XDP_SOCK:
3794	4321	verbose(env, "R%d pointer arithmetic on %s prohibited\n",
3795	4322	dst, reg_type_str[ptr_reg->type]);
3796	4323	return -EACCES;
@@ -4268,6 +4795,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 471 unmodified lines
4268	4795	struct bpf_reg_state regs = state->regs, dst_reg, *src_reg;
4269	4796	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
4270	4797	u8 opcode = BPF_OP(insn->code);
	4798	+ int err;
4271	4799
4272	4800	dst_reg = &regs[insn->dst_reg];
4273	4801	src_reg = NULL;
@@ -4294,11 +4822,17 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, · 20 unmodified lines
4294	4822	* This is legal, but we have to reverse our
4295	4823	* src/dest handling in computing the range
4296	4824	*/
	4825	+ err = mark_chain_precision(env, insn->dst_reg);
	4826	+ if (err)
	4827	+ return err;
4297	4828	return adjust_ptr_min_max_vals(env, insn,
4298	4829	src_reg, dst_reg);
4299	4830	}
4300	4831	} else if (ptr_reg) {
4301	4832	/* pointer += scalar */
	4833	+ err = mark_chain_precision(env, insn->src_reg);
	4834	+ if (err)
	4835	+ return err;
4302	4836	return adjust_ptr_min_max_vals(env, insn,
4303	4837	dst_reg, src_reg);
4304	4838	}
@@ -5030,6 +5564,9 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state, · 725 unmodified lines
5030	5564	if (reg->map_ptr->inner_map_meta) {
5031	5565	reg->type = CONST_PTR_TO_MAP;
5032	5566	reg->map_ptr = reg->map_ptr->inner_map_meta;
	5567	+ } else if (reg->map_ptr->map_type ==
	5568	+ BPF_MAP_TYPE_XSKMAP) {
	5569	+ reg->type = PTR_TO_XDP_SOCK;
5033	5570	} else {
5034	5571	reg->type = PTR_TO_MAP_VALUE;
5035	5572	}
@@ -5201,9 +5738,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 165 unmodified lines
5201	5738	struct bpf_verifier_state *this_branch = env->cur_state;
5202	5739	struct bpf_verifier_state *other_branch;
5203	5740	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
5204		− struct bpf_reg_state dst_reg, other_branch_regs;
	5741	+ struct bpf_reg_state dst_reg, other_branch_regs, *src_reg = NULL;
5205	5742	u8 opcode = BPF_OP(insn->code);
5206	5743	bool is_jmp32;
	5744	+ int pred = -1;
5207	5745	int err;
5208	5746
5209	5747	/* Only conditional jumps are expected to reach here. */
@@ -5228,6 +5766,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 18 unmodified lines
5228	5766	insn->src_reg);
5229	5767	return -EACCES;
5230	5768	}
	5769	+ src_reg = &regs[insn->src_reg];
5231	5770	} else {
5232	5771	if (insn->src_reg != BPF_REG_0) {
5233	5772	verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
@@ -5243,20 +5782,29 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, · 9 unmodified lines
5243	5782	dst_reg = &regs[insn->dst_reg];
5244	5783	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
5245	5784
5246		− if (BPF_SRC(insn->code) == BPF_K) {
5247		− int pred = is_branch_taken(dst_reg, insn->imm, opcode,
5248		− is_jmp32);
5249		−
5250		− if (pred == 1) {
5251		− /* only follow the goto, ignore fall-through */
5252		− *insn_idx += insn->off;
5253		− return 0;
5254		− } else if (pred == 0) {
5255		− /* only follow fall-through branch, since
5256		− * that's where the program will go
5257		− */
5258		− return 0;
5259		− }
	5785	+ if (BPF_SRC(insn->code) == BPF_K)
	5786	+ pred = is_branch_taken(dst_reg, insn->imm,
	5787	+ opcode, is_jmp32);
	5788	+ else if (src_reg->type == SCALAR_VALUE &&
	5789	+ tnum_is_const(src_reg->var_off))
	5790	+ pred = is_branch_taken(dst_reg, src_reg->var_off.value,
	5791	+ opcode, is_jmp32);
	5792	+ if (pred >= 0) {
	5793	+ err = mark_chain_precision(env, insn->dst_reg);
	5794	+ if (BPF_SRC(insn->code) == BPF_X && !err)
	5795	+ err = mark_chain_precision(env, insn->src_reg);
	5796	+ if (err)
	5797	+ return err;
	5798	+ }
	5799	+ if (pred == 1) {
	5800	+ /* only follow the goto, ignore fall-through */
	5801	+ *insn_idx += insn->off;
	5802	+ return 0;
	5803	+ } else if (pred == 0) {
	5804	+ /* only follow fall-through branch, since
	5805	+ * that's where the program will go
	5806	+ */
	5807	+ return 0;
5260	5808	}
5261	5809
5262	5810	other_branch = push_stack(env, insn_idx + insn->off + 1, insn_idx,
@@ -5616,7 +6164,8 @@ static void init_explored_state(struct bpf_verifier_env *env, int idx) · 353 unmodified lines
5616	6164	* w - next instruction
5617	6165	* e - edge
5618	6166	*/
5619		−static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
	6167	+static int push_insn(int t, int w, int e, struct bpf_verifier_env *env,
	6168	+ bool loop_ok)
5620	6169	{
5621	6170	int *insn_stack = env->cfg.insn_stack;
5622	6171	int *insn_state = env->cfg.insn_state;
@@ -5646,6 +6195,8 @@ static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) · 23 unmodified lines
5646	6195	insn_stack[env->cfg.cur_stack++] = w;
5647	6196	return 1;
5648	6197	} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
	6198	+ if (loop_ok && env->allow_ptr_leaks)
	6199	+ return 0;
5649	6200	verbose_linfo(env, t, "%d: ", t);
5650	6201	verbose_linfo(env, w, "%d: ", w);
5651	6202	verbose(env, "back-edge from insn %d to %d\n", t, w);
@@ -5697,7 +6248,7 @@ peek_stack: · 45 unmodified lines
5697	6248	if (opcode == BPF_EXIT) {
5698	6249	goto mark_explored;
5699	6250	} else if (opcode == BPF_CALL) {
5700		− ret = push_insn(t, t + 1, FALLTHROUGH, env);
	6251	+ ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
5701	6252	if (ret == 1)
5702	6253	goto peek_stack;
5703	6254	else if (ret < 0)
@@ -5706,7 +6257,8 @@ peek_stack: · 2 unmodified lines
5706	6257	init_explored_state(env, t + 1);
5707	6258	if (insns[t].src_reg == BPF_PSEUDO_CALL) {
5708	6259	init_explored_state(env, t);
5709		− ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
	6260	+ ret = push_insn(t, t + insns[t].imm + 1, BRANCH,
	6261	+ env, false);
5710	6262	if (ret == 1)
5711	6263	goto peek_stack;
5712	6264	else if (ret < 0)
@@ -5719,11 +6271,16 @@ peek_stack: · 6 unmodified lines
5719	6271	}
5720	6272	/* unconditional jump with single edge */
5721	6273	ret = push_insn(t, t + insns[t].off + 1,
5722		− FALLTHROUGH, env);
	6274	+ FALLTHROUGH, env, true);
5723	6275	if (ret == 1)
5724	6276	goto peek_stack;
5725	6277	else if (ret < 0)
5726	6278	goto err_free;
	6279	+ /* unconditional jmp is not a good pruning point,
	6280	+ * but it's marked, since backtracking needs
	6281	+ * to record jmp history in is_state_visited().
	6282	+ */
	6283	+ init_explored_state(env, t + insns[t].off + 1);
5727	6284	/* tell verifier to check for equivalent states
5728	6285	* after every call and jump
5729	6286	*/
@@ -5732,13 +6289,13 @@ peek_stack: · 2 unmodified lines
5732	6289	} else {
5733	6290	/* conditional jump with two edges */
5734	6291	init_explored_state(env, t);
5735		− ret = push_insn(t, t + 1, FALLTHROUGH, env);
	6292	+ ret = push_insn(t, t + 1, FALLTHROUGH, env, true);
5736	6293	if (ret == 1)
5737	6294	goto peek_stack;
5738	6295	else if (ret < 0)
5739	6296	goto err_free;
5740	6297
5741		− ret = push_insn(t, t + insns[t].off + 1, BRANCH, env);
	6298	+ ret = push_insn(t, t + insns[t].off + 1, BRANCH, env, true);
5742	6299	if (ret == 1)
5743	6300	goto peek_stack;
5744	6301	else if (ret < 0)
@@ -5748,7 +6305,7 @@ peek_stack: · 3 unmodified lines
5748	6305	/* all other non-branch instructions with single
5749	6306	* fall-through edge
5750	6307	*/
5751		− ret = push_insn(t, t + 1, FALLTHROUGH, env);
	6308	+ ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
5752	6309	if (ret == 1)
5753	6310	goto peek_stack;
5754	6311	else if (ret < 0)
@@ -6181,6 +6738,8 @@ static void clean_live_states(struct bpf_verifier_env *env, int insn, · 426 unmodified lines
6181	6738
6182	6739	sl = *explored_state(env, insn);
6183	6740	while (sl) {
	6741	+ if (sl->state.branches)
	6742	+ goto next;
6184	6743	if (sl->state.insn_idx != insn \|\|
6185	6744	sl->state.curframe != cur->curframe)
6186	6745	goto next;
@@ -6222,6 +6781,8 @@ static bool regsafe(struct bpf_reg_state rold, struct bpf_reg_state rcur, · 35 unmodified lines
6222	6781	switch (rold->type) {
6223	6782	case SCALAR_VALUE:
6224	6783	if (rcur->type == SCALAR_VALUE) {
	6784	+ if (!rold->precise && !rcur->precise)
	6785	+ return true;
6225	6786	/* new val must satisfy old val knowledge */
6226	6787	return range_within(rold, rcur) &&
6227	6788	tnum_in(rold->var_off, rcur->var_off);
@@ -6294,6 +6855,7 @@ static bool regsafe(struct bpf_reg_state rold, struct bpf_reg_state rcur, · 66 unmodified lines
6294	6855	case PTR_TO_SOCK_COMMON_OR_NULL:
6295	6856	case PTR_TO_TCP_SOCK:
6296	6857	case PTR_TO_TCP_SOCK_OR_NULL:
	6858	+ case PTR_TO_XDP_SOCK:
6297	6859	/* Only valid matches are exact, which memcmp() above
6298	6860	* would have accepted
6299	6861	*/
@@ -6544,19 +7106,52 @@ static int propagate_liveness(struct bpf_verifier_env *env, · 244 unmodified lines
6544	7106	return 0;
6545	7107	}
6546	7108
	7109	+static bool states_maybe_looping(struct bpf_verifier_state *old,
	7110	+ struct bpf_verifier_state *cur)
	7111	+{
	7112	+ struct bpf_func_state fold, fcur;
	7113	+ int i, fr = cur->curframe;
	7114	+
	7115	+ if (old->curframe != fr)
	7116	+ return false;
	7117	+
	7118	+ fold = old->frame[fr];
	7119	+ fcur = cur->frame[fr];
	7120	+ for (i = 0; i < MAX_BPF_REG; i++)
	7121	+ if (memcmp(&fold->regs[i], &fcur->regs[i],
	7122	+ offsetof(struct bpf_reg_state, parent)))
	7123	+ return false;
	7124	+ return true;
	7125	+}
	7126	+
	7127	+
6547	7128	static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
6548	7129	{
6549	7130	struct bpf_verifier_state_list *new_sl;
6550	7131	struct bpf_verifier_state_list sl, *pprev;
6551	7132	struct bpf_verifier_state cur = env->cur_state, new;
6552	7133	int i, j, err, states_cnt = 0;
	7134	+ bool add_new_state = false;
6553	7135
	7136	+ cur->last_insn_idx = env->prev_insn_idx;
6554	7137	if (!env->insn_aux_data[insn_idx].prune_point)
6555	7138	/* this 'insn_idx' instruction wasn't marked, so we will not
6556	7139	* be doing state search here
6557	7140	*/
6558	7141	return 0;
6559	7142
	7143	+ /* bpf progs typically have pruning point every 4 instructions
	7144	+ * http://vger.kernel.org/bpfconf2019.html#session-1
	7145	+ * Do not add new state for future pruning if the verifier hasn't seen
	7146	+ * at least 2 jumps and at least 8 instructions.
	7147	+ * This heuristics helps decrease 'total_states' and 'peak_states' metric.
	7148	+ * In tests that amounts to up to 50% reduction into total verifier
	7149	+ * memory consumption and 20% verifier time speedup.
	7150	+ */
	7151	+ if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
	7152	+ env->insn_processed - env->prev_insn_processed >= 8)
	7153	+ add_new_state = true;
	7154	+
6560	7155	pprev = explored_state(env, insn_idx);
6561	7156	sl = *pprev;
6562	7157
@@ -6566,6 +7161,30 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 3 unmodified lines
6566	7161	states_cnt++;
6567	7162	if (sl->state.insn_idx != insn_idx)
6568	7163	goto next;
	7164	+ if (sl->state.branches) {
	7165	+ if (states_maybe_looping(&sl->state, cur) &&
	7166	+ states_equal(env, &sl->state, cur)) {
	7167	+ verbose_linfo(env, insn_idx, "; ");
	7168	+ verbose(env, "infinite loop detected at insn %d\n", insn_idx);
	7169	+ return -EINVAL;
	7170	+ }
	7171	+ /* if the verifier is processing a loop, avoid adding new state
	7172	+ * too often, since different loop iterations have distinct
	7173	+ * states and may not help future pruning.
	7174	+ * This threshold shouldn't be too low to make sure that
	7175	+ * a loop with large bound will be rejected quickly.
	7176	+ * The most abusive loop will be:
	7177	+ * r1 += 1
	7178	+ * if r1 < 1000000 goto pc-2
	7179	+ * 1M insn_procssed limit / 100 == 10k peak states.
	7180	+ * This threshold shouldn't be too high either, since states
	7181	+ * at the end of the loop are likely to be useful in pruning.
	7182	+ */
	7183	+ if (env->jmps_processed - env->prev_jmps_processed < 20 &&
	7184	+ env->insn_processed - env->prev_insn_processed < 100)
	7185	+ add_new_state = false;
	7186	+ goto miss;
	7187	+ }
6569	7188	if (states_equal(env, &sl->state, cur)) {
6570	7189	sl->hit_cnt++;
6571	7190	/* reached equivalent register/stack state,
@@ -6583,7 +7202,15 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 11 unmodified lines
6583	7202	return err;
6584	7203	return 1;
6585	7204	}
6586		− sl->miss_cnt++;
	7205	+miss:
	7206	+ /* when new state is not going to be added do not increase miss count.
	7207	+ * Otherwise several loop iterations will remove the state
	7208	+ * recorded earlier. The goal of these heuristics is to have
	7209	+ * states from some iterations of the loop (some in the beginning
	7210	+ * and some at the end) to help pruning.
	7211	+ */
	7212	+ if (add_new_state)
	7213	+ sl->miss_cnt++;
6587	7214	/* heuristic to determine whether this state is beneficial
6588	7215	* to keep checking from state equivalence point of view.
6589	7216	* Higher numbers increase max_states_per_insn and verification time,
@@ -6595,6 +7222,11 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) · 5 unmodified lines
6595	7222	*/
6596	7223	*pprev = sl->next;
6597	7224	if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) {
	7225	+ u32 br = sl->state.branches;
	7226	+
	7227	+ WARN_ONCE(br,
	7228	+ "BUG live_done but branches_to_explore %d\n",
	7229	+ br);
6598	7230	free_verifier_state(&sl->state, false);
6599	7231	kfree(sl);
6600	7232	env->peak_states--;
@@ -6618,20 +7250,27 @@ next: · 17 unmodified lines
6618	7250	env->max_states_per_insn = states_cnt;
6619	7251
6620	7252	if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
6621		− return 0;
	7253	+ return push_jmp_history(env, cur);
	7254	+
	7255	+ if (!add_new_state)
	7256	+ return push_jmp_history(env, cur);
6622	7257
6623		− /* there were no equivalent states, remember current one.
6624		− * technically the current state is not proven to be safe yet,
	7258	+ /* There were no equivalent states, remember the current one.
	7259	+ * Technically the current state is not proven to be safe yet,
6625	7260	* but it will either reach outer most bpf_exit (which means it's safe)
6626		− * or it will be rejected. Since there are no loops, we won't be
	7261	+ * or it will be rejected. When there are no loops the verifier won't be
6627	7262	* seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
6628		− * again on the way to bpf_exit
	7263	+ * again on the way to bpf_exit.
	7264	+ * When looping the sl->state.branches will be > 0 and this state
	7265	+ * will not be considered for equivalence until branches == 0.
6629	7266	*/
6630	7267	new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
6631	7268	if (!new_sl)
6632	7269	return -ENOMEM;
6633	7270	env->total_states++;
6634	7271	env->peak_states++;
	7272	+ env->prev_jmps_processed = env->jmps_processed;
	7273	+ env->prev_insn_processed = env->insn_processed;
6635	7274
6636	7275	/* add new state to the head of linked list */
6637	7276	new = &new_sl->state;
@@ -6642,6 +7281,12 @@ next: · 4 unmodified lines
6642	7281	return err;
6643	7282	}
6644	7283	new->insn_idx = insn_idx;
	7284	+ WARN_ONCE(new->branches != 1,
	7285	+ "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
	7286	+
	7287	+ cur->parent = new;
	7288	+ cur->first_insn_idx = insn_idx;
	7289	+ clear_jmp_history(cur);
6645	7290	new_sl->next = *explored_state(env, insn_idx);
6646	7291	*explored_state(env, insn_idx) = new_sl;
6647	7292	/* connect new state to parentage chain. Current frame needs all
@@ -6651,17 +7296,18 @@ next: · 3 unmodified lines
6651	7296	* the state of the call instruction (with WRITTEN set), and r0 comes
6652	7297	* from callee with its full parentage chain, anyway.
6653	7298	*/
6654		− for (j = 0; j <= cur->curframe; j++)
6655		− for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
6656		− cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
6657	7299	/* clear write marks in current state: the writes we did are not writes
6658	7300	* our child did, so they don't screen off its reads from us.
6659	7301	* (There are no read marks in current state, because reads always mark
6660	7302	* their parent and current state never has children yet. Only
6661	7303	* explored_states can get read marks.)
6662	7304	*/
6663		− for (i = 0; i < BPF_REG_FP; i++)
6664		− cur->frame[cur->curframe]->regs[i].live = REG_LIVE_NONE;
	7305	+ for (j = 0; j <= cur->curframe; j++) {
	7306	+ for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
	7307	+ cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
	7308	+ for (i = 0; i < BPF_REG_FP; i++)
	7309	+ cur->frame[j]->regs[i].live = REG_LIVE_NONE;
	7310	+ }
6665	7311
6666	7312	/* all stack frames are accessible from callee, clear them all */
6667	7313	for (j = 0; j <= cur->curframe; j++) {
@@ -6688,6 +7334,7 @@ static bool reg_type_mismatch_ok(enum bpf_reg_type type) · 20 unmodified lines
6688	7334	case PTR_TO_SOCK_COMMON_OR_NULL:
6689	7335	case PTR_TO_TCP_SOCK:
6690	7336	case PTR_TO_TCP_SOCK_OR_NULL:
	7337	+ case PTR_TO_XDP_SOCK:
6691	7338	return false;
6692	7339	default:
6693	7340	return true;
@@ -6719,6 +7366,7 @@ static int do_check(struct bpf_verifier_env *env) · 25 unmodified lines
6719	7366	struct bpf_reg_state *regs;
6720	7367	int insn_cnt = env->prog->len;
6721	7368	bool do_print_state = false;
	7369	+ int prev_insn_idx = -1;
6722	7370
6723	7371	env->prev_linfo = NULL;
6724	7372
@@ -6727,6 +7375,7 @@ static int do_check(struct bpf_verifier_env *env) · 2 unmodified lines
6727	7375	return -ENOMEM;
6728	7376	state->curframe = 0;
6729	7377	state->speculative = false;
	7378	+ state->branches = 1;
6730	7379	state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
6731	7380	if (!state->frame[0]) {
6732	7381	kfree(state);
@@ -6743,6 +7392,7 @@ static int do_check(struct bpf_verifier_env *env) · 10 unmodified lines
6743	7392	u8 class;
6744	7393	int err;
6745	7394
	7395	+ env->prev_insn_idx = prev_insn_idx;
6746	7396	if (env->insn_idx >= insn_cnt) {
6747	7397	verbose(env, "invalid insn idx %d insn_cnt %d\n",
6748	7398	env->insn_idx, insn_cnt);
@@ -6815,6 +7465,7 @@ static int do_check(struct bpf_verifier_env *env) · 66 unmodified lines
6815	7465
6816	7466	regs = cur_regs(env);
6817	7467	env->insn_aux_data[env->insn_idx].seen = true;
	7468	+ prev_insn_idx = env->insn_idx;
6818	7469
6819	7470	if (class == BPF_ALU \|\| class == BPF_ALU64) {
6820	7471	err = check_alu_op(env, insn);
@@ -6933,6 +7584,7 @@ static int do_check(struct bpf_verifier_env *env) · 112 unmodified lines
6933	7584	} else if (class == BPF_JMP \|\| class == BPF_JMP32) {
6934	7585	u8 opcode = BPF_OP(insn->code);
6935	7586
	7587	+ env->jmps_processed++;
6936	7588	if (opcode == BPF_CALL) {
6937	7589	if (BPF_SRC(insn->code) != BPF_K \|\|
6938	7590	insn->off != 0 \|\|
@@ -6987,7 +7639,6 @@ static int do_check(struct bpf_verifier_env *env) · 48 unmodified lines
6987	7639
6988	7640	if (state->curframe) {
6989	7641	/* exit from nested function */
6990		− env->prev_insn_idx = env->insn_idx;
6991	7642	err = prepare_func_exit(env, &env->insn_idx);
6992	7643	if (err)
6993	7644	return err;
@@ -7018,7 +7669,8 @@ static int do_check(struct bpf_verifier_env *env) · 24 unmodified lines
7018	7669	if (err)
7019	7670	return err;
7020	7671	process_bpf_exit:
7021		− err = pop_stack(env, &env->prev_insn_idx,
	7672	+ update_branch_counts(env, env->cur_state);
	7673	+ err = pop_stack(env, &prev_insn_idx,
7022	7674	&env->insn_idx);
7023	7675	if (err < 0) {
7024	7676	if (err != -ENOENT)
@@ -7821,6 +8473,9 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 796 unmodified lines
7821	8473	case PTR_TO_TCP_SOCK:
7822	8474	convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
7823	8475	break;
	8476	+ case PTR_TO_XDP_SOCK:
	8477	+ convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
	8478	+ break;
7824	8479	default:
7825	8480	continue;
7826	8481	}

kernel/bpf/xskmap.c+8−1

@@ -17,8 +17,8 @@ struct xsk_map { · 16 unmodified lines
17	17
18	18	static struct bpf_map xsk_map_alloc(union bpf_attr attr)
19	19	{
20		− int cpu, err = -EINVAL;
21	20	struct xsk_map *m;
	21	+ int cpu, err;
22	22	u64 cost;
23	23
24	24	if (!capable(CAP_NET_ADMIN))
@@ -151,6 +151,12 @@ void __xsk_map_flush(struct bpf_map *map) · 126 unmodified lines
151	151	}
152	152
153	153	static void xsk_map_lookup_elem(struct bpf_map map, void *key)
	154	+{
	155	+ WARN_ON_ONCE(!rcu_read_lock_held());
	156	+ return __xsk_map_lookup_elem(map, (u32 )key);
	157	+}
	158	+
	159	+static void xsk_map_lookup_elem_sys_only(struct bpf_map map, void *key)
154	160	{
155	161	return ERR_PTR(-EOPNOTSUPP);
156	162	}
@@ -218,6 +224,7 @@ const struct bpf_map_ops xsk_map_ops = { · 61 unmodified lines
218	224	.map_free = xsk_map_free,
219	225	.map_get_next_key = xsk_map_get_next_key,
220	226	.map_lookup_elem = xsk_map_lookup_elem,
	227	+ .map_lookup_elem_sys_only = xsk_map_lookup_elem_sys_only,
221	228	.map_update_elem = xsk_map_update_elem,
222	229	.map_delete_elem = xsk_map_delete_elem,
223	230	.map_check_btf = map_check_no_btf,

net/core/filter.c+86−0

@@ -5695,6 +5695,46 @@ BPF_CALL_1(bpf_skb_ecn_set_ce, struct sk_buff *, skb) · 5694 unmodified lines
5695	5695	return INET_ECN_set_ce(skb);
5696	5696	}
5697	5697
	5698	+bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type,
	5699	+ struct bpf_insn_access_aux *info)
	5700	+{
	5701	+ if (off < 0 \|\| off >= offsetofend(struct bpf_xdp_sock, queue_id))
	5702	+ return false;
	5703	+
	5704	+ if (off % size != 0)
	5705	+ return false;
	5706	+
	5707	+ switch (off) {
	5708	+ default:
	5709	+ return size == sizeof(__u32);
	5710	+ }
	5711	+}
	5712	+
	5713	+u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type,
	5714	+ const struct bpf_insn *si,
	5715	+ struct bpf_insn *insn_buf,
	5716	+ struct bpf_prog prog, u32 target_size)
	5717	+{
	5718	+ struct bpf_insn *insn = insn_buf;
	5719	+
	5720	+#define BPF_XDP_SOCK_GET(FIELD) \
	5721	+ do { \
	5722	+ BUILD_BUG_ON(FIELD_SIZEOF(struct xdp_sock, FIELD) > \
	5723	+ FIELD_SIZEOF(struct bpf_xdp_sock, FIELD)); \
	5724	+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct xdp_sock, FIELD),\
	5725	+ si->dst_reg, si->src_reg, \
	5726	+ offsetof(struct xdp_sock, FIELD)); \
	5727	+ } while (0)
	5728	+
	5729	+ switch (si->off) {
	5730	+ case offsetof(struct bpf_xdp_sock, queue_id):
	5731	+ BPF_XDP_SOCK_GET(queue_id);
	5732	+ break;
	5733	+ }
	5734	+
	5735	+ return insn - insn_buf;
	5736	+}
	5737	+
5698	5738	static const struct bpf_func_proto bpf_skb_ecn_set_ce_proto = {
5699	5739	.func = bpf_skb_ecn_set_ce,
5700	5740	.gpl_only = false,
@@ -5897,6 +5937,10 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 196 unmodified lines
5897	5937	case BPF_FUNC_skc_lookup_tcp:
5898	5938	return &bpf_sock_addr_skc_lookup_tcp_proto;
5899	5939	#endif /* CONFIG_INET */
	5940	+ case BPF_FUNC_sk_storage_get:
	5941	+ return &bpf_sk_storage_get_proto;
	5942	+ case BPF_FUNC_sk_storage_delete:
	5943	+ return &bpf_sk_storage_delete_proto;
5900	5944	default:
5901	5945	return bpf_base_func_proto(func_id);
5902	5946	}
@@ -5934,6 +5978,10 @@ cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 31 unmodified lines
5934	5978	return &bpf_sk_storage_get_proto;
5935	5979	case BPF_FUNC_sk_storage_delete:
5936	5980	return &bpf_sk_storage_delete_proto;
	5981	+#ifdef CONFIG_SOCK_CGROUP_DATA
	5982	+ case BPF_FUNC_skb_cgroup_id:
	5983	+ return &bpf_skb_cgroup_id_proto;
	5984	+#endif
5937	5985	#ifdef CONFIG_INET
5938	5986	case BPF_FUNC_tcp_sock:
5939	5987	return &bpf_tcp_sock_proto;
@@ -6114,6 +6162,14 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 174 unmodified lines
6114	6162	return &bpf_get_local_storage_proto;
6115	6163	case BPF_FUNC_perf_event_output:
6116	6164	return &bpf_sockopt_event_output_proto;
	6165	+ case BPF_FUNC_sk_storage_get:
	6166	+ return &bpf_sk_storage_get_proto;
	6167	+ case BPF_FUNC_sk_storage_delete:
	6168	+ return &bpf_sk_storage_delete_proto;
	6169	+#ifdef CONFIG_INET
	6170	+ case BPF_FUNC_tcp_sock:
	6171	+ return &bpf_tcp_sock_proto;
	6172	+#endif /* CONFIG_INET */
6117	6173	default:
6118	6174	return bpf_base_func_proto(func_id);
6119	6175	}
@@ -6801,6 +6857,13 @@ static bool sock_addr_is_valid_access(int off, int size, · 681 unmodified lines
6801	6857	if (size != size_default)
6802	6858	return false;
6803	6859	break;
	6860	+ case offsetof(struct bpf_sock_addr, sk):
	6861	+ if (type != BPF_READ)
	6862	+ return false;
	6863	+ if (size != sizeof(__u64))
	6864	+ return false;
	6865	+ info->reg_type = PTR_TO_SOCKET;
	6866	+ break;
6804	6867	default:
6805	6868	if (type == BPF_READ) {
6806	6869	if (size != size_default)
@@ -6844,6 +6907,11 @@ static bool sock_ops_is_valid_access(int off, int size, · 37 unmodified lines
6844	6907	if (size != sizeof(__u64))
6845	6908	return false;
6846	6909	break;
	6910	+ case offsetof(struct bpf_sock_ops, sk):
	6911	+ if (size != sizeof(__u64))
	6912	+ return false;
	6913	+ info->reg_type = PTR_TO_SOCKET_OR_NULL;
	6914	+ break;
6847	6915	default:
6848	6916	if (size != size_default)
6849	6917	return false;
@@ -7751,6 +7819,11 @@ static u32 sock_addr_convert_ctx_access(enum bpf_access_type type, · 901 unmodified lines
7751	7819	struct bpf_sock_addr_kern, struct in6_addr, t_ctx,
7752	7820	s6_addr32[0], BPF_SIZE(si->code), off, tmp_reg);
7753	7821	break;
	7822	+ case offsetof(struct bpf_sock_addr, sk):
	7823	+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_sock_addr_kern, sk),
	7824	+ si->dst_reg, si->src_reg,
	7825	+ offsetof(struct bpf_sock_addr_kern, sk));
	7826	+ break;
7754	7827	}
7755	7828
7756	7829	return insn - insn_buf;
@@ -8010,6 +8083,19 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, · 253 unmodified lines
8010	8083	SOCK_OPS_GET_OR_SET_FIELD(sk_txhash, sk_txhash,
8011	8084	struct sock, type);
8012	8085	break;
	8086	+ case offsetof(struct bpf_sock_ops, sk):
	8087	+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
	8088	+ struct bpf_sock_ops_kern,
	8089	+ is_fullsock),
	8090	+ si->dst_reg, si->src_reg,
	8091	+ offsetof(struct bpf_sock_ops_kern,
	8092	+ is_fullsock));
	8093	+ *insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 1);
	8094	+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(
	8095	+ struct bpf_sock_ops_kern, sk),
	8096	+ si->dst_reg, si->src_reg,
	8097	+ offsetof(struct bpf_sock_ops_kern, sk));
	8098	+ break;
8013	8099	}
8014	8100	return insn - insn_buf;
8015	8101	}

net/core/sock.c+4−0

@@ -1039,6 +1039,10 @@ set_rcvbuf: · 1038 unmodified lines
1039	1039	}
1040	1040	break;
1041	1041
	1042	+ case SO_DETACH_REUSEPORT_BPF:
	1043	+ ret = reuseport_detach_prog(sk);
	1044	+ break;
	1045	+
1042	1046	case SO_DETACH_FILTER:
1043	1047	ret = sk_detach_filter(sk);
1044	1048	break;

net/core/sock_reuseport.c+24−0

@@ -332,3 +332,27 @@ int reuseport_attach_prog(struct sock sk, struct bpf_prog prog) · 331 unmodified lines
332	332	return 0;
333	333	}
334	334	EXPORT_SYMBOL(reuseport_attach_prog);
	335	+
	336	+int reuseport_detach_prog(struct sock *sk)
	337	+{
	338	+ struct sock_reuseport *reuse;
	339	+ struct bpf_prog *old_prog;
	340	+
	341	+ if (!rcu_access_pointer(sk->sk_reuseport_cb))
	342	+ return sk->sk_reuseport ? -ENOENT : -EINVAL;
	343	+
	344	+ old_prog = NULL;
	345	+ spin_lock_bh(&reuseport_lock);
	346	+ reuse = rcu_dereference_protected(sk->sk_reuseport_cb,
	347	+ lockdep_is_held(&reuseport_lock));
	348	+ rcu_swap_protected(reuse->prog, old_prog,
	349	+ lockdep_is_held(&reuseport_lock));
	350	+ spin_unlock_bh(&reuseport_lock);
	351	+
	352	+ if (!old_prog)
	353	+ return -ENOENT;
	354	+
	355	+ sk_reuseport_prog_free(old_prog);
	356	+ return 0;
	357	+}
	358	+EXPORT_SYMBOL(reuseport_detach_prog);

samples/bpf/Makefile+13−10

@@ -170,21 +170,12 @@ always += ibumad_kern.o · 169 unmodified lines
170	170	always += hbm_out_kern.o
171	171
172	172	KBUILD_HOSTCFLAGS += -I$(objtree)/usr/include
173		−KBUILD_HOSTCFLAGS += -I$(srctree)/tools/lib/
	173	+KBUILD_HOSTCFLAGS += -I$(srctree)/tools/lib/bpf/
174	174	KBUILD_HOSTCFLAGS += -I$(srctree)/tools/testing/selftests/bpf/
175	175	KBUILD_HOSTCFLAGS += -I$(srctree)/tools/lib/ -I$(srctree)/tools/include
176	176	KBUILD_HOSTCFLAGS += -I$(srctree)/tools/perf
177	177
178	178	HOSTCFLAGS_bpf_load.o += -I$(objtree)/usr/include -Wno-unused-variable
179		−HOSTCFLAGS_trace_helpers.o += -I$(srctree)/tools/lib/bpf/
180		−
181		−HOSTCFLAGS_trace_output_user.o += -I$(srctree)/tools/lib/bpf/
182		−HOSTCFLAGS_offwaketime_user.o += -I$(srctree)/tools/lib/bpf/
183		−HOSTCFLAGS_spintest_user.o += -I$(srctree)/tools/lib/bpf/
184		−HOSTCFLAGS_trace_event_user.o += -I$(srctree)/tools/lib/bpf/
185		−HOSTCFLAGS_sampleip_user.o += -I$(srctree)/tools/lib/bpf/
186		−HOSTCFLAGS_task_fd_query_user.o += -I$(srctree)/tools/lib/bpf/
187		−HOSTCFLAGS_xdp_sample_pkts_user.o += -I$(srctree)/tools/lib/bpf/
188	179
189	180	KBUILD_HOSTLDLIBS += $(LIBBPF) -lelf
190	181	HOSTLDLIBS_tracex4 += -lrt
@@ -206,6 +197,17 @@ HOSTCC = $(CROSS_COMPILE)gcc · 15 unmodified lines
206	197	CLANG_ARCH_ARGS = -target $(ARCH)
207	198	endif
208	199
	200	+# Don't evaluate probes and warnings if we need to run make recursively
	201	+ifneq ($(src),)
	202	+HDR_PROBE := $(shell echo "\#include <linux/types.h>\n struct list_head { int a; }; int main() { return 0; }" \| \
	203	+ $(HOSTCC) $(KBUILD_HOSTCFLAGS) -x c - -o /dev/null 2>/dev/null && \
	204	+ echo okay)
	205	+
	206	+ifeq ($(HDR_PROBE),)
	207	+$(warning WARNING: Detected possible issues with include path.)
	208	+$(warning WARNING: Please install kernel headers locally (make headers_install).)
	209	+endif
	210	+
209	211	BTF_LLC_PROBE := $(shell $(LLC) -march=bpf -mattr=help 2>&1 \| grep dwarfris)
210	212	BTF_PAHOLE_PROBE := $(shell $(BTF_PAHOLE) --help 2>&1 \| grep BTF)
211	213	BTF_OBJCOPY_PROBE := $(shell $(LLVM_OBJCOPY) --help 2>&1 \| grep -i 'usage.*llvm')
@@ -223,6 +225,7 @@ ifneq ($(and $(BTF_LLC_PROBE),$(BTF_PAHOLE_PROBE),$(BTF_OBJCOPY_PROBE)),) · 11 unmodified lines
223	225	DWARF2BTF = y
224	226	endif
225	227	endif
	228	+endif
226	229
227	230	# Trick to allow make to be run from this directory
228	231	all:

samples/bpf/fds_example.c+1−1

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14
15	15	#include <bpf/bpf.h>
16	16
17		−#include "bpf/libbpf.h"
	17	+#include "libbpf.h"
18	18	#include "bpf_insn.h"
19	19	#include "sock_example.h"
20	20

samples/bpf/hbm.c+3−3

@@ -50,8 +50,8 @@ · 49 unmodified lines
50	50	#include "cgroup_helpers.h"
51	51	#include "hbm.h"
52	52	#include "bpf_util.h"
53		−#include "bpf/bpf.h"
54		−#include "bpf/libbpf.h"
	53	+#include "bpf.h"
	54	+#include "libbpf.h"
55	55
56	56	bool outFlag = true;
57	57	int minRate = 1000; /* cgroup rate limit in Mbps */
@@ -411,7 +411,7 @@ static void Usage(void) · 353 unmodified lines
411	411	" -l also limit flows using loopback\n"
412	412	" -n <#> to create cgroup \"/hbm#\" and attach prog\n"
413	413	" Default is /hbm1\n"
414		− " --no_cn disable CN notifcations\n"
	414	+ " --no_cn disable CN notifications\n"
415	415	" -r <rate> Rate in Mbps\n"
416	416	" -s Update HBM stats\n"
417	417	" -t <time> Exit after specified seconds (default is 0)\n"

samples/bpf/ibumad_user.c+1−1

@@ -25,7 +25,7 @@ · 24 unmodified lines
25	25
26	26	#include "bpf_load.h"
27	27	#include "bpf_util.h"
28		−#include "bpf/libbpf.h"
	28	+#include "libbpf.h"
29	29
30	30	static void dump_counts(int fd)
31	31	{

samples/bpf/sockex1_user.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3	3	#include <assert.h>
4	4	#include <linux/bpf.h>
5	5	#include <bpf/bpf.h>
6		−#include "bpf/libbpf.h"
	6	+#include "libbpf.h"
7	7	#include "sock_example.h"
8	8	#include <unistd.h>
9	9	#include <arpa/inet.h>

samples/bpf/sockex2_user.c+1−1

@@ -3,7 +3,7 @@ · 2 unmodified lines
3	3	#include <assert.h>
4	4	#include <linux/bpf.h>
5	5	#include <bpf/bpf.h>
6		−#include "bpf/libbpf.h"
	6	+#include "libbpf.h"
7	7	#include "sock_example.h"
8	8	#include <unistd.h>
9	9	#include <arpa/inet.h>

samples/bpf/xdp1_user.c+2−2

@@ -15,8 +15,8 @@ · 14 unmodified lines
15	15	#include <net/if.h>
16	16
17	17	#include "bpf_util.h"
18		−#include "bpf/bpf.h"
19		−#include "bpf/libbpf.h"
	18	+#include "bpf.h"
	19	+#include "libbpf.h"
20	20
21	21	static int ifindex;
22	22	static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;

samples/bpf/xdp_adjust_tail_user.c+2−2

@@ -18,8 +18,8 @@ · 17 unmodified lines
18	18	#include <netinet/ether.h>
19	19	#include <unistd.h>
20	20	#include <time.h>
21		−#include "bpf/bpf.h"
22		−#include "bpf/libbpf.h"
	21	+#include "bpf.h"
	22	+#include "libbpf.h"
23	23
24	24	#define STATS_INTERVAL_S 2U
25	25

samples/bpf/xdp_fwd_user.c+1−1

@@ -24,7 +24,7 @@ · 23 unmodified lines
24	24	#include <fcntl.h>
25	25	#include <libgen.h>
26	26
27		−#include "bpf/libbpf.h"
	27	+#include "libbpf.h"
28	28	#include <bpf/bpf.h>
29	29
30	30

samples/bpf/xdp_redirect_cpu_user.c+1−1

@@ -25,7 +25,7 @@ static const char *__doc__ = · 24 unmodified lines
25	25	#define MAX_PROG 6
26	26
27	27	#include <bpf/bpf.h>
28		−#include "bpf/libbpf.h"
	28	+#include "libbpf.h"
29	29
30	30	#include "bpf_util.h"
31	31

samples/bpf/xdp_redirect_map_user.c+1−1

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	16
17	17	#include "bpf_util.h"
18	18	#include <bpf/bpf.h>
19		−#include "bpf/libbpf.h"
	19	+#include "libbpf.h"
20	20
21	21	static int ifindex_in;
22	22	static int ifindex_out;

samples/bpf/xdp_redirect_user.c+1−1

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	16
17	17	#include "bpf_util.h"
18	18	#include <bpf/bpf.h>
19		−#include "bpf/libbpf.h"
	19	+#include "libbpf.h"
20	20
21	21	static int ifindex_in;
22	22	static int ifindex_out;

samples/bpf/xdp_router_ipv4_user.c+1−1

@@ -21,7 +21,7 @@ · 20 unmodified lines
21	21	#include <sys/ioctl.h>
22	22	#include <sys/syscall.h>
23	23	#include "bpf_util.h"
24		−#include "bpf/libbpf.h"
	24	+#include "libbpf.h"
25	25	#include <sys/resource.h>
26	26	#include <libgen.h>
27	27

samples/bpf/xdp_rxq_info_user.c+2−2

@@ -22,8 +22,8 @@ static const char *__doc__ = " XDP RX-queue info extract example\n\n" · 21 unmodified lines
22	22	#include <arpa/inet.h>
23	23	#include <linux/if_link.h>
24	24
25		−#include "bpf/bpf.h"
26		−#include "bpf/libbpf.h"
	25	+#include "bpf.h"
	26	+#include "libbpf.h"
27	27	#include "bpf_util.h"
28	28
29	29	static int ifindex = -1;

samples/bpf/xdp_tx_iptunnel_user.c+1−1

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14	#include <netinet/ether.h>
15	15	#include <unistd.h>
16	16	#include <time.h>
17		−#include "bpf/libbpf.h"
	17	+#include "libbpf.h"
18	18	#include <bpf/bpf.h>
19	19	#include "bpf_util.h"
20	20	#include "xdp_tx_iptunnel_common.h"

samples/bpf/xdpsock_user.c+2−2

@@ -27,8 +27,8 @@ · 26 unmodified lines
27	27	#include <time.h>
28	28	#include <unistd.h>
29	29
30		−#include "bpf/libbpf.h"
31		−#include "bpf/xsk.h"
	30	+#include "libbpf.h"
	31	+#include "xsk.h"
32	32	#include <bpf/bpf.h>
33	33
34	34	#ifndef SOL_XDP

tools/bpf/bpftool/common.c+5−48

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#include <sys/vfs.h>
22	22
23	23	#include <bpf.h>
	24	+#include <libbpf.h> /* libbpf_num_possible_cpus */
24	25
25	26	#include "main.h"
26	27
@@ -439,57 +440,13 @@ unsigned int get_page_size(void) · 412 unmodified lines
439	440
440	441	unsigned int get_possible_cpus(void)
441	442	{
442		− static unsigned int result;
443		− char buf[128];
444		− long int n;
445		− char *ptr;
446		− int fd;
447		−
448		− if (result)
449		− return result;
450		−
451		− fd = open("/sys/devices/system/cpu/possible", O_RDONLY);
452		− if (fd < 0) {
453		− p_err("can't open sysfs possible cpus");
454		− exit(-1);
455		− }
456		−
457		− n = read(fd, buf, sizeof(buf));
458		− if (n < 2) {
459		− p_err("can't read sysfs possible cpus");
460		− exit(-1);
461		− }
462		− close(fd);
	443	+ int cpus = libbpf_num_possible_cpus();
463	444
464		− if (n == sizeof(buf)) {
465		− p_err("read sysfs possible cpus overflow");
	445	+ if (cpus < 0) {
	446	+ p_err("Can't get # of possible cpus: %s", strerror(-cpus));
466	447	exit(-1);
467	448	}
468		−
469		− ptr = buf;
470		− n = 0;
471		− while (ptr && ptr != '\n') {
472		− unsigned int a, b;
473		−
474		− if (sscanf(ptr, "%u-%u", &a, &b) == 2) {
475		− n += b - a + 1;
476		−
477		− ptr = strchr(ptr, '-') + 1;
478		− } else if (sscanf(ptr, "%u", &a) == 1) {
479		− n++;
480		− } else {
481		− assert(0);
482		− }
483		−
484		− while (isdigit(*ptr))
485		− ptr++;
486		− if (*ptr == ',')
487		− ptr++;
488		− }
489		−
490		− result = n;
491		−
492		− return result;
	449	+ return cpus;
493	450	}
494	451
495	452	static char *

tools/include/uapi/asm-generic/socket.h+147−0

@@ -0,0 +1,147 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
	2	+#ifndef __ASM_GENERIC_SOCKET_H
	3	+#define __ASM_GENERIC_SOCKET_H
	4	+
	5	+#include <linux/posix_types.h>
	6	+#include <asm/sockios.h>
	7	+
	8	+/* For setsockopt(2) */
	9	+#define SOL_SOCKET 1
	10	+
	11	+#define SO_DEBUG 1
	12	+#define SO_REUSEADDR 2
	13	+#define SO_TYPE 3
	14	+#define SO_ERROR 4
	15	+#define SO_DONTROUTE 5
	16	+#define SO_BROADCAST 6
	17	+#define SO_SNDBUF 7
	18	+#define SO_RCVBUF 8
	19	+#define SO_SNDBUFFORCE 32
	20	+#define SO_RCVBUFFORCE 33
	21	+#define SO_KEEPALIVE 9
	22	+#define SO_OOBINLINE 10
	23	+#define SO_NO_CHECK 11
	24	+#define SO_PRIORITY 12
	25	+#define SO_LINGER 13
	26	+#define SO_BSDCOMPAT 14
	27	+#define SO_REUSEPORT 15
	28	+#ifndef SO_PASSCRED /* powerpc only differs in these */
	29	+#define SO_PASSCRED 16
	30	+#define SO_PEERCRED 17
	31	+#define SO_RCVLOWAT 18
	32	+#define SO_SNDLOWAT 19
	33	+#define SO_RCVTIMEO_OLD 20
	34	+#define SO_SNDTIMEO_OLD 21
	35	+#endif
	36	+
	37	+/* Security levels - as per NRL IPv6 - don't actually do anything */
	38	+#define SO_SECURITY_AUTHENTICATION 22
	39	+#define SO_SECURITY_ENCRYPTION_TRANSPORT 23
	40	+#define SO_SECURITY_ENCRYPTION_NETWORK 24
	41	+
	42	+#define SO_BINDTODEVICE 25
	43	+
	44	+/* Socket filtering */
	45	+#define SO_ATTACH_FILTER 26
	46	+#define SO_DETACH_FILTER 27
	47	+#define SO_GET_FILTER SO_ATTACH_FILTER
	48	+
	49	+#define SO_PEERNAME 28
	50	+
	51	+#define SO_ACCEPTCONN 30
	52	+
	53	+#define SO_PEERSEC 31
	54	+#define SO_PASSSEC 34
	55	+
	56	+#define SO_MARK 36
	57	+
	58	+#define SO_PROTOCOL 38
	59	+#define SO_DOMAIN 39
	60	+
	61	+#define SO_RXQ_OVFL 40
	62	+
	63	+#define SO_WIFI_STATUS 41
	64	+#define SCM_WIFI_STATUS SO_WIFI_STATUS
	65	+#define SO_PEEK_OFF 42
	66	+
	67	+/* Instruct lower device to use last 4-bytes of skb data as FCS */
	68	+#define SO_NOFCS 43
	69	+
	70	+#define SO_LOCK_FILTER 44
	71	+
	72	+#define SO_SELECT_ERR_QUEUE 45
	73	+
	74	+#define SO_BUSY_POLL 46
	75	+
	76	+#define SO_MAX_PACING_RATE 47
	77	+
	78	+#define SO_BPF_EXTENSIONS 48
	79	+
	80	+#define SO_INCOMING_CPU 49
	81	+
	82	+#define SO_ATTACH_BPF 50
	83	+#define SO_DETACH_BPF SO_DETACH_FILTER
	84	+
	85	+#define SO_ATTACH_REUSEPORT_CBPF 51
	86	+#define SO_ATTACH_REUSEPORT_EBPF 52
	87	+
	88	+#define SO_CNX_ADVICE 53
	89	+
	90	+#define SCM_TIMESTAMPING_OPT_STATS 54
	91	+
	92	+#define SO_MEMINFO 55
	93	+
	94	+#define SO_INCOMING_NAPI_ID 56
	95	+
	96	+#define SO_COOKIE 57
	97	+
	98	+#define SCM_TIMESTAMPING_PKTINFO 58
	99	+
	100	+#define SO_PEERGROUPS 59
	101	+
	102	+#define SO_ZEROCOPY 60
	103	+
	104	+#define SO_TXTIME 61
	105	+#define SCM_TXTIME SO_TXTIME
	106	+
	107	+#define SO_BINDTOIFINDEX 62
	108	+
	109	+#define SO_TIMESTAMP_OLD 29
	110	+#define SO_TIMESTAMPNS_OLD 35
	111	+#define SO_TIMESTAMPING_OLD 37
	112	+
	113	+#define SO_TIMESTAMP_NEW 63
	114	+#define SO_TIMESTAMPNS_NEW 64
	115	+#define SO_TIMESTAMPING_NEW 65
	116	+
	117	+#define SO_RCVTIMEO_NEW 66
	118	+#define SO_SNDTIMEO_NEW 67
	119	+
	120	+#define SO_DETACH_REUSEPORT_BPF 68
	121	+
	122	+#if !defined(__KERNEL__)
	123	+
	124	+#if __BITS_PER_LONG == 64 \|\| (defined(__x86_64__) && defined(__ILP32__))
	125	+/* on 64-bit and x32, avoid the ?: operator */
	126	+#define SO_TIMESTAMP SO_TIMESTAMP_OLD
	127	+#define SO_TIMESTAMPNS SO_TIMESTAMPNS_OLD
	128	+#define SO_TIMESTAMPING SO_TIMESTAMPING_OLD
	129	+
	130	+#define SO_RCVTIMEO SO_RCVTIMEO_OLD
	131	+#define SO_SNDTIMEO SO_SNDTIMEO_OLD
	132	+#else
	133	+#define SO_TIMESTAMP (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMP_OLD : SO_TIMESTAMP_NEW)
	134	+#define SO_TIMESTAMPNS (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPNS_OLD : SO_TIMESTAMPNS_NEW)
	135	+#define SO_TIMESTAMPING (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPING_OLD : SO_TIMESTAMPING_NEW)
	136	+
	137	+#define SO_RCVTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_RCVTIMEO_OLD : SO_RCVTIMEO_NEW)
	138	+#define SO_SNDTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_SNDTIMEO_OLD : SO_SNDTIMEO_NEW)
	139	+#endif
	140	+
	141	+#define SCM_TIMESTAMP SO_TIMESTAMP
	142	+#define SCM_TIMESTAMPNS SO_TIMESTAMPNS
	143	+#define SCM_TIMESTAMPING SO_TIMESTAMPING
	144	+
	145	+#endif
	146	+
	147	+#endif /* __ASM_GENERIC_SOCKET_H */

tools/include/uapi/linux/bpf.h+6−0

@@ -3085,6 +3085,10 @@ struct bpf_sock_tuple { · 3084 unmodified lines
3085	3085	};
3086	3086	};
3087	3087
	3088	+struct bpf_xdp_sock {
	3089	+ __u32 queue_id;
	3090	+};
	3091	+
3088	3092	#define XDP_PACKET_HEADROOM 256
3089	3093
3090	3094	/* User return codes for XDP prog type.
@@ -3245,6 +3249,7 @@ struct bpf_sock_addr { · 154 unmodified lines
3245	3249	__u32 msg_src_ip6[4]; /* Allows 1,2,4-byte read an 4-byte write.
3246	3250	* Stored in network byte order.
3247	3251	*/
	3252	+ __bpf_md_ptr(struct bpf_sock *, sk);
3248	3253	};
3249	3254
3250	3255	/* User bpf_sock_ops struct to access socket values and specify request ops
@@ -3296,6 +3301,7 @@ struct bpf_sock_ops { · 45 unmodified lines
3296	3301	__u32 sk_txhash;
3297	3302	__u64 bytes_received;
3298	3303	__u64 bytes_acked;
	3304	+ __bpf_md_ptr(struct bpf_sock *, sk);
3299	3305	};
3300	3306
3301	3307	/* Definitions for bpf_sock_ops_cb_flags */

tools/lib/bpf/bpf.c+2−5

@@ -26,10 +26,11 @@ · 25 unmodified lines
26	26	#include <memory.h>
27	27	#include <unistd.h>
28	28	#include <asm/unistd.h>
	29	+#include <errno.h>
29	30	#include <linux/bpf.h>
30	31	#include "bpf.h"
31	32	#include "libbpf.h"
32		−#include <errno.h>
	33	+#include "libbpf_internal.h"
33	34
34	35	/*
35	36	* When building perf, unistd.h is overridden. __NR_bpf is
@@ -53,10 +54,6 @@ · 17 unmodified lines
53	54	# endif
54	55	#endif
55	56
56		−#ifndef min
57		−#define min(x, y) ((x) < (y) ? (x) : (y))
58		−#endif
59		−
60	57	static inline __u64 ptr_to_u64(const void *ptr)
61	58	{
62	59	return (__u64) (unsigned long) ptr;

tools/lib/bpf/bpf_prog_linfo.c+1−4

@@ -6,10 +6,7 @@ · 5 unmodified lines
6	6	#include <linux/err.h>
7	7	#include <linux/bpf.h>
8	8	#include "libbpf.h"
9		−
10		−#ifndef min
11		−#define min(x, y) ((x) < (y) ? (x) : (y))
12		−#endif
	9	+#include "libbpf_internal.h"
13	10
14	11	struct bpf_prog_linfo {
15	12	void *raw_linfo;

tools/lib/bpf/btf.c+0−3

@@ -16,9 +16,6 @@ · 15 unmodified lines
16	16	#include "libbpf_internal.h"
17	17	#include "hashmap.h"
18	18
19		−#define max(a, b) ((a) > (b) ? (a) : (b))
20		−#define min(a, b) ((a) < (b) ? (a) : (b))
21		−
22	19	#define BTF_MAX_NR_TYPES 0x7fffffff
23	20	#define BTF_MAX_STR_OFFSET 0x7fffffff
24	21

tools/lib/bpf/btf.h+1−0

@@ -17,6 +17,7 @@ extern "C" { · 16 unmodified lines
17	17
18	18	#define BTF_ELF_SEC ".BTF"
19	19	#define BTF_EXT_ELF_SEC ".BTF.ext"
	20	+#define MAPS_ELF_SEC ".maps"
20	21
21	22	struct btf;
22	23	struct btf_ext;

tools/lib/bpf/btf_dump.c+0−3

@@ -18,9 +18,6 @@ · 17 unmodified lines
18	18	#include "libbpf.h"
19	19	#include "libbpf_internal.h"
20	20
21		−#define min(x, y) ((x) < (y) ? (x) : (y))
22		−#define max(x, y) ((x) < (y) ? (y) : (x))
23		−
24	21	static const char PREFIXES[] = "\t\t\t\t\t\t\t\t\t\t\t\t\t";
25	22	static const size_t PREFIX_CNT = sizeof(PREFIXES) - 1;
26	23

tools/lib/bpf/libbpf.c+689−238

@@ -207,7 +207,8 @@ static const char * const libbpf_type_to_btf_name[] = { · 206 unmodified lines
207	207	struct bpf_map {
208	208	int fd;
209	209	char *name;
210		− size_t offset;
	210	+ int sec_idx;
	211	+ size_t sec_offset;
211	212	int map_ifindex;
212	213	int inner_map_fd;
213	214	struct bpf_map_def def;
@@ -234,6 +235,7 @@ struct bpf_object { · 20 unmodified lines
234	235	size_t nr_programs;
235	236	struct bpf_map *maps;
236	237	size_t nr_maps;
	238	+ size_t maps_cap;
237	239	struct bpf_secdata sections;
238	240
239	241	bool loaded;
@@ -260,6 +262,7 @@ struct bpf_object { · 20 unmodified lines
260	262	} *reloc;
261	263	int nr_reloc;
262	264	int maps_shndx;
	265	+ int btf_maps_shndx;
263	266	int text_shndx;
264	267	int data_shndx;
265	268	int rodata_shndx;
@@ -512,6 +515,7 @@ static struct bpf_object bpf_object__new(const char path, · 246 unmodified lines
512	515	obj->efile.obj_buf = obj_buf;
513	516	obj->efile.obj_buf_sz = obj_buf_sz;
514	517	obj->efile.maps_shndx = -1;
	518	+ obj->efile.btf_maps_shndx = -1;
515	519	obj->efile.data_shndx = -1;
516	520	obj->efile.rodata_shndx = -1;
517	521	obj->efile.bss_shndx = -1;
@@ -646,7 +650,9 @@ static int compare_bpf_map(const void _a, const void _b) · 128 unmodified lines
646	650	const struct bpf_map *a = _a;
647	651	const struct bpf_map *b = _b;
648	652
649		− return a->offset - b->offset;
	653	+ if (a->sec_idx != b->sec_idx)
	654	+ return a->sec_idx - b->sec_idx;
	655	+ return a->sec_offset - b->sec_offset;
650	656	}
651	657
652	658	static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
@@ -763,24 +769,55 @@ int bpf_object__variable_offset(const struct bpf_object obj, const char name, · 110 unmodified lines
763	769	return -ENOENT;
764	770	}
765	771
766		−static bool bpf_object__has_maps(const struct bpf_object *obj)
	772	+static struct bpf_map bpf_object__add_map(struct bpf_object obj)
767	773	{
768		− return obj->efile.maps_shndx >= 0 \|\|
769		− obj->efile.data_shndx >= 0 \|\|
770		− obj->efile.rodata_shndx >= 0 \|\|
771		− obj->efile.bss_shndx >= 0;
	774	+ struct bpf_map *new_maps;
	775	+ size_t new_cap;
	776	+ int i;
	777	+
	778	+ if (obj->nr_maps < obj->maps_cap)
	779	+ return &obj->maps[obj->nr_maps++];
	780	+
	781	+ new_cap = max(4ul, obj->maps_cap * 3 / 2);
	782	+ new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps));
	783	+ if (!new_maps) {
	784	+ pr_warning("alloc maps for object failed\n");
	785	+ return ERR_PTR(-ENOMEM);
	786	+ }
	787	+
	788	+ obj->maps_cap = new_cap;
	789	+ obj->maps = new_maps;
	790	+
	791	+ /* zero out new maps */
	792	+ memset(obj->maps + obj->nr_maps, 0,
	793	+ (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps));
	794	+ /*
	795	+ * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin)
	796	+ * when failure (zclose won't close negative fd)).
	797	+ */
	798	+ for (i = obj->nr_maps; i < obj->maps_cap; i++) {
	799	+ obj->maps[i].fd = -1;
	800	+ obj->maps[i].inner_map_fd = -1;
	801	+ }
	802	+
	803	+ return &obj->maps[obj->nr_maps++];
772	804	}
773	805
774	806	static int
775		−bpf_object__init_internal_map(struct bpf_object obj, struct bpf_map map,
776		− enum libbpf_map_type type, Elf_Data *data,
777		− void **data_buff)
	807	+bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
	808	+ int sec_idx, Elf_Data data, void *data_buff)
778	809	{
779		− struct bpf_map_def *def = &map->def;
780	810	char map_name[BPF_OBJ_NAME_LEN];
	811	+ struct bpf_map_def *def;
	812	+ struct bpf_map *map;
	813	+
	814	+ map = bpf_object__add_map(obj);
	815	+ if (IS_ERR(map))
	816	+ return PTR_ERR(map);
781	817
782	818	map->libbpf_type = type;
783		− map->offset = ~(typeof(map->offset))0;
	819	+ map->sec_idx = sec_idx;
	820	+ map->sec_offset = 0;
784	821	snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name,
785	822	libbpf_type_to_btf_name[type]);
786	823	map->name = strdup(map_name);
@@ -788,7 +825,10 @@ bpf_object__init_internal_map(struct bpf_object obj, struct bpf_map map, · 1 unmodified line
788	825	pr_warning("failed to alloc map name\n");
789	826	return -ENOMEM;
790	827	}
	828	+ pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n",
	829	+ map_name, map->sec_idx, map->sec_offset);
791	830
	831	+ def = &map->def;
792	832	def->type = BPF_MAP_TYPE_ARRAY;
793	833	def->key_size = sizeof(int);
794	834	def->value_size = data->d_size;
@@ -808,29 +848,61 @@ bpf_object__init_internal_map(struct bpf_object obj, struct bpf_map map, · 13 unmodified lines
808	848	return 0;
809	849	}
810	850
811		−static int bpf_object__init_maps(struct bpf_object *obj, int flags)
	851	+static int bpf_object__init_global_data_maps(struct bpf_object *obj)
	852	+{
	853	+ int err;
	854	+
	855	+ if (!obj->caps.global_data)
	856	+ return 0;
	857	+ /*
	858	+ * Populate obj->maps with libbpf internal maps.
	859	+ */
	860	+ if (obj->efile.data_shndx >= 0) {
	861	+ err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
	862	+ obj->efile.data_shndx,
	863	+ obj->efile.data,
	864	+ &obj->sections.data);
	865	+ if (err)
	866	+ return err;
	867	+ }
	868	+ if (obj->efile.rodata_shndx >= 0) {
	869	+ err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
	870	+ obj->efile.rodata_shndx,
	871	+ obj->efile.rodata,
	872	+ &obj->sections.rodata);
	873	+ if (err)
	874	+ return err;
	875	+ }
	876	+ if (obj->efile.bss_shndx >= 0) {
	877	+ err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
	878	+ obj->efile.bss_shndx,
	879	+ obj->efile.bss, NULL);
	880	+ if (err)
	881	+ return err;
	882	+ }
	883	+ return 0;
	884	+}
	885	+
	886	+static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict)
812	887	{
813		− int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0;
814		− bool strict = !(flags & MAPS_RELAX_COMPAT);
815	888	Elf_Data *symbols = obj->efile.symbols;
	889	+ int i, map_def_sz = 0, nr_maps = 0, nr_syms;
816	890	Elf_Data *data = NULL;
817		− int ret = 0;
	891	+ Elf_Scn *scn;
	892	+
	893	+ if (obj->efile.maps_shndx < 0)
	894	+ return 0;
818	895
819	896	if (!symbols)
820	897	return -EINVAL;
821		− nr_syms = symbols->d_size / sizeof(GElf_Sym);
822	898
823		− if (obj->efile.maps_shndx >= 0) {
824		− Elf_Scn *scn = elf_getscn(obj->efile.elf,
825		− obj->efile.maps_shndx);
826		−
827		− if (scn)
828		− data = elf_getdata(scn, NULL);
829		− if (!scn \|\| !data) {
830		− pr_warning("failed to get Elf_Data from map section %d\n",
831		− obj->efile.maps_shndx);
832		− return -EINVAL;
833		− }
	899	+ scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
	900	+ if (scn)
	901	+ data = elf_getdata(scn, NULL);
	902	+ if (!scn \|\| !data) {
	903	+ pr_warning("failed to get Elf_Data from map section %d\n",
	904	+ obj->efile.maps_shndx);
	905	+ return -EINVAL;
834	906	}
835	907
836	908	/*
@@ -840,16 +912,8 @@ static int bpf_object__init_maps(struct bpf_object *obj, int flags) · 3 unmodified lines
840	912	*
841	913	* TODO: Detect array of map and report error.
842	914	*/
843		− if (obj->caps.global_data) {
844		− if (obj->efile.data_shndx >= 0)
845		− nr_maps_glob++;
846		− if (obj->efile.rodata_shndx >= 0)
847		− nr_maps_glob++;
848		− if (obj->efile.bss_shndx >= 0)
849		− nr_maps_glob++;
850		− }
851		−
852		− for (i = 0; data && i < nr_syms; i++) {
	915	+ nr_syms = symbols->d_size / sizeof(GElf_Sym);
	916	+ for (i = 0; i < nr_syms; i++) {
853	917	GElf_Sym sym;
854	918
855	919	if (!gelf_getsym(symbols, i, &sym))
@@ -858,79 +922,59 @@ static int bpf_object__init_maps(struct bpf_object *obj, int flags) · 2 unmodified lines
858	922	continue;
859	923	nr_maps++;
860	924	}
861		−
862		− if (!nr_maps && !nr_maps_glob)
863		− return 0;
864		−
865	925	/* Assume equally sized map definitions */
866		− if (data) {
867		− pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
868		− nr_maps, data->d_size);
869		−
870		− map_def_sz = data->d_size / nr_maps;
871		− if (!data->d_size \|\| (data->d_size % nr_maps) != 0) {
872		− pr_warning("unable to determine map definition size "
873		− "section %s, %d maps in %zd bytes\n",
874		− obj->path, nr_maps, data->d_size);
875		− return -EINVAL;
876		− }
877		− }
878		−
879		− nr_maps += nr_maps_glob;
880		− obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
881		− if (!obj->maps) {
882		− pr_warning("alloc maps for object failed\n");
883		− return -ENOMEM;
884		− }
885		− obj->nr_maps = nr_maps;
886		−
887		− for (i = 0; i < nr_maps; i++) {
888		− /*
889		− * fill all fd with -1 so won't close incorrect
890		− * fd (fd=0 is stdin) when failure (zclose won't close
891		− * negative fd)).
892		− */
893		− obj->maps[i].fd = -1;
894		− obj->maps[i].inner_map_fd = -1;
	926	+ pr_debug("maps in %s: %d maps in %zd bytes\n",
	927	+ obj->path, nr_maps, data->d_size);
	928	+
	929	+ map_def_sz = data->d_size / nr_maps;
	930	+ if (!data->d_size \|\| (data->d_size % nr_maps) != 0) {
	931	+ pr_warning("unable to determine map definition size "
	932	+ "section %s, %d maps in %zd bytes\n",
	933	+ obj->path, nr_maps, data->d_size);
	934	+ return -EINVAL;
895	935	}
896	936
897		− /*
898		− * Fill obj->maps using data in "maps" section.
899		− */
900		− for (i = 0, map_idx = 0; data && i < nr_syms; i++) {
	937	+ /* Fill obj->maps using data in "maps" section. */
	938	+ for (i = 0; i < nr_syms; i++) {
901	939	GElf_Sym sym;
902	940	const char *map_name;
903	941	struct bpf_map_def *def;
	942	+ struct bpf_map *map;
904	943
905	944	if (!gelf_getsym(symbols, i, &sym))
906	945	continue;
907	946	if (sym.st_shndx != obj->efile.maps_shndx)
908	947	continue;
909	948
910		− map_name = elf_strptr(obj->efile.elf,
911		− obj->efile.strtabidx,
	949	+ map = bpf_object__add_map(obj);
	950	+ if (IS_ERR(map))
	951	+ return PTR_ERR(map);
	952	+
	953	+ map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
912	954	sym.st_name);
913	955	if (!map_name) {
914	956	pr_warning("failed to get map #%d name sym string for obj %s\n",
915		− map_idx, obj->path);
	957	+ i, obj->path);
916	958	return -LIBBPF_ERRNO__FORMAT;
917	959	}
918	960
919		− obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC;
920		− obj->maps[map_idx].offset = sym.st_value;
	961	+ map->libbpf_type = LIBBPF_MAP_UNSPEC;
	962	+ map->sec_idx = sym.st_shndx;
	963	+ map->sec_offset = sym.st_value;
	964	+ pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n",
	965	+ map_name, map->sec_idx, map->sec_offset);
921	966	if (sym.st_value + map_def_sz > data->d_size) {
922	967	pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
923	968	obj->path, map_name);
924	969	return -EINVAL;
925	970	}
926	971
927		− obj->maps[map_idx].name = strdup(map_name);
928		− if (!obj->maps[map_idx].name) {
	972	+ map->name = strdup(map_name);
	973	+ if (!map->name) {
929	974	pr_warning("failed to alloc map name\n");
930	975	return -ENOMEM;
931	976	}
932		− pr_debug("map %d is \"%s\"\n", map_idx,
933		− obj->maps[map_idx].name);
	977	+ pr_debug("map %d is \"%s\"\n", i, map->name);
934	978	def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
935	979	/*
936	980	* If the definition of the map in the object file fits in
@@ -939,7 +983,7 @@ static int bpf_object__init_maps(struct bpf_object *obj, int flags) · 2 unmodified lines
939	983	* calloc above.
940	984	*/
941	985	if (map_def_sz <= sizeof(struct bpf_map_def)) {
942		− memcpy(&obj->maps[map_idx].def, def, map_def_sz);
	986	+ memcpy(&map->def, def, map_def_sz);
943	987	} else {
944	988	/*
945	989	* Here the map structure being read is bigger than what
@@ -959,37 +1003,340 @@ static int bpf_object__init_maps(struct bpf_object *obj, int flags) · 13 unmodified lines
959	1003	return -EINVAL;
960	1004	}
961	1005	}
962		− memcpy(&obj->maps[map_idx].def, def,
963		− sizeof(struct bpf_map_def));
	1006	+ memcpy(&map->def, def, sizeof(struct bpf_map_def));
964	1007	}
965		− map_idx++;
966	1008	}
	1009	+ return 0;
	1010	+}
967	1011
968		− if (!obj->caps.global_data)
969		− goto finalize;
	1012	+static const struct btf_type skip_mods_and_typedefs(const struct btf btf,
	1013	+ __u32 id)
	1014	+{
	1015	+ const struct btf_type *t = btf__type_by_id(btf, id);
970	1016
971		− /*
972		− * Populate rest of obj->maps with libbpf internal maps.
973		− */
974		− if (obj->efile.data_shndx >= 0)
975		− ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
976		− LIBBPF_MAP_DATA,
977		− obj->efile.data,
978		− &obj->sections.data);
979		− if (!ret && obj->efile.rodata_shndx >= 0)
980		− ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
981		− LIBBPF_MAP_RODATA,
982		− obj->efile.rodata,
983		− &obj->sections.rodata);
984		− if (!ret && obj->efile.bss_shndx >= 0)
985		− ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
986		− LIBBPF_MAP_BSS,
987		− obj->efile.bss, NULL);
988		−finalize:
989		− if (!ret)
	1017	+ while (true) {
	1018	+ switch (BTF_INFO_KIND(t->info)) {
	1019	+ case BTF_KIND_VOLATILE:
	1020	+ case BTF_KIND_CONST:
	1021	+ case BTF_KIND_RESTRICT:
	1022	+ case BTF_KIND_TYPEDEF:
	1023	+ t = btf__type_by_id(btf, t->type);
	1024	+ break;
	1025	+ default:
	1026	+ return t;
	1027	+ }
	1028	+ }
	1029	+}
	1030	+
	1031	+static bool get_map_field_int(const char *map_name,
	1032	+ const struct btf *btf,
	1033	+ const struct btf_type *def,
	1034	+ const struct btf_member *m,
	1035	+ const void data, __u32 res) {
	1036	+ const struct btf_type *t = skip_mods_and_typedefs(btf, m->type);
	1037	+ const char *name = btf__name_by_offset(btf, m->name_off);
	1038	+ __u32 int_info = (const __u32 )(const void *)(t + 1);
	1039	+
	1040	+ if (BTF_INFO_KIND(t->info) != BTF_KIND_INT) {
	1041	+ pr_warning("map '%s': attr '%s': expected INT, got %u.\n",
	1042	+ map_name, name, BTF_INFO_KIND(t->info));
	1043	+ return false;
	1044	+ }
	1045	+ if (t->size != 4 \|\| BTF_INT_BITS(int_info) != 32 \|\|
	1046	+ BTF_INT_OFFSET(int_info)) {
	1047	+ pr_warning("map '%s': attr '%s': expected 32-bit non-bitfield integer, "
	1048	+ "got %u-byte (%d-bit) one with bit offset %d.\n",
	1049	+ map_name, name, t->size, BTF_INT_BITS(int_info),
	1050	+ BTF_INT_OFFSET(int_info));
	1051	+ return false;
	1052	+ }
	1053	+ if (BTF_INFO_KFLAG(def->info) && BTF_MEMBER_BITFIELD_SIZE(m->offset)) {
	1054	+ pr_warning("map '%s': attr '%s': bitfield is not supported.\n",
	1055	+ map_name, name);
	1056	+ return false;
	1057	+ }
	1058	+ if (m->offset % 32) {
	1059	+ pr_warning("map '%s': attr '%s': unaligned fields are not supported.\n",
	1060	+ map_name, name);
	1061	+ return false;
	1062	+ }
	1063	+
	1064	+ res = (const __u32 *)(data + m->offset / 8);
	1065	+ return true;
	1066	+}
	1067	+
	1068	+static int bpf_object__init_user_btf_map(struct bpf_object *obj,
	1069	+ const struct btf_type *sec,
	1070	+ int var_idx, int sec_idx,
	1071	+ const Elf_Data *data, bool strict)
	1072	+{
	1073	+ const struct btf_type var, def, *t;
	1074	+ const struct btf_var_secinfo *vi;
	1075	+ const struct btf_var *var_extra;
	1076	+ const struct btf_member *m;
	1077	+ const void *def_data;
	1078	+ const char *map_name;
	1079	+ struct bpf_map *map;
	1080	+ int vlen, i;
	1081	+
	1082	+ vi = (const struct btf_var_secinfo )(const void )(sec + 1) + var_idx;
	1083	+ var = btf__type_by_id(obj->btf, vi->type);
	1084	+ var_extra = (const void *)(var + 1);
	1085	+ map_name = btf__name_by_offset(obj->btf, var->name_off);
	1086	+ vlen = BTF_INFO_VLEN(var->info);
	1087	+
	1088	+ if (map_name == NULL \|\| map_name[0] == '\0') {
	1089	+ pr_warning("map #%d: empty name.\n", var_idx);
	1090	+ return -EINVAL;
	1091	+ }
	1092	+ if ((__u64)vi->offset + vi->size > data->d_size) {
	1093	+ pr_warning("map '%s' BTF data is corrupted.\n", map_name);
	1094	+ return -EINVAL;
	1095	+ }
	1096	+ if (BTF_INFO_KIND(var->info) != BTF_KIND_VAR) {
	1097	+ pr_warning("map '%s': unexpected var kind %u.\n",
	1098	+ map_name, BTF_INFO_KIND(var->info));
	1099	+ return -EINVAL;
	1100	+ }
	1101	+ if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED &&
	1102	+ var_extra->linkage != BTF_VAR_STATIC) {
	1103	+ pr_warning("map '%s': unsupported var linkage %u.\n",
	1104	+ map_name, var_extra->linkage);
	1105	+ return -EOPNOTSUPP;
	1106	+ }
	1107	+
	1108	+ def = skip_mods_and_typedefs(obj->btf, var->type);
	1109	+ if (BTF_INFO_KIND(def->info) != BTF_KIND_STRUCT) {
	1110	+ pr_warning("map '%s': unexpected def kind %u.\n",
	1111	+ map_name, BTF_INFO_KIND(var->info));
	1112	+ return -EINVAL;
	1113	+ }
	1114	+ if (def->size > vi->size) {
	1115	+ pr_warning("map '%s': invalid def size.\n", map_name);
	1116	+ return -EINVAL;
	1117	+ }
	1118	+
	1119	+ map = bpf_object__add_map(obj);
	1120	+ if (IS_ERR(map))
	1121	+ return PTR_ERR(map);
	1122	+ map->name = strdup(map_name);
	1123	+ if (!map->name) {
	1124	+ pr_warning("map '%s': failed to alloc map name.\n", map_name);
	1125	+ return -ENOMEM;
	1126	+ }
	1127	+ map->libbpf_type = LIBBPF_MAP_UNSPEC;
	1128	+ map->def.type = BPF_MAP_TYPE_UNSPEC;
	1129	+ map->sec_idx = sec_idx;
	1130	+ map->sec_offset = vi->offset;
	1131	+ pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
	1132	+ map_name, map->sec_idx, map->sec_offset);
	1133	+
	1134	+ def_data = data->d_buf + vi->offset;
	1135	+ vlen = BTF_INFO_VLEN(def->info);
	1136	+ m = (const void *)(def + 1);
	1137	+ for (i = 0; i < vlen; i++, m++) {
	1138	+ const char *name = btf__name_by_offset(obj->btf, m->name_off);
	1139	+
	1140	+ if (!name) {
	1141	+ pr_warning("map '%s': invalid field #%d.\n",
	1142	+ map_name, i);
	1143	+ return -EINVAL;
	1144	+ }
	1145	+ if (strcmp(name, "type") == 0) {
	1146	+ if (!get_map_field_int(map_name, obj->btf, def, m,
	1147	+ def_data, &map->def.type))
	1148	+ return -EINVAL;
	1149	+ pr_debug("map '%s': found type = %u.\n",
	1150	+ map_name, map->def.type);
	1151	+ } else if (strcmp(name, "max_entries") == 0) {
	1152	+ if (!get_map_field_int(map_name, obj->btf, def, m,
	1153	+ def_data, &map->def.max_entries))
	1154	+ return -EINVAL;
	1155	+ pr_debug("map '%s': found max_entries = %u.\n",
	1156	+ map_name, map->def.max_entries);
	1157	+ } else if (strcmp(name, "map_flags") == 0) {
	1158	+ if (!get_map_field_int(map_name, obj->btf, def, m,
	1159	+ def_data, &map->def.map_flags))
	1160	+ return -EINVAL;
	1161	+ pr_debug("map '%s': found map_flags = %u.\n",
	1162	+ map_name, map->def.map_flags);
	1163	+ } else if (strcmp(name, "key_size") == 0) {
	1164	+ __u32 sz;
	1165	+
	1166	+ if (!get_map_field_int(map_name, obj->btf, def, m,
	1167	+ def_data, &sz))
	1168	+ return -EINVAL;
	1169	+ pr_debug("map '%s': found key_size = %u.\n",
	1170	+ map_name, sz);
	1171	+ if (map->def.key_size && map->def.key_size != sz) {
	1172	+ pr_warning("map '%s': conflictling key size %u != %u.\n",
	1173	+ map_name, map->def.key_size, sz);
	1174	+ return -EINVAL;
	1175	+ }
	1176	+ map->def.key_size = sz;
	1177	+ } else if (strcmp(name, "key") == 0) {
	1178	+ __s64 sz;
	1179	+
	1180	+ t = btf__type_by_id(obj->btf, m->type);
	1181	+ if (!t) {
	1182	+ pr_warning("map '%s': key type [%d] not found.\n",
	1183	+ map_name, m->type);
	1184	+ return -EINVAL;
	1185	+ }
	1186	+ if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) {
	1187	+ pr_warning("map '%s': key spec is not PTR: %u.\n",
	1188	+ map_name, BTF_INFO_KIND(t->info));
	1189	+ return -EINVAL;
	1190	+ }
	1191	+ sz = btf__resolve_size(obj->btf, t->type);
	1192	+ if (sz < 0) {
	1193	+ pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n",
	1194	+ map_name, t->type, sz);
	1195	+ return sz;
	1196	+ }
	1197	+ pr_debug("map '%s': found key [%u], sz = %lld.\n",
	1198	+ map_name, t->type, sz);
	1199	+ if (map->def.key_size && map->def.key_size != sz) {
	1200	+ pr_warning("map '%s': conflictling key size %u != %lld.\n",
	1201	+ map_name, map->def.key_size, sz);
	1202	+ return -EINVAL;
	1203	+ }
	1204	+ map->def.key_size = sz;
	1205	+ map->btf_key_type_id = t->type;
	1206	+ } else if (strcmp(name, "value_size") == 0) {
	1207	+ __u32 sz;
	1208	+
	1209	+ if (!get_map_field_int(map_name, obj->btf, def, m,
	1210	+ def_data, &sz))
	1211	+ return -EINVAL;
	1212	+ pr_debug("map '%s': found value_size = %u.\n",
	1213	+ map_name, sz);
	1214	+ if (map->def.value_size && map->def.value_size != sz) {
	1215	+ pr_warning("map '%s': conflictling value size %u != %u.\n",
	1216	+ map_name, map->def.value_size, sz);
	1217	+ return -EINVAL;
	1218	+ }
	1219	+ map->def.value_size = sz;
	1220	+ } else if (strcmp(name, "value") == 0) {
	1221	+ __s64 sz;
	1222	+
	1223	+ t = btf__type_by_id(obj->btf, m->type);
	1224	+ if (!t) {
	1225	+ pr_warning("map '%s': value type [%d] not found.\n",
	1226	+ map_name, m->type);
	1227	+ return -EINVAL;
	1228	+ }
	1229	+ if (BTF_INFO_KIND(t->info) != BTF_KIND_PTR) {
	1230	+ pr_warning("map '%s': value spec is not PTR: %u.\n",
	1231	+ map_name, BTF_INFO_KIND(t->info));
	1232	+ return -EINVAL;
	1233	+ }
	1234	+ sz = btf__resolve_size(obj->btf, t->type);
	1235	+ if (sz < 0) {
	1236	+ pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n",
	1237	+ map_name, t->type, sz);
	1238	+ return sz;
	1239	+ }
	1240	+ pr_debug("map '%s': found value [%u], sz = %lld.\n",
	1241	+ map_name, t->type, sz);
	1242	+ if (map->def.value_size && map->def.value_size != sz) {
	1243	+ pr_warning("map '%s': conflictling value size %u != %lld.\n",
	1244	+ map_name, map->def.value_size, sz);
	1245	+ return -EINVAL;
	1246	+ }
	1247	+ map->def.value_size = sz;
	1248	+ map->btf_value_type_id = t->type;
	1249	+ } else {
	1250	+ if (strict) {
	1251	+ pr_warning("map '%s': unknown field '%s'.\n",
	1252	+ map_name, name);
	1253	+ return -ENOTSUP;
	1254	+ }
	1255	+ pr_debug("map '%s': ignoring unknown field '%s'.\n",
	1256	+ map_name, name);
	1257	+ }
	1258	+ }
	1259	+
	1260	+ if (map->def.type == BPF_MAP_TYPE_UNSPEC) {
	1261	+ pr_warning("map '%s': map type isn't specified.\n", map_name);
	1262	+ return -EINVAL;
	1263	+ }
	1264	+
	1265	+ return 0;
	1266	+}
	1267	+
	1268	+static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict)
	1269	+{
	1270	+ const struct btf_type *sec = NULL;
	1271	+ int nr_types, i, vlen, err;
	1272	+ const struct btf_type *t;
	1273	+ const char *name;
	1274	+ Elf_Data *data;
	1275	+ Elf_Scn *scn;
	1276	+
	1277	+ if (obj->efile.btf_maps_shndx < 0)
	1278	+ return 0;
	1279	+
	1280	+ scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx);
	1281	+ if (scn)
	1282	+ data = elf_getdata(scn, NULL);
	1283	+ if (!scn \|\| !data) {
	1284	+ pr_warning("failed to get Elf_Data from map section %d (%s)\n",
	1285	+ obj->efile.maps_shndx, MAPS_ELF_SEC);
	1286	+ return -EINVAL;
	1287	+ }
	1288	+
	1289	+ nr_types = btf__get_nr_types(obj->btf);
	1290	+ for (i = 1; i <= nr_types; i++) {
	1291	+ t = btf__type_by_id(obj->btf, i);
	1292	+ if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
	1293	+ continue;
	1294	+ name = btf__name_by_offset(obj->btf, t->name_off);
	1295	+ if (strcmp(name, MAPS_ELF_SEC) == 0) {
	1296	+ sec = t;
	1297	+ break;
	1298	+ }
	1299	+ }
	1300	+
	1301	+ if (!sec) {
	1302	+ pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
	1303	+ return -ENOENT;
	1304	+ }
	1305	+
	1306	+ vlen = BTF_INFO_VLEN(sec->info);
	1307	+ for (i = 0; i < vlen; i++) {
	1308	+ err = bpf_object__init_user_btf_map(obj, sec, i,
	1309	+ obj->efile.btf_maps_shndx,
	1310	+ data, strict);
	1311	+ if (err)
	1312	+ return err;
	1313	+ }
	1314	+
	1315	+ return 0;
	1316	+}
	1317	+
	1318	+static int bpf_object__init_maps(struct bpf_object *obj, int flags)
	1319	+{
	1320	+ bool strict = !(flags & MAPS_RELAX_COMPAT);
	1321	+ int err;
	1322	+
	1323	+ err = bpf_object__init_user_maps(obj, strict);
	1324	+ if (err)
	1325	+ return err;
	1326	+
	1327	+ err = bpf_object__init_user_btf_maps(obj, strict);
	1328	+ if (err)
	1329	+ return err;
	1330	+
	1331	+ err = bpf_object__init_global_data_maps(obj);
	1332	+ if (err)
	1333	+ return err;
	1334	+
	1335	+ if (obj->nr_maps) {
990	1336	qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]),
991	1337	compare_bpf_map);
992		− return ret;
	1338	+ }
	1339	+ return 0;
993	1340	}
994	1341
995	1342	static bool section_have_execinstr(struct bpf_object *obj, int idx)
@@ -1078,6 +1425,86 @@ static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) · 82 unmodified lines
1078	1425	}
1079	1426	}
1080	1427
	1428	+static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj)
	1429	+{
	1430	+ return obj->efile.btf_maps_shndx >= 0;
	1431	+}
	1432	+
	1433	+static int bpf_object__init_btf(struct bpf_object *obj,
	1434	+ Elf_Data *btf_data,
	1435	+ Elf_Data *btf_ext_data)
	1436	+{
	1437	+ bool btf_required = bpf_object__is_btf_mandatory(obj);
	1438	+ int err = 0;
	1439	+
	1440	+ if (btf_data) {
	1441	+ obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
	1442	+ if (IS_ERR(obj->btf)) {
	1443	+ pr_warning("Error loading ELF section %s: %d.\n",
	1444	+ BTF_ELF_SEC, err);
	1445	+ goto out;
	1446	+ }
	1447	+ err = btf__finalize_data(obj, obj->btf);
	1448	+ if (err) {
	1449	+ pr_warning("Error finalizing %s: %d.\n",
	1450	+ BTF_ELF_SEC, err);
	1451	+ goto out;
	1452	+ }
	1453	+ }
	1454	+ if (btf_ext_data) {
	1455	+ if (!obj->btf) {
	1456	+ pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
	1457	+ BTF_EXT_ELF_SEC, BTF_ELF_SEC);
	1458	+ goto out;
	1459	+ }
	1460	+ obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
	1461	+ btf_ext_data->d_size);
	1462	+ if (IS_ERR(obj->btf_ext)) {
	1463	+ pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
	1464	+ BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext));
	1465	+ obj->btf_ext = NULL;
	1466	+ goto out;
	1467	+ }
	1468	+ }
	1469	+out:
	1470	+ if (err \|\| IS_ERR(obj->btf)) {
	1471	+ if (btf_required)
	1472	+ err = err ? : PTR_ERR(obj->btf);
	1473	+ else
	1474	+ err = 0;
	1475	+ if (!IS_ERR_OR_NULL(obj->btf))
	1476	+ btf__free(obj->btf);
	1477	+ obj->btf = NULL;
	1478	+ }
	1479	+ if (btf_required && !obj->btf) {
	1480	+ pr_warning("BTF is required, but is missing or corrupted.\n");
	1481	+ return err == 0 ? -ENOENT : err;
	1482	+ }
	1483	+ return 0;
	1484	+}
	1485	+
	1486	+static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
	1487	+{
	1488	+ int err = 0;
	1489	+
	1490	+ if (!obj->btf)
	1491	+ return 0;
	1492	+
	1493	+ bpf_object__sanitize_btf(obj);
	1494	+ bpf_object__sanitize_btf_ext(obj);
	1495	+
	1496	+ err = btf__load(obj->btf);
	1497	+ if (err) {
	1498	+ pr_warning("Error loading %s into kernel: %d.\n",
	1499	+ BTF_ELF_SEC, err);
	1500	+ btf__free(obj->btf);
	1501	+ obj->btf = NULL;
	1502	+ if (bpf_object__is_btf_mandatory(obj))
	1503	+ return err;
	1504	+ }
	1505	+ return 0;
	1506	+}
	1507	+
1081	1508	static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
1082	1509	{
1083	1510	Elf *elf = obj->efile.elf;
@@ -1102,24 +1529,21 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 18 unmodified lines
1102	1529	if (gelf_getshdr(scn, &sh) != &sh) {
1103	1530	pr_warning("failed to get section(%d) header from %s\n",
1104	1531	idx, obj->path);
1105		− err = -LIBBPF_ERRNO__FORMAT;
1106		− goto out;
	1532	+ return -LIBBPF_ERRNO__FORMAT;
1107	1533	}
1108	1534
1109	1535	name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
1110	1536	if (!name) {
1111	1537	pr_warning("failed to get section(%d) name from %s\n",
1112	1538	idx, obj->path);
1113		− err = -LIBBPF_ERRNO__FORMAT;
1114		− goto out;
	1539	+ return -LIBBPF_ERRNO__FORMAT;
1115	1540	}
1116	1541
1117	1542	data = elf_getdata(scn, 0);
1118	1543	if (!data) {
1119	1544	pr_warning("failed to get section(%d) data from %s(%s)\n",
1120	1545	idx, name, obj->path);
1121		− err = -LIBBPF_ERRNO__FORMAT;
1122		− goto out;
	1546	+ return -LIBBPF_ERRNO__FORMAT;
1123	1547	}
1124	1548	pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
1125	1549	idx, name, (unsigned long)data->d_size,
@@ -1130,12 +1554,18 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 4 unmodified lines
1130	1554	err = bpf_object__init_license(obj,
1131	1555	data->d_buf,
1132	1556	data->d_size);
	1557	+ if (err)
	1558	+ return err;
1133	1559	} else if (strcmp(name, "version") == 0) {
1134	1560	err = bpf_object__init_kversion(obj,
1135	1561	data->d_buf,
1136	1562	data->d_size);
	1563	+ if (err)
	1564	+ return err;
1137	1565	} else if (strcmp(name, "maps") == 0) {
1138	1566	obj->efile.maps_shndx = idx;
	1567	+ } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
	1568	+ obj->efile.btf_maps_shndx = idx;
1139	1569	} else if (strcmp(name, BTF_ELF_SEC) == 0) {
1140	1570	btf_data = data;
1141	1571	} else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
@@ -1144,11 +1574,10 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 2 unmodified lines
1144	1574	if (obj->efile.symbols) {
1145	1575	pr_warning("bpf: multiple SYMTAB in %s\n",
1146	1576	obj->path);
1147		− err = -LIBBPF_ERRNO__FORMAT;
1148		− } else {
1149		− obj->efile.symbols = data;
1150		− obj->efile.strtabidx = sh.sh_link;
	1577	+ return -LIBBPF_ERRNO__FORMAT;
1151	1578	}
	1579	+ obj->efile.symbols = data;
	1580	+ obj->efile.strtabidx = sh.sh_link;
1152	1581	} else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
1153	1582	if (sh.sh_flags & SHF_EXECINSTR) {
1154	1583	if (strcmp(name, ".text") == 0)
@@ -1162,6 +1591,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 7 unmodified lines
1162	1591
1163	1592	pr_warning("failed to alloc program %s (%s): %s",
1164	1593	name, obj->path, cp);
	1594	+ return err;
1165	1595	}
1166	1596	} else if (strcmp(name, ".data") == 0) {
1167	1597	obj->efile.data = data;
@@ -1173,8 +1603,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 5 unmodified lines
1173	1603	pr_debug("skip section(%d) %s\n", idx, name);
1174	1604	}
1175	1605	} else if (sh.sh_type == SHT_REL) {
	1606	+ int nr_reloc = obj->efile.nr_reloc;
1176	1607	void *reloc = obj->efile.reloc;
1177		− int nr_reloc = obj->efile.nr_reloc + 1;
1178	1608	int sec = sh.sh_info; /* points to other section */
1179	1609
1180	1610	/* Only do relo for section with exec instructions */
@@ -1184,79 +1614,37 @@ static int bpf_object__elf_collect(struct bpf_object *obj, int flags) · 3 unmodified lines
1184	1614	continue;
1185	1615	}
1186	1616
1187		− reloc = reallocarray(reloc, nr_reloc,
	1617	+ reloc = reallocarray(reloc, nr_reloc + 1,
1188	1618	sizeof(*obj->efile.reloc));
1189	1619	if (!reloc) {
1190	1620	pr_warning("realloc failed\n");
1191		− err = -ENOMEM;
1192		− } else {
1193		− int n = nr_reloc - 1;
	1621	+ return -ENOMEM;
	1622	+ }
1194	1623
1195		− obj->efile.reloc = reloc;
1196		− obj->efile.nr_reloc = nr_reloc;
	1624	+ obj->efile.reloc = reloc;
	1625	+ obj->efile.nr_reloc++;
1197	1626
1198		− obj->efile.reloc[n].shdr = sh;
1199		− obj->efile.reloc[n].data = data;
1200		− }
	1627	+ obj->efile.reloc[nr_reloc].shdr = sh;
	1628	+ obj->efile.reloc[nr_reloc].data = data;
1201	1629	} else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) {
1202	1630	obj->efile.bss = data;
1203	1631	obj->efile.bss_shndx = idx;
1204	1632	} else {
1205	1633	pr_debug("skip section(%d) %s\n", idx, name);
1206	1634	}
1207		− if (err)
1208		− goto out;
1209	1635	}
1210	1636
1211	1637	if (!obj->efile.strtabidx \|\| obj->efile.strtabidx >= idx) {
1212	1638	pr_warning("Corrupted ELF file: index of strtab invalid\n");
1213	1639	return -LIBBPF_ERRNO__FORMAT;
1214	1640	}
1215		− if (btf_data) {
1216		− obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
1217		− if (IS_ERR(obj->btf)) {
1218		− pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
1219		− BTF_ELF_SEC, PTR_ERR(obj->btf));
1220		− obj->btf = NULL;
1221		− } else {
1222		− err = btf__finalize_data(obj, obj->btf);
1223		− if (!err) {
1224		− bpf_object__sanitize_btf(obj);
1225		− err = btf__load(obj->btf);
1226		− }
1227		− if (err) {
1228		− pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n",
1229		− BTF_ELF_SEC, err);
1230		− btf__free(obj->btf);
1231		− obj->btf = NULL;
1232		− err = 0;
1233		− }
1234		− }
1235		− }
1236		− if (btf_ext_data) {
1237		− if (!obj->btf) {
1238		− pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
1239		− BTF_EXT_ELF_SEC, BTF_ELF_SEC);
1240		− } else {
1241		− obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
1242		− btf_ext_data->d_size);
1243		− if (IS_ERR(obj->btf_ext)) {
1244		− pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
1245		− BTF_EXT_ELF_SEC,
1246		− PTR_ERR(obj->btf_ext));
1247		− obj->btf_ext = NULL;
1248		− } else {
1249		− bpf_object__sanitize_btf_ext(obj);
1250		− }
1251		− }
1252		− }
1253		− if (bpf_object__has_maps(obj)) {
	1641	+ err = bpf_object__init_btf(obj, btf_data, btf_ext_data);
	1642	+ if (!err)
1254	1643	err = bpf_object__init_maps(obj, flags);
1255		− if (err)
1256		− goto out;
1257		− }
1258		− err = bpf_object__init_prog_names(obj);
1259		−out:
	1644	+ if (!err)
	1645	+ err = bpf_object__sanitize_and_load_btf(obj);
	1646	+ if (!err)
	1647	+ err = bpf_object__init_prog_names(obj);
1260	1648	return err;
1261	1649	}
1262	1650
@@ -1275,7 +1663,8 @@ bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) · 12 unmodified lines
1275	1663	}
1276	1664
1277	1665	struct bpf_program *
1278		−bpf_object__find_program_by_title(struct bpf_object obj, const char title)
	1666	+bpf_object__find_program_by_title(const struct bpf_object *obj,
	1667	+ const char *title)
1279	1668	{
1280	1669	struct bpf_program *pos;
1281	1670
@@ -1297,7 +1686,8 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj, · 15 unmodified lines
1297	1686	static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
1298	1687	int shndx)
1299	1688	{
1300		− return shndx == obj->efile.maps_shndx;
	1689	+ return shndx == obj->efile.maps_shndx \|\|
	1690	+ shndx == obj->efile.btf_maps_shndx;
1301	1691	}
1302	1692
1303	1693	static bool bpf_object__relo_in_known_section(const struct bpf_object *obj,
@@ -1341,14 +1731,14 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 37 unmodified lines
1341	1731	prog->nr_reloc = nrels;
1342	1732
1343	1733	for (i = 0; i < nrels; i++) {
1344		− GElf_Sym sym;
1345		− GElf_Rel rel;
1346		− unsigned int insn_idx;
1347		− unsigned int shdr_idx;
1348	1734	struct bpf_insn *insns = prog->insns;
1349	1735	enum libbpf_map_type type;
	1736	+ unsigned int insn_idx;
	1737	+ unsigned int shdr_idx;
1350	1738	const char *name;
1351	1739	size_t map_idx;
	1740	+ GElf_Sym sym;
	1741	+ GElf_Rel rel;
1352	1742
1353	1743	if (!gelf_getrel(data, i, &rel)) {
1354	1744	pr_warning("relocation: failed to get %d reloc\n", i);
@@ -1416,9 +1806,13 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 61 unmodified lines
1416	1806	if (maps[map_idx].libbpf_type != type)
1417	1807	continue;
1418	1808	if (type != LIBBPF_MAP_UNSPEC \|\|
1419		− maps[map_idx].offset == sym.st_value) {
1420		− pr_debug("relocation: find map %zd (%s) for insn %u\n",
1421		− map_idx, maps[map_idx].name, insn_idx);
	1809	+ (maps[map_idx].sec_idx == sym.st_shndx &&
	1810	+ maps[map_idx].sec_offset == sym.st_value)) {
	1811	+ pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n",
	1812	+ map_idx, maps[map_idx].name,
	1813	+ maps[map_idx].sec_idx,
	1814	+ maps[map_idx].sec_offset,
	1815	+ insn_idx);
1422	1816	break;
1423	1817	}
1424	1818	}
@@ -1438,14 +1832,18 @@ bpf_program__collect_reloc(struct bpf_program prog, GElf_Shdr shdr, · 13 unmodified lines
1438	1832	return 0;
1439	1833	}
1440	1834
1441		−static int bpf_map_find_btf_info(struct bpf_map map, const struct btf btf)
	1835	+static int bpf_map_find_btf_info(struct bpf_object obj, struct bpf_map map)
1442	1836	{
1443	1837	struct bpf_map_def *def = &map->def;
1444	1838	__u32 key_type_id = 0, value_type_id = 0;
1445	1839	int ret;
1446	1840
	1841	+ /* if it's BTF-defined map, we don't need to search for type IDs */
	1842	+ if (map->sec_idx == obj->efile.btf_maps_shndx)
	1843	+ return 0;
	1844	+
1447	1845	if (!bpf_map__is_internal(map)) {
1448		− ret = btf__get_map_kv_tids(btf, map->name, def->key_size,
	1846	+ ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size,
1449	1847	def->value_size, &key_type_id,
1450	1848	&value_type_id);
1451	1849	} else {
@@ -1453,7 +1851,7 @@ static int bpf_map_find_btf_info(struct bpf_map map, const struct btf btf) · 1 unmodified line
1453	1851	* LLVM annotates global data differently in BTF, that is,
1454	1852	* only as '.data', '.bss' or '.rodata'.
1455	1853	*/
1456		− ret = btf__find_by_name(btf,
	1854	+ ret = btf__find_by_name(obj->btf,
1457	1855	libbpf_type_to_btf_name[map->libbpf_type]);
1458	1856	}
1459	1857	if (ret < 0)
@@ -1740,25 +2138,26 @@ bpf_object__create_maps(struct bpf_object *obj) · 280 unmodified lines
1740	2138	create_attr.key_size = def->key_size;
1741	2139	create_attr.value_size = def->value_size;
1742	2140	create_attr.max_entries = def->max_entries;
1743		− create_attr.btf_fd = -1;
	2141	+ create_attr.btf_fd = 0;
1744	2142	create_attr.btf_key_type_id = 0;
1745	2143	create_attr.btf_value_type_id = 0;
1746	2144	if (bpf_map_type__is_map_in_map(def->type) &&
1747	2145	map->inner_map_fd >= 0)
1748	2146	create_attr.inner_map_fd = map->inner_map_fd;
1749	2147
1750		− if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
	2148	+ if (obj->btf && !bpf_map_find_btf_info(obj, map)) {
1751	2149	create_attr.btf_fd = btf__fd(obj->btf);
1752	2150	create_attr.btf_key_type_id = map->btf_key_type_id;
1753	2151	create_attr.btf_value_type_id = map->btf_value_type_id;
1754	2152	}
1755	2153
1756	2154	*pfd = bpf_create_map_xattr(&create_attr);
1757		− if (*pfd < 0 && create_attr.btf_fd >= 0) {
	2155	+ if (*pfd < 0 && (create_attr.btf_key_type_id \|\|
	2156	+ create_attr.btf_value_type_id)) {
1758	2157	cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1759	2158	pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1760	2159	map->name, cp, errno);
1761		− create_attr.btf_fd = -1;
	2160	+ create_attr.btf_fd = 0;
1762	2161	create_attr.btf_key_type_id = 0;
1763	2162	create_attr.btf_value_type_id = 0;
1764	2163	map->btf_key_type_id = 0;
@@ -2049,7 +2448,7 @@ load_program(struct bpf_program prog, struct bpf_insn insns, int insns_cnt, · 284 unmodified lines
2049	2448	load_attr.license = license;
2050	2449	load_attr.kern_version = kern_version;
2051	2450	load_attr.prog_ifindex = prog->prog_ifindex;
2052		− load_attr.prog_btf_fd = prog->btf_fd;
	2451	+ load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0;
2053	2452	load_attr.func_info = prog->func_info;
2054	2453	load_attr.func_info_rec_size = prog->func_info_rec_size;
2055	2454	load_attr.func_info_cnt = prog->func_info_cnt;
@@ -2195,8 +2594,8 @@ out: · 139 unmodified lines
2195	2594	return err;
2196	2595	}
2197	2596
2198		−static bool bpf_program__is_function_storage(struct bpf_program *prog,
2199		− struct bpf_object *obj)
	2597	+static bool bpf_program__is_function_storage(const struct bpf_program *prog,
	2598	+ const struct bpf_object *obj)
2200	2599	{
2201	2600	return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
2202	2601	}
@@ -2902,17 +3301,17 @@ bpf_object__next(struct bpf_object *prev) · 699 unmodified lines
2902	3301	return next;
2903	3302	}
2904	3303
2905		−const char bpf_object__name(struct bpf_object obj)
	3304	+const char bpf_object__name(const struct bpf_object obj)
2906	3305	{
2907	3306	return obj ? obj->path : ERR_PTR(-EINVAL);
2908	3307	}
2909	3308
2910		−unsigned int bpf_object__kversion(struct bpf_object *obj)
	3309	+unsigned int bpf_object__kversion(const struct bpf_object *obj)
2911	3310	{
2912	3311	return obj ? obj->kern_version : 0;
2913	3312	}
2914	3313
2915		−struct btf bpf_object__btf(struct bpf_object obj)
	3314	+struct btf bpf_object__btf(const struct bpf_object obj)
2916	3315	{
2917	3316	return obj ? obj->btf : NULL;
2918	3317	}
@@ -2933,13 +3332,14 @@ int bpf_object__set_priv(struct bpf_object obj, void priv, · 14 unmodified lines
2933	3332	return 0;
2934	3333	}
2935	3334
2936		−void bpf_object__priv(struct bpf_object obj)
	3335	+void bpf_object__priv(const struct bpf_object obj)
2937	3336	{
2938	3337	return obj ? obj->priv : ERR_PTR(-EINVAL);
2939	3338	}
2940	3339
2941	3340	static struct bpf_program *
2942		−__bpf_program__iter(struct bpf_program p, struct bpf_object obj, bool forward)
	3341	+__bpf_program__iter(const struct bpf_program p, const struct bpf_object obj,
	3342	+ bool forward)
2943	3343	{
2944	3344	size_t nr_programs = obj->nr_programs;
2945	3345	ssize_t idx;
@@ -2964,7 +3364,7 @@ __bpf_program__iter(struct bpf_program p, struct bpf_object obj, bool forward) · 18 unmodified lines
2964	3364	}
2965	3365
2966	3366	struct bpf_program *
2967		−bpf_program__next(struct bpf_program prev, struct bpf_object obj)
	3367	+bpf_program__next(struct bpf_program prev, const struct bpf_object obj)
2968	3368	{
2969	3369	struct bpf_program *prog = prev;
2970	3370
@@ -2976,7 +3376,7 @@ bpf_program__next(struct bpf_program prev, struct bpf_object obj) · 5 unmodified lines
2976	3376	}
2977	3377
2978	3378	struct bpf_program *
2979		−bpf_program__prev(struct bpf_program next, struct bpf_object obj)
	3379	+bpf_program__prev(struct bpf_program next, const struct bpf_object obj)
2980	3380	{
2981	3381	struct bpf_program *prog = next;
2982	3382
@@ -2998,7 +3398,7 @@ int bpf_program__set_priv(struct bpf_program prog, void priv, · 15 unmodified lines
2998	3398	return 0;
2999	3399	}
3000	3400
3001		−void bpf_program__priv(struct bpf_program prog)
	3401	+void bpf_program__priv(const struct bpf_program prog)
3002	3402	{
3003	3403	return prog ? prog->priv : ERR_PTR(-EINVAL);
3004	3404	}
@@ -3008,7 +3408,7 @@ void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) · 3 unmodified lines
3008	3408	prog->prog_ifindex = ifindex;
3009	3409	}
3010	3410
3011		−const char bpf_program__title(struct bpf_program prog, bool needs_copy)
	3411	+const char bpf_program__title(const struct bpf_program prog, bool needs_copy)
3012	3412	{
3013	3413	const char *title;
3014	3414
@@ -3024,7 +3424,7 @@ const char bpf_program__title(struct bpf_program prog, bool needs_copy) · 9 unmodified lines
3024	3424	return title;
3025	3425	}
3026	3426
3027		−int bpf_program__fd(struct bpf_program *prog)
	3427	+int bpf_program__fd(const struct bpf_program *prog)
3028	3428	{
3029	3429	return bpf_program__nth_fd(prog, 0);
3030	3430	}
@@ -3057,7 +3457,7 @@ int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, · 26 unmodified lines
3057	3457	return 0;
3058	3458	}
3059	3459
3060		−int bpf_program__nth_fd(struct bpf_program *prog, int n)
	3460	+int bpf_program__nth_fd(const struct bpf_program *prog, int n)
3061	3461	{
3062	3462	int fd;
3063	3463
@@ -3085,25 +3485,25 @@ void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) · 21 unmodified lines
3085	3485	prog->type = type;
3086	3486	}
3087	3487
3088		−static bool bpf_program__is_type(struct bpf_program *prog,
	3488	+static bool bpf_program__is_type(const struct bpf_program *prog,
3089	3489	enum bpf_prog_type type)
3090	3490	{
3091	3491	return prog ? (prog->type == type) : false;
3092	3492	}
3093	3493
3094		−#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
3095		−int bpf_program__set_##NAME(struct bpf_program *prog) \
3096		−{ \
3097		− if (!prog) \
3098		− return -EINVAL; \
3099		− bpf_program__set_type(prog, TYPE); \
3100		− return 0; \
3101		−} \
3102		− \
3103		−bool bpf_program__is_##NAME(struct bpf_program *prog) \
3104		−{ \
3105		− return bpf_program__is_type(prog, TYPE); \
3106		−} \
	3494	+#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
	3495	+int bpf_program__set_##NAME(struct bpf_program *prog) \
	3496	+{ \
	3497	+ if (!prog) \
	3498	+ return -EINVAL; \
	3499	+ bpf_program__set_type(prog, TYPE); \
	3500	+ return 0; \
	3501	+} \
	3502	+ \
	3503	+bool bpf_program__is_##NAME(const struct bpf_program *prog) \
	3504	+{ \
	3505	+ return bpf_program__is_type(prog, TYPE); \
	3506	+} \
3107	3507
3108	3508	BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
3109	3509	BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
@@ -3302,17 +3702,17 @@ bpf_program__identify_section(struct bpf_program *prog, · 192 unmodified lines
3302	3702	expected_attach_type);
3303	3703	}
3304	3704
3305		−int bpf_map__fd(struct bpf_map *map)
	3705	+int bpf_map__fd(const struct bpf_map *map)
3306	3706	{
3307	3707	return map ? map->fd : -EINVAL;
3308	3708	}
3309	3709
3310		−const struct bpf_map_def bpf_map__def(struct bpf_map map)
	3710	+const struct bpf_map_def bpf_map__def(const struct bpf_map map)
3311	3711	{
3312	3712	return map ? &map->def : ERR_PTR(-EINVAL);
3313	3713	}
3314	3714
3315		−const char bpf_map__name(struct bpf_map map)
	3715	+const char bpf_map__name(const struct bpf_map map)
3316	3716	{
3317	3717	return map ? map->name : NULL;
3318	3718	}
@@ -3343,17 +3743,17 @@ int bpf_map__set_priv(struct bpf_map map, void priv, · 24 unmodified lines
3343	3743	return 0;
3344	3744	}
3345	3745
3346		−void bpf_map__priv(struct bpf_map map)
	3746	+void bpf_map__priv(const struct bpf_map map)
3347	3747	{
3348	3748	return map ? map->priv : ERR_PTR(-EINVAL);
3349	3749	}
3350	3750
3351		−bool bpf_map__is_offload_neutral(struct bpf_map *map)
	3751	+bool bpf_map__is_offload_neutral(const struct bpf_map *map)
3352	3752	{
3353	3753	return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
3354	3754	}
3355	3755
3356		−bool bpf_map__is_internal(struct bpf_map *map)
	3756	+bool bpf_map__is_internal(const struct bpf_map *map)
3357	3757	{
3358	3758	return map->libbpf_type != LIBBPF_MAP_UNSPEC;
3359	3759	}
@@ -3378,7 +3778,7 @@ int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) · 18 unmodified lines
3378	3778	}
3379	3779
3380	3780	static struct bpf_map *
3381		−__bpf_map__iter(struct bpf_map m, struct bpf_object obj, int i)
	3781	+__bpf_map__iter(const struct bpf_map m, const struct bpf_object obj, int i)
3382	3782	{
3383	3783	ssize_t idx;
3384	3784	struct bpf_map s, e;
@@ -3402,7 +3802,7 @@ __bpf_map__iter(struct bpf_map m, struct bpf_object obj, int i) · 17 unmodified lines
3402	3802	}
3403	3803
3404	3804	struct bpf_map *
3405		−bpf_map__next(struct bpf_map prev, struct bpf_object obj)
	3805	+bpf_map__next(const struct bpf_map prev, const struct bpf_object obj)
3406	3806	{
3407	3807	if (prev == NULL)
3408	3808	return obj->maps;
@@ -3411,7 +3811,7 @@ bpf_map__next(struct bpf_map prev, struct bpf_object obj) · 2 unmodified lines
3411	3811	}
3412	3812
3413	3813	struct bpf_map *
3414		−bpf_map__prev(struct bpf_map next, struct bpf_object obj)
	3814	+bpf_map__prev(const struct bpf_map next, const struct bpf_object obj)
3415	3815	{
3416	3816	if (next == NULL) {
3417	3817	if (!obj->nr_maps)
@@ -3423,7 +3823,7 @@ bpf_map__prev(struct bpf_map next, struct bpf_object obj) · 5 unmodified lines
3423	3823	}
3424	3824
3425	3825	struct bpf_map *
3426		−bpf_object__find_map_by_name(struct bpf_object obj, const char name)
	3826	+bpf_object__find_map_by_name(const struct bpf_object obj, const char name)
3427	3827	{
3428	3828	struct bpf_map *pos;
3429	3829
@@ -3435,7 +3835,7 @@ bpf_object__find_map_by_name(struct bpf_object obj, const char name) · 5 unmodified lines
3435	3835	}
3436	3836
3437	3837	int
3438		−bpf_object__find_map_fd_by_name(struct bpf_object obj, const char name)
	3838	+bpf_object__find_map_fd_by_name(const struct bpf_object obj, const char name)
3439	3839	{
3440	3840	return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
3441	3841	}
@@ -3443,13 +3843,7 @@ bpf_object__find_map_fd_by_name(struct bpf_object obj, const char name) · 1 unmodified line
3443	3843	struct bpf_map *
3444	3844	bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
3445	3845	{
3446		− int i;
3447		−
3448		− for (i = 0; i < obj->nr_maps; i++) {
3449		− if (obj->maps[i].offset == offset)
3450		− return &obj->maps[i];
3451		− }
3452		− return ERR_PTR(-ENOENT);
	3846	+ return ERR_PTR(-ENOTSUP);
3453	3847	}
3454	3848
3455	3849	long libbpf_get_error(const void *ptr)
@@ -3835,3 +4229,60 @@ void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) · 379 unmodified lines
3835	4229	desc->array_offset, addr);
3836	4230	}
3837	4231	}
	4232	+
	4233	+int libbpf_num_possible_cpus(void)
	4234	+{
	4235	+ static const char *fcpu = "/sys/devices/system/cpu/possible";
	4236	+ int len = 0, n = 0, il = 0, ir = 0;
	4237	+ unsigned int start = 0, end = 0;
	4238	+ static int cpus;
	4239	+ char buf[128];
	4240	+ int error = 0;
	4241	+ int fd = -1;
	4242	+
	4243	+ if (cpus > 0)
	4244	+ return cpus;
	4245	+
	4246	+ fd = open(fcpu, O_RDONLY);
	4247	+ if (fd < 0) {
	4248	+ error = errno;
	4249	+ pr_warning("Failed to open file %s: %s\n",
	4250	+ fcpu, strerror(error));
	4251	+ return -error;
	4252	+ }
	4253	+ len = read(fd, buf, sizeof(buf));
	4254	+ close(fd);
	4255	+ if (len <= 0) {
	4256	+ error = len ? errno : EINVAL;
	4257	+ pr_warning("Failed to read # of possible cpus from %s: %s\n",
	4258	+ fcpu, strerror(error));
	4259	+ return -error;
	4260	+ }
	4261	+ if (len == sizeof(buf)) {
	4262	+ pr_warning("File %s size overflow\n", fcpu);
	4263	+ return -EOVERFLOW;
	4264	+ }
	4265	+ buf[len] = '\0';
	4266	+
	4267	+ for (ir = 0, cpus = 0; ir <= len; ir++) {
	4268	+ /* Each sub string separated by ',' has format \d+-\d+ or \d+ */
	4269	+ if (buf[ir] == ',' \|\| buf[ir] == '\0') {
	4270	+ buf[ir] = '\0';
	4271	+ n = sscanf(&buf[il], "%u-%u", &start, &end);
	4272	+ if (n <= 0) {
	4273	+ pr_warning("Failed to get # CPUs from %s\n",
	4274	+ &buf[il]);
	4275	+ return -EINVAL;
	4276	+ } else if (n == 1) {
	4277	+ end = start;
	4278	+ }
	4279	+ cpus += end - start + 1;
	4280	+ il = ir + 1;
	4281	+ }
	4282	+ }
	4283	+ if (cpus <= 0) {
	4284	+ pr_warning("Invalid #CPUs %d from %s\n", cpus, fcpu);
	4285	+ return -EINVAL;
	4286	+ }
	4287	+ return cpus;
	4288	+}

tools/lib/bpf/libbpf.h+47−31

@@ -98,15 +98,16 @@ struct bpf_object_load_attr { · 97 unmodified lines
98	98	LIBBPF_API int bpf_object__load(struct bpf_object *obj);
99	99	LIBBPF_API int bpf_object__load_xattr(struct bpf_object_load_attr *attr);
100	100	LIBBPF_API int bpf_object__unload(struct bpf_object *obj);
101		−LIBBPF_API const char bpf_object__name(struct bpf_object obj);
102		−LIBBPF_API unsigned int bpf_object__kversion(struct bpf_object *obj);
	101	+LIBBPF_API const char bpf_object__name(const struct bpf_object obj);
	102	+LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj);
103	103
104	104	struct btf;
105		−LIBBPF_API struct btf bpf_object__btf(struct bpf_object obj);
	105	+LIBBPF_API struct btf bpf_object__btf(const struct bpf_object obj);
106	106	LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj);
107	107
108	108	LIBBPF_API struct bpf_program *
109		−bpf_object__find_program_by_title(struct bpf_object obj, const char title);
	109	+bpf_object__find_program_by_title(const struct bpf_object *obj,
	110	+ const char *title);
110	111
111	112	LIBBPF_API struct bpf_object bpf_object__next(struct bpf_object prev);
112	113	#define bpf_object__for_each_safe(pos, tmp) \
@@ -118,7 +119,7 @@ LIBBPF_API struct bpf_object bpf_object__next(struct bpf_object prev); · 5 unmodified lines
118	119	typedef void (bpf_object_clear_priv_t)(struct bpf_object , void *);
119	120	LIBBPF_API int bpf_object__set_priv(struct bpf_object obj, void priv,
120	121	bpf_object_clear_priv_t clear_priv);
121		−LIBBPF_API void bpf_object__priv(struct bpf_object prog);
	122	+LIBBPF_API void bpf_object__priv(const struct bpf_object prog);
122	123
123	124	LIBBPF_API int
124	125	libbpf_prog_type_by_name(const char name, enum bpf_prog_type prog_type,
@@ -129,7 +130,7 @@ LIBBPF_API int libbpf_attach_type_by_name(const char *name, · 4 unmodified lines
129	130	/* Accessors of bpf_program */
130	131	struct bpf_program;
131	132	LIBBPF_API struct bpf_program bpf_program__next(struct bpf_program prog,
132		− struct bpf_object *obj);
	133	+ const struct bpf_object *obj);
133	134
134	135	#define bpf_object__for_each_program(pos, obj) \
135	136	for ((pos) = bpf_program__next(NULL, (obj)); \
@@ -137,24 +138,23 @@ LIBBPF_API struct bpf_program bpf_program__next(struct bpf_program prog, · 1 unmodified line
137	138	(pos) = bpf_program__next((pos), (obj)))
138	139
139	140	LIBBPF_API struct bpf_program bpf_program__prev(struct bpf_program prog,
140		− struct bpf_object *obj);
	141	+ const struct bpf_object *obj);
141	142
142		−typedef void (bpf_program_clear_priv_t)(struct bpf_program ,
143		− void *);
	143	+typedef void (bpf_program_clear_priv_t)(struct bpf_program , void *);
144	144
145	145	LIBBPF_API int bpf_program__set_priv(struct bpf_program prog, void priv,
146	146	bpf_program_clear_priv_t clear_priv);
147	147
148		−LIBBPF_API void bpf_program__priv(struct bpf_program prog);
	148	+LIBBPF_API void bpf_program__priv(const struct bpf_program prog);
149	149	LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog,
150	150	__u32 ifindex);
151	151
152		−LIBBPF_API const char bpf_program__title(struct bpf_program prog,
	152	+LIBBPF_API const char bpf_program__title(const struct bpf_program prog,
153	153	bool needs_copy);
154	154
155	155	LIBBPF_API int bpf_program__load(struct bpf_program prog, char license,
156	156	__u32 kern_version);
157		−LIBBPF_API int bpf_program__fd(struct bpf_program *prog);
	157	+LIBBPF_API int bpf_program__fd(const struct bpf_program *prog);
158	158	LIBBPF_API int bpf_program__pin_instance(struct bpf_program *prog,
159	159	const char *path,
160	160	int instance);
@@ -227,7 +227,7 @@ typedef int (bpf_program_prep_t)(struct bpf_program prog, int n, · 66 unmodified lines
227	227	LIBBPF_API int bpf_program__set_prep(struct bpf_program *prog, int nr_instance,
228	228	bpf_program_prep_t prep);
229	229
230		−LIBBPF_API int bpf_program__nth_fd(struct bpf_program *prog, int n);
	230	+LIBBPF_API int bpf_program__nth_fd(const struct bpf_program *prog, int n);
231	231
232	232	/*
233	233	* Adjust type of BPF program. Default is kprobe.
@@ -246,14 +246,14 @@ LIBBPF_API void · 12 unmodified lines
246	246	bpf_program__set_expected_attach_type(struct bpf_program *prog,
247	247	enum bpf_attach_type type);
248	248
249		−LIBBPF_API bool bpf_program__is_socket_filter(struct bpf_program *prog);
250		−LIBBPF_API bool bpf_program__is_tracepoint(struct bpf_program *prog);
251		−LIBBPF_API bool bpf_program__is_raw_tracepoint(struct bpf_program *prog);
252		−LIBBPF_API bool bpf_program__is_kprobe(struct bpf_program *prog);
253		−LIBBPF_API bool bpf_program__is_sched_cls(struct bpf_program *prog);
254		−LIBBPF_API bool bpf_program__is_sched_act(struct bpf_program *prog);
255		−LIBBPF_API bool bpf_program__is_xdp(struct bpf_program *prog);
256		−LIBBPF_API bool bpf_program__is_perf_event(struct bpf_program *prog);
	249	+LIBBPF_API bool bpf_program__is_socket_filter(const struct bpf_program *prog);
	250	+LIBBPF_API bool bpf_program__is_tracepoint(const struct bpf_program *prog);
	251	+LIBBPF_API bool bpf_program__is_raw_tracepoint(const struct bpf_program *prog);
	252	+LIBBPF_API bool bpf_program__is_kprobe(const struct bpf_program *prog);
	253	+LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog);
	254	+LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog);
	255	+LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog);
	256	+LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog);
257	257
258	258	/*
259	259	* No need for __attribute__((packed)), all members of 'bpf_map_def'
@@ -275,10 +275,10 @@ struct bpf_map_def { · 15 unmodified lines
275	275	*/
276	276	struct bpf_map;
277	277	LIBBPF_API struct bpf_map *
278		−bpf_object__find_map_by_name(struct bpf_object obj, const char name);
	278	+bpf_object__find_map_by_name(const struct bpf_object obj, const char name);
279	279
280	280	LIBBPF_API int
281		−bpf_object__find_map_fd_by_name(struct bpf_object obj, const char name);
	281	+bpf_object__find_map_fd_by_name(const struct bpf_object obj, const char name);
282	282
283	283	/*
284	284	* Get bpf_map through the offset of corresponding struct bpf_map_def
@@ -288,7 +288,7 @@ LIBBPF_API struct bpf_map * · 3 unmodified lines
288	288	bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset);
289	289
290	290	LIBBPF_API struct bpf_map *
291		−bpf_map__next(struct bpf_map map, struct bpf_object obj);
	291	+bpf_map__next(const struct bpf_map map, const struct bpf_object obj);
292	292	#define bpf_object__for_each_map(pos, obj) \
293	293	for ((pos) = bpf_map__next(NULL, (obj)); \
294	294	(pos) != NULL; \
@@ -296,22 +296,22 @@ bpf_map__next(struct bpf_map map, struct bpf_object obj); · 1 unmodified line
296	296	#define bpf_map__for_each bpf_object__for_each_map
297	297
298	298	LIBBPF_API struct bpf_map *
299		−bpf_map__prev(struct bpf_map map, struct bpf_object obj);
	299	+bpf_map__prev(const struct bpf_map map, const struct bpf_object obj);
300	300
301		−LIBBPF_API int bpf_map__fd(struct bpf_map *map);
302		−LIBBPF_API const struct bpf_map_def bpf_map__def(struct bpf_map map);
303		−LIBBPF_API const char bpf_map__name(struct bpf_map map);
	301	+LIBBPF_API int bpf_map__fd(const struct bpf_map *map);
	302	+LIBBPF_API const struct bpf_map_def bpf_map__def(const struct bpf_map map);
	303	+LIBBPF_API const char bpf_map__name(const struct bpf_map map);
304	304	LIBBPF_API __u32 bpf_map__btf_key_type_id(const struct bpf_map *map);
305	305	LIBBPF_API __u32 bpf_map__btf_value_type_id(const struct bpf_map *map);
306	306
307	307	typedef void (bpf_map_clear_priv_t)(struct bpf_map , void *);
308	308	LIBBPF_API int bpf_map__set_priv(struct bpf_map map, void priv,
309	309	bpf_map_clear_priv_t clear_priv);
310		−LIBBPF_API void bpf_map__priv(struct bpf_map map);
	310	+LIBBPF_API void bpf_map__priv(const struct bpf_map map);
311	311	LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd);
312	312	LIBBPF_API int bpf_map__resize(struct bpf_map *map, __u32 max_entries);
313		−LIBBPF_API bool bpf_map__is_offload_neutral(struct bpf_map *map);
314		−LIBBPF_API bool bpf_map__is_internal(struct bpf_map *map);
	313	+LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map);
	314	+LIBBPF_API bool bpf_map__is_internal(const struct bpf_map *map);
315	315	LIBBPF_API void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex);
316	316	LIBBPF_API int bpf_map__pin(struct bpf_map map, const char path);
317	317	LIBBPF_API int bpf_map__unpin(struct bpf_map map, const char path);
@@ -454,6 +454,22 @@ bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear); · 136 unmodified lines
454	454	LIBBPF_API void
455	455	bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear);
456	456
	457	+/*
	458	+ * A helper function to get the number of possible CPUs before looking up
	459	+ * per-CPU maps. Negative errno is returned on failure.
	460	+ *
	461	+ * Example usage:
	462	+ *
	463	+ * int ncpus = libbpf_num_possible_cpus();
	464	+ * if (ncpus < 0) {
	465	+ * // error handling
	466	+ * }
	467	+ * long values[ncpus];
	468	+ * bpf_map_lookup_elem(per_cpu_map_fd, key, values);
	469	+ *
	470	+ */
	471	+LIBBPF_API int libbpf_num_possible_cpus(void);
	472	+
457	473	#ifdef __cplusplus
458	474	} /* extern "C" */
459	475	#endif

tools/lib/bpf/libbpf.map+1−0

@@ -172,4 +172,5 @@ LIBBPF_0.0.4 { · 171 unmodified lines
172	172	btf_dump__new;
173	173	btf__parse_elf;
174	174	bpf_object__load_xattr;
	175	+ libbpf_num_possible_cpus;
175	176	} LIBBPF_0.0.3;

tools/lib/bpf/libbpf_internal.h+7−0

@@ -23,6 +23,13 @@ · 22 unmodified lines
23	23	#define BTF_PARAM_ENC(name, type) (name), (type)
24	24	#define BTF_VAR_SECINFO_ENC(type, offset, size) (type), (offset), (size)
25	25
	26	+#ifndef min
	27	+# define min(x, y) ((x) < (y) ? (x) : (y))
	28	+#endif
	29	+#ifndef max
	30	+# define max(x, y) ((x) < (y) ? (y) : (x))
	31	+#endif
	32	+
26	33	extern void libbpf_print(enum libbpf_print_level level,
27	34	const char *format, ...)
28	35	__attribute__((format(printf, 2, 3)));

tools/lib/bpf/xsk.c+28−75

@@ -60,10 +60,8 @@ struct xsk_socket { · 59 unmodified lines
60	60	struct xsk_umem *umem;
61	61	struct xsk_socket_config config;
62	62	int fd;
63		− int xsks_map;
64	63	int ifindex;
65	64	int prog_fd;
66		− int qidconf_map_fd;
67	65	int xsks_map_fd;
68	66	__u32 queue_id;
69	67	char ifname[IFNAMSIZ];
@@ -265,15 +263,11 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 195 unmodified lines
265	263	/* This is the C-program:
266	264	* SEC("xdp_sock") int xdp_sock_prog(struct xdp_md *ctx)
267	265	* {
268		− * int *qidconf, index = ctx->rx_queue_index;
	266	+ * int index = ctx->rx_queue_index;
269	267	*
270	268	* // A set entry here means that the correspnding queue_id
271	269	* // has an active AF_XDP socket bound to it.
272		− * qidconf = bpf_map_lookup_elem(&qidconf_map, &index);
273		− * if (!qidconf)
274		− * return XDP_ABORTED;
275		− *
276		− * if (*qidconf)
	270	+ * if (bpf_map_lookup_elem(&xsks_map, &index))
277	271	* return bpf_redirect_map(&xsks_map, index, 0);
278	272	*
279	273	* return XDP_PASS;
@@ -286,15 +280,10 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) · 6 unmodified lines
286	280	BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_1, -4),
287	281	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
288	282	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
289		− BPF_LD_MAP_FD(BPF_REG_1, xsk->qidconf_map_fd),
	283	+ BPF_LD_MAP_FD(BPF_REG_1, xsk->xsks_map_fd),
290	284	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
291	285	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
292		− BPF_MOV32_IMM(BPF_REG_0, 0),
293		− /* if r1 == 0 goto +8 */
294		− BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 8),
295	286	BPF_MOV32_IMM(BPF_REG_0, 2),
296		− /* r1 = (u32 )(r1 + 0) */
297		− BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, 0),
298	287	/* if r1 == 0 goto +5 */
299	288	BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, 5),
300	289	/* r2 = (u32 )(r10 - 4) */
@@ -366,18 +355,11 @@ static int xsk_create_bpf_maps(struct xsk_socket *xsk) · 65 unmodified lines
366	355	if (max_queues < 0)
367	356	return max_queues;
368	357
369		− fd = bpf_create_map_name(BPF_MAP_TYPE_ARRAY, "qidconf_map",
	358	+ fd = bpf_create_map_name(BPF_MAP_TYPE_XSKMAP, "xsks_map",
370	359	sizeof(int), sizeof(int), max_queues, 0);
371	360	if (fd < 0)
372	361	return fd;
373		− xsk->qidconf_map_fd = fd;
374	362
375		− fd = bpf_create_map_name(BPF_MAP_TYPE_XSKMAP, "xsks_map",
376		− sizeof(int), sizeof(int), max_queues, 0);
377		− if (fd < 0) {
378		− close(xsk->qidconf_map_fd);
379		− return fd;
380		− }
381	363	xsk->xsks_map_fd = fd;
382	364
383	365	return 0;
@@ -385,10 +367,8 @@ static int xsk_create_bpf_maps(struct xsk_socket *xsk) · 1 unmodified line
385	367
386	368	static void xsk_delete_bpf_maps(struct xsk_socket *xsk)
387	369	{
388		− close(xsk->qidconf_map_fd);
	370	+ bpf_map_delete_elem(xsk->xsks_map_fd, &xsk->queue_id);
389	371	close(xsk->xsks_map_fd);
390		− xsk->qidconf_map_fd = -1;
391		− xsk->xsks_map_fd = -1;
392	372	}
393	373
394	374	static int xsk_lookup_bpf_maps(struct xsk_socket *xsk)
@@ -417,10 +397,9 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) · 22 unmodified lines
417	397	if (err)
418	398	goto out_map_ids;
419	399
420		− for (i = 0; i < prog_info.nr_map_ids; i++) {
421		− if (xsk->qidconf_map_fd != -1 && xsk->xsks_map_fd != -1)
422		− break;
	400	+ xsk->xsks_map_fd = -1;
423	401
	402	+ for (i = 0; i < prog_info.nr_map_ids; i++) {
424	403	fd = bpf_map_get_fd_by_id(map_ids[i]);
425	404	if (fd < 0)
426	405	continue;
@@ -431,11 +410,6 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) · 4 unmodified lines
431	410	continue;
432	411	}
433	412
434		− if (!strcmp(map_info.name, "qidconf_map")) {
435		− xsk->qidconf_map_fd = fd;
436		− continue;
437		− }
438		−
439	413	if (!strcmp(map_info.name, "xsks_map")) {
440	414	xsk->xsks_map_fd = fd;
441	415	continue;
@@ -445,40 +419,18 @@ static int xsk_lookup_bpf_maps(struct xsk_socket *xsk) · 3 unmodified lines
445	419	}
446	420
447	421	err = 0;
448		− if (xsk->qidconf_map_fd < 0 \|\| xsk->xsks_map_fd < 0) {
	422	+ if (xsk->xsks_map_fd == -1)
449	423	err = -ENOENT;
450		− xsk_delete_bpf_maps(xsk);
451		− }
452	424
453	425	out_map_ids:
454	426	free(map_ids);
455	427	return err;
456	428	}
457	429
458		−static void xsk_clear_bpf_maps(struct xsk_socket *xsk)
459		−{
460		− int qid = false;
461		−
462		− bpf_map_update_elem(xsk->qidconf_map_fd, &xsk->queue_id, &qid, 0);
463		− bpf_map_delete_elem(xsk->xsks_map_fd, &xsk->queue_id);
464		−}
465		−
466	430	static int xsk_set_bpf_maps(struct xsk_socket *xsk)
467	431	{
468		− int qid = true, fd = xsk->fd, err;
469		−
470		− err = bpf_map_update_elem(xsk->qidconf_map_fd, &xsk->queue_id, &qid, 0);
471		− if (err)
472		− goto out;
473		−
474		− err = bpf_map_update_elem(xsk->xsks_map_fd, &xsk->queue_id, &fd, 0);
475		− if (err)
476		− goto out;
477		−
478		− return 0;
479		−out:
480		− xsk_clear_bpf_maps(xsk);
481		− return err;
	432	+ return bpf_map_update_elem(xsk->xsks_map_fd, &xsk->queue_id,
	433	+ &xsk->fd, 0);
482	434	}
483	435
484	436	static int xsk_setup_xdp_prog(struct xsk_socket *xsk)
@@ -497,26 +449,27 @@ static int xsk_setup_xdp_prog(struct xsk_socket *xsk) · 12 unmodified lines
497	449	return err;
498	450
499	451	err = xsk_load_xdp_prog(xsk);
500		− if (err)
501		− goto out_maps;
	452	+ if (err) {
	453	+ xsk_delete_bpf_maps(xsk);
	454	+ return err;
	455	+ }
502	456	} else {
503	457	xsk->prog_fd = bpf_prog_get_fd_by_id(prog_id);
504	458	err = xsk_lookup_bpf_maps(xsk);
505		− if (err)
506		− goto out_load;
	459	+ if (err) {
	460	+ close(xsk->prog_fd);
	461	+ return err;
	462	+ }
507	463	}
508	464
509	465	err = xsk_set_bpf_maps(xsk);
510		− if (err)
511		− goto out_load;
	466	+ if (err) {
	467	+ xsk_delete_bpf_maps(xsk);
	468	+ close(xsk->prog_fd);
	469	+ return err;
	470	+ }
512	471
513	472	return 0;
514		−
515		−out_load:
516		− close(xsk->prog_fd);
517		−out_maps:
518		− xsk_delete_bpf_maps(xsk);
519		− return err;
520	473	}
521	474
522	475	int xsk_socket__create(struct xsk_socket *xsk_ptr, const char ifname,
@@ -643,9 +596,7 @@ int xsk_socket__create(struct xsk_socket *xsk_ptr, const char ifname, · 120 unmodified lines
643	596	goto out_mmap_tx;
644	597	}
645	598
646		− xsk->qidconf_map_fd = -1;
647		− xsk->xsks_map_fd = -1;
648		−
	599	+ xsk->prog_fd = -1;
649	600	if (!(xsk->config.libbpf_flags & XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD)) {
650	601	err = xsk_setup_xdp_prog(xsk);
651	602	if (err)
@@ -708,8 +659,10 @@ void xsk_socket__delete(struct xsk_socket *xsk) · 56 unmodified lines
708	659	if (!xsk)
709	660	return;
710	661
711		− xsk_clear_bpf_maps(xsk);
712		− xsk_delete_bpf_maps(xsk);
	662	+ if (xsk->prog_fd != -1) {
	663	+ xsk_delete_bpf_maps(xsk);
	664	+ close(xsk->prog_fd);
	665	+ }
713	666
714	667	optlen = sizeof(off);
715	668	err = getsockopt(xsk->fd, SOL_XDP, XDP_MMAP_OFFSETS, &off, &optlen);

tools/testing/selftests/bpf/Makefile+2−1

@@ -280,4 +280,5 @@ $(OUTPUT)/verifier/tests.h: $(VERIFIER_TESTS_DIR) $(VERIFIER_TEST_FILES) · 279 unmodified lines
280	280	) > $(VERIFIER_TESTS_H))
281	281
282	282	EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(ALU32_BUILD_DIR) \
283		− $(VERIFIER_TESTS_H) $(PROG_TESTS_H) $(MAP_TESTS_H)
	283	+ $(VERIFIER_TESTS_H) $(PROG_TESTS_H) $(MAP_TESTS_H) \
	284	+ feature

tools/testing/selftests/bpf/bpf_endian.h+1−0

@@ -2,6 +2,7 @@ · 1 unmodified line
2	2	#ifndef __BPF_ENDIAN__
3	3	#define __BPF_ENDIAN__
4	4
	5	+#include <linux/stddef.h>
5	6	#include <linux/swab.h>
6	7
7	8	/* LLVM's BPF target selects the endianness of the CPU

tools/testing/selftests/bpf/bpf_helpers.h+2−2

@@ -31,7 +31,7 @@ static int (bpf_map_pop_elem)(void map, void *value) = · 30 unmodified lines
31	31	(void *) BPF_FUNC_map_pop_elem;
32	32	static int (bpf_map_peek_elem)(void map, void *value) =
33	33	(void *) BPF_FUNC_map_peek_elem;
34		−static int (bpf_probe_read)(void dst, int size, void *unsafe_ptr) =
	34	+static int (bpf_probe_read)(void dst, int size, const void *unsafe_ptr) =
35	35	(void *) BPF_FUNC_probe_read;
36	36	static unsigned long long (*bpf_ktime_get_ns)(void) =
37	37	(void *) BPF_FUNC_ktime_get_ns;
@@ -62,7 +62,7 @@ static int (bpf_perf_event_output)(void ctx, void *map, · 24 unmodified lines
62	62	(void *) BPF_FUNC_perf_event_output;
63	63	static int (bpf_get_stackid)(void ctx, void *map, int flags) =
64	64	(void *) BPF_FUNC_get_stackid;
65		−static int (bpf_probe_write_user)(void dst, void *src, int size) =
	65	+static int (bpf_probe_write_user)(void dst, const void *src, int size) =
66	66	(void *) BPF_FUNC_probe_write_user;
67	67	static int (bpf_current_task_under_cgroup)(void map, int index) =
68	68	(void *) BPF_FUNC_current_task_under_cgroup;

tools/testing/selftests/bpf/bpf_util.h+5−32

@@ -6,44 +6,17 @@ · 5 unmodified lines
6	6	#include <stdlib.h>
7	7	#include <string.h>
8	8	#include <errno.h>
	9	+#include <libbpf.h> /* libbpf_num_possible_cpus */
9	10
10	11	static inline unsigned int bpf_num_possible_cpus(void)
11	12	{
12		− static const char *fcpu = "/sys/devices/system/cpu/possible";
13		− unsigned int start, end, possible_cpus = 0;
14		− char buff[128];
15		− FILE *fp;
16		− int len, n, i, j = 0;
	13	+ int possible_cpus = libbpf_num_possible_cpus();
17	14
18		− fp = fopen(fcpu, "r");
19		− if (!fp) {
20		− printf("Failed to open %s: '%s'!\n", fcpu, strerror(errno));
	15	+ if (possible_cpus < 0) {
	16	+ printf("Failed to get # of possible cpus: '%s'!\n",
	17	+ strerror(-possible_cpus));
21	18	exit(1);
22	19	}
23		−
24		− if (!fgets(buff, sizeof(buff), fp)) {
25		− printf("Failed to read %s!\n", fcpu);
26		− exit(1);
27		− }
28		−
29		− len = strlen(buff);
30		− for (i = 0; i <= len; i++) {
31		− if (buff[i] == ',' \|\| buff[i] == '\0') {
32		− buff[i] = '\0';
33		− n = sscanf(&buff[j], "%u-%u", &start, &end);
34		− if (n <= 0) {
35		− printf("Failed to retrieve # possible CPUs!\n");
36		− exit(1);
37		− } else if (n == 1) {
38		− end = start;
39		− }
40		− possible_cpus += end - start + 1;
41		− j = i + 1;
42		− }
43		− }
44		−
45		− fclose(fp);
46		−
47	20	return possible_cpus;
48	21	}
49	22

tools/testing/selftests/bpf/cgroup_helpers.c+1−1

@@ -47,7 +47,7 @@ int enable_all_controllers(char *cgroup_path) · 46 unmodified lines
47	47	char buf[PATH_MAX];
48	48	char c, c2;
49	49	int fd, cfd;
50		− size_t len;
	50	+ ssize_t len;
51	51
52	52	snprintf(path, sizeof(path), "%s/cgroup.controllers", cgroup_path);
53	53	fd = open(path, O_RDONLY);

tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c+56−11

@@ -5,7 +5,7 @@ static int libbpf_debug_print(enum libbpf_print_level level, · 4 unmodified lines
5	5	const char *format, va_list args)
6	6	{
7	7	if (level != LIBBPF_DEBUG)
8		− return 0;
	8	+ return vfprintf(stderr, format, args);
9	9
10	10	if (!strstr(format, "verifier log"))
11	11	return 0;
@@ -32,24 +32,69 @@ static int check_load(const char *file, enum bpf_prog_type type) · 20 unmodified lines
32	32
33	33	void test_bpf_verif_scale(void)
34	34	{
35		− const char *scale[] = {
36		− "./test_verif_scale1.o", "./test_verif_scale2.o", "./test_verif_scale3.o"
	35	+ const char *sched_cls[] = {
	36	+ "./test_verif_scale1.o", "./test_verif_scale2.o", "./test_verif_scale3.o",
37	37	};
38		− const char *pyperf[] = {
39		− "./pyperf50.o", "./pyperf100.o", "./pyperf180.o"
	38	+ const char *raw_tp[] = {
	39	+ /* full unroll by llvm */
	40	+ "./pyperf50.o", "./pyperf100.o", "./pyperf180.o",
	41	+
	42	+ /* partial unroll. llvm will unroll loop ~150 times.
	43	+ * C loop count -> 600.
	44	+ * Asm loop count -> 4.
	45	+ * 16k insns in loop body.
	46	+ * Total of 5 such loops. Total program size ~82k insns.
	47	+ */
	48	+ "./pyperf600.o",
	49	+
	50	+ /* no unroll at all.
	51	+ * C loop count -> 600.
	52	+ * ASM loop count -> 600.
	53	+ * ~110 insns in loop body.
	54	+ * Total of 5 such loops. Total program size ~1500 insns.
	55	+ */
	56	+ "./pyperf600_nounroll.o",
	57	+
	58	+ "./loop1.o", "./loop2.o",
	59	+
	60	+ /* partial unroll. 19k insn in a loop.
	61	+ * Total program size 20.8k insn.
	62	+ * ~350k processed_insns
	63	+ */
	64	+ "./strobemeta.o",
	65	+
	66	+ /* no unroll, tiny loops */
	67	+ "./strobemeta_nounroll1.o",
	68	+ "./strobemeta_nounroll2.o",
	69	+ };
	70	+ const char *cg_sysctl[] = {
	71	+ "./test_sysctl_loop1.o", "./test_sysctl_loop2.o",
40	72	};
41	73	int err, i;
42	74
43	75	if (verifier_stats)
44	76	libbpf_set_print(libbpf_debug_print);
45	77
46		− for (i = 0; i < ARRAY_SIZE(scale); i++) {
47		− err = check_load(scale[i], BPF_PROG_TYPE_SCHED_CLS);
48		− printf("test_scale:%s:%s\n", scale[i], err ? "FAIL" : "OK");
	78	+ err = check_load("./loop3.o", BPF_PROG_TYPE_RAW_TRACEPOINT);
	79	+ printf("test_scale:loop3:%s\n", err ? (error_cnt--, "OK") : "FAIL");
	80	+
	81	+ for (i = 0; i < ARRAY_SIZE(sched_cls); i++) {
	82	+ err = check_load(sched_cls[i], BPF_PROG_TYPE_SCHED_CLS);
	83	+ printf("test_scale:%s:%s\n", sched_cls[i], err ? "FAIL" : "OK");
49	84	}
50	85
51		− for (i = 0; i < ARRAY_SIZE(pyperf); i++) {
52		− err = check_load(pyperf[i], BPF_PROG_TYPE_RAW_TRACEPOINT);
53		− printf("test_scale:%s:%s\n", pyperf[i], err ? "FAIL" : "OK");
	86	+ for (i = 0; i < ARRAY_SIZE(raw_tp); i++) {
	87	+ err = check_load(raw_tp[i], BPF_PROG_TYPE_RAW_TRACEPOINT);
	88	+ printf("test_scale:%s:%s\n", raw_tp[i], err ? "FAIL" : "OK");
54	89	}
	90	+
	91	+ for (i = 0; i < ARRAY_SIZE(cg_sysctl); i++) {
	92	+ err = check_load(cg_sysctl[i], BPF_PROG_TYPE_CGROUP_SYSCTL);
	93	+ printf("test_scale:%s:%s\n", cg_sysctl[i], err ? "FAIL" : "OK");
	94	+ }
	95	+ err = check_load("./test_xdp_loop.o", BPF_PROG_TYPE_XDP);
	96	+ printf("test_scale:test_xdp_loop:%s\n", err ? "FAIL" : "OK");
	97	+
	98	+ err = check_load("./test_seg6_loop.o", BPF_PROG_TYPE_LWT_SEG6LOCAL);
	99	+ printf("test_scale:test_seg6_loop:%s\n", err ? "FAIL" : "OK");
55	100	}

tools/testing/selftests/bpf/progs/bpf_flow.c+13−5

@@ -57,17 +57,25 @@ struct frag_hdr { · 56 unmodified lines
57	57	__be32 identification;
58	58	};
59	59
60		−struct bpf_map_def SEC("maps") jmp_table = {
	60	+struct {
	61	+ __u32 type;
	62	+ __u32 max_entries;
	63	+ __u32 key_size;
	64	+ __u32 value_size;
	65	+} jmp_table SEC(".maps") = {
61	66	.type = BPF_MAP_TYPE_PROG_ARRAY,
	67	+ .max_entries = 8,
62	68	.key_size = sizeof(__u32),
63	69	.value_size = sizeof(__u32),
64		− .max_entries = 8
65	70	};
66	71
67		−struct bpf_map_def SEC("maps") last_dissection = {
	72	+struct {
	73	+ __u32 type;
	74	+ __u32 max_entries;
	75	+ __u32 *key;
	76	+ struct bpf_flow_keys *value;
	77	+} last_dissection SEC(".maps") = {
68	78	.type = BPF_MAP_TYPE_ARRAY,
69		− .key_size = sizeof(__u32),
70		− .value_size = sizeof(struct bpf_flow_keys),
71	79	.max_entries = 1,
72	80	};
73	81

tools/testing/selftests/bpf/progs/loop1.c+28−0

@@ -0,0 +1,28 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+#include <linux/sched.h>
	4	+#include <linux/ptrace.h>
	5	+#include <stdint.h>
	6	+#include <stddef.h>
	7	+#include <stdbool.h>
	8	+#include <linux/bpf.h>
	9	+#include "bpf_helpers.h"
	10	+
	11	+char _license[] SEC("license") = "GPL";
	12	+
	13	+SEC("raw_tracepoint/kfree_skb")
	14	+int nested_loops(volatile struct pt_regs* ctx)
	15	+{
	16	+ int i, j, sum = 0, m;
	17	+
	18	+ for (j = 0; j < 300; j++)
	19	+ for (i = 0; i < j; i++) {
	20	+ if (j & 1)
	21	+ m = ctx->rax;
	22	+ else
	23	+ m = j;
	24	+ sum += i * m;
	25	+ }
	26	+
	27	+ return sum;
	28	+}

tools/testing/selftests/bpf/progs/loop2.c+28−0

@@ -0,0 +1,28 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+#include <linux/sched.h>
	4	+#include <linux/ptrace.h>
	5	+#include <stdint.h>
	6	+#include <stddef.h>
	7	+#include <stdbool.h>
	8	+#include <linux/bpf.h>
	9	+#include "bpf_helpers.h"
	10	+
	11	+char _license[] SEC("license") = "GPL";
	12	+
	13	+SEC("raw_tracepoint/consume_skb")
	14	+int while_true(volatile struct pt_regs* ctx)
	15	+{
	16	+ int i = 0;
	17	+
	18	+ while (true) {
	19	+ if (ctx->rax & 1)
	20	+ i += 3;
	21	+ else
	22	+ i += 7;
	23	+ if (i > 40)
	24	+ break;
	25	+ }
	26	+
	27	+ return i;
	28	+}

tools/testing/selftests/bpf/progs/loop3.c+22−0

@@ -0,0 +1,22 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+#include <linux/sched.h>
	4	+#include <linux/ptrace.h>
	5	+#include <stdint.h>
	6	+#include <stddef.h>
	7	+#include <stdbool.h>
	8	+#include <linux/bpf.h>
	9	+#include "bpf_helpers.h"
	10	+
	11	+char _license[] SEC("license") = "GPL";
	12	+
	13	+SEC("raw_tracepoint/consume_skb")
	14	+int while_true(volatile struct pt_regs* ctx)
	15	+{
	16	+ __u64 i = 0, sum = 0;
	17	+ do {
	18	+ i++;
	19	+ sum += ctx->rax;
	20	+ } while (i < 0x100000000ULL);
	21	+ return sum;
	22	+}

tools/testing/selftests/bpf/progs/netcnt_prog.c+10−12

@@ -10,24 +10,22 @@ · 9 unmodified lines
10	10	#define REFRESH_TIME_NS 100000000
11	11	#define NS_PER_SEC 1000000000
12	12
13		−struct bpf_map_def SEC("maps") percpu_netcnt = {
	13	+struct {
	14	+ __u32 type;
	15	+ struct bpf_cgroup_storage_key *key;
	16	+ struct percpu_net_cnt *value;
	17	+} percpu_netcnt SEC(".maps") = {
14	18	.type = BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,
15		− .key_size = sizeof(struct bpf_cgroup_storage_key),
16		− .value_size = sizeof(struct percpu_net_cnt),
17	19	};
18	20
19		−BPF_ANNOTATE_KV_PAIR(percpu_netcnt, struct bpf_cgroup_storage_key,
20		− struct percpu_net_cnt);
21		−
22		−struct bpf_map_def SEC("maps") netcnt = {
	21	+struct {
	22	+ __u32 type;
	23	+ struct bpf_cgroup_storage_key *key;
	24	+ struct net_cnt *value;
	25	+} netcnt SEC(".maps") = {
23	26	.type = BPF_MAP_TYPE_CGROUP_STORAGE,
24		− .key_size = sizeof(struct bpf_cgroup_storage_key),
25		− .value_size = sizeof(struct net_cnt),
26	27	};
27	28
28		−BPF_ANNOTATE_KV_PAIR(netcnt, struct bpf_cgroup_storage_key,
29		− struct net_cnt);
30		−
31	29	SEC("cgroup/skb")
32	30	int bpf_nextcnt(struct __sk_buff *skb)
33	31	{

tools/testing/selftests/bpf/progs/pyperf.h+5−1

@@ -220,7 +220,11 @@ static inline __attribute__((__always_inline__)) int __on_event(struct pt_regs * · 219 unmodified lines
220	220	int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym);
221	221	if (symbol_counter == NULL)
222	222	return 0;
223		−#pragma unroll
	223	+#ifdef NO_UNROLL
	224	+#pragma clang loop unroll(disable)
	225	+#else
	226	+#pragma clang loop unroll(full)
	227	+#endif
224	228	/* Unwind python stack */
225	229	for (int i = 0; i < STACK_MAX_LEN; ++i) {
226	230	if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) {

tools/testing/selftests/bpf/progs/pyperf600.c+9−0

@@ -0,0 +1,9 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+#define STACK_MAX_LEN 600
	4	+/* clang will not unroll the loop 600 times.
	5	+ * Instead it will unroll it to the amount it deemed
	6	+ * appropriate, but the loop will still execute 600 times.
	7	+ * Total program size is around 90k insns
	8	+ */
	9	+#include "pyperf.h"

tools/testing/selftests/bpf/progs/pyperf600_nounroll.c+8−0

@@ -0,0 +1,8 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+#define STACK_MAX_LEN 600
	4	+#define NO_UNROLL
	5	+/* clang will not unroll at all.
	6	+ * Total program size is around 2k insns
	7	+ */
	8	+#include "pyperf.h"

tools/testing/selftests/bpf/progs/socket_cookie_prog.c+31−18

@@ -7,25 +7,36 @@ · 6 unmodified lines
7	7	#include "bpf_helpers.h"
8	8	#include "bpf_endian.h"
9	9
10		−struct bpf_map_def SEC("maps") socket_cookies = {
11		− .type = BPF_MAP_TYPE_HASH,
12		− .key_size = sizeof(__u64),
13		− .value_size = sizeof(__u32),
14		− .max_entries = 1 << 8,
	10	+struct socket_cookie {
	11	+ __u64 cookie_key;
	12	+ __u32 cookie_value;
	13	+};
	14	+
	15	+struct {
	16	+ __u32 type;
	17	+ __u32 map_flags;
	18	+ int *key;
	19	+ struct socket_cookie *value;
	20	+} socket_cookies SEC(".maps") = {
	21	+ .type = BPF_MAP_TYPE_SK_STORAGE,
	22	+ .map_flags = BPF_F_NO_PREALLOC,
15	23	};
16	24
17	25	SEC("cgroup/connect6")
18	26	int set_cookie(struct bpf_sock_addr *ctx)
19	27	{
20		− __u32 cookie_value = 0xFF;
21		− __u64 cookie_key;
	28	+ struct socket_cookie *p;
22	29
23	30	if (ctx->family != AF_INET6 \|\| ctx->user_family != AF_INET6)
24	31	return 1;
25	32
26		− cookie_key = bpf_get_socket_cookie(ctx);
27		− if (bpf_map_update_elem(&socket_cookies, &cookie_key, &cookie_value, 0))
28		− return 0;
	33	+ p = bpf_sk_storage_get(&socket_cookies, ctx->sk, 0,
	34	+ BPF_SK_STORAGE_GET_F_CREATE);
	35	+ if (!p)
	36	+ return 1;
	37	+
	38	+ p->cookie_value = 0xFF;
	39	+ p->cookie_key = bpf_get_socket_cookie(ctx);
29	40
30	41	return 1;
31	42	}
@@ -33,9 +44,8 @@ int set_cookie(struct bpf_sock_addr *ctx) · 1 unmodified line
33	44	SEC("sockops")
34	45	int update_cookie(struct bpf_sock_ops *ctx)
35	46	{
36		− __u32 new_cookie_value;
37		− __u32 *cookie_value;
38		− __u64 cookie_key;
	47	+ struct bpf_sock *sk;
	48	+ struct socket_cookie *p;
39	49
40	50	if (ctx->family != AF_INET6)
41	51	return 1;
@@ -43,14 +53,17 @@ int update_cookie(struct bpf_sock_ops *ctx) · 1 unmodified line
43	53	if (ctx->op != BPF_SOCK_OPS_TCP_CONNECT_CB)
44	54	return 1;
45	55
46		− cookie_key = bpf_get_socket_cookie(ctx);
	56	+ if (!ctx->sk)
	57	+ return 1;
	58	+
	59	+ p = bpf_sk_storage_get(&socket_cookies, ctx->sk, 0, 0);
	60	+ if (!p)
	61	+ return 1;
47	62
48		− cookie_value = bpf_map_lookup_elem(&socket_cookies, &cookie_key);
49		− if (!cookie_value)
	63	+ if (p->cookie_key != bpf_get_socket_cookie(ctx))
50	64	return 1;
51	65
52		− new_cookie_value = (ctx->local_port << 8) \| *cookie_value;
53		− bpf_map_update_elem(&socket_cookies, &cookie_key, &new_cookie_value, 0);
	66	+ p->cookie_value = (ctx->local_port << 8) \| p->cookie_value;
54	67
55	68	return 1;
56	69	}

tools/testing/selftests/bpf/progs/sockmap_parse_prog.c+0−1

@@ -1,6 +1,5 @@
1	1	#include <linux/bpf.h>
2	2	#include "bpf_helpers.h"
3		−#include "bpf_util.h"
4	3	#include "bpf_endian.h"
5	4
6	5	int _version SEC("version") = 1;

tools/testing/selftests/bpf/progs/sockmap_tcp_msg_prog.c+1−1

@@ -1,6 +1,6 @@
1	1	#include <linux/bpf.h>
	2	+
2	3	#include "bpf_helpers.h"
3		−#include "bpf_util.h"
4	4	#include "bpf_endian.h"
5	5
6	6	int _version SEC("version") = 1;

tools/testing/selftests/bpf/progs/sockmap_verdict_prog.c+0−1

@@ -1,6 +1,5 @@
1	1	#include <linux/bpf.h>
2	2	#include "bpf_helpers.h"
3		−#include "bpf_util.h"
4	3	#include "bpf_endian.h"
5	4
6	5	int _version SEC("version") = 1;

tools/testing/selftests/bpf/progs/strobemeta.c+10−0

@@ -0,0 +1,10 @@
	1	+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
	2	+// Copyright (c) 2019 Facebook
	3	+
	4	+#define STROBE_MAX_INTS 2
	5	+#define STROBE_MAX_STRS 25
	6	+#define STROBE_MAX_MAPS 100
	7	+#define STROBE_MAX_MAP_ENTRIES 20
	8	+/* full unroll by llvm #undef NO_UNROLL */
	9	+#include "strobemeta.h"
	10	+

tools/testing/selftests/bpf/progs/strobemeta.h+528−0

@@ -0,0 +1,528 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+
	4	+#include <stdint.h>
	5	+#include <stddef.h>
	6	+#include <stdbool.h>
	7	+#include <linux/bpf.h>
	8	+#include <linux/ptrace.h>
	9	+#include <linux/sched.h>
	10	+#include <linux/types.h>
	11	+#include "bpf_helpers.h"
	12	+
	13	+typedef uint32_t pid_t;
	14	+struct task_struct {};
	15	+
	16	+#define TASK_COMM_LEN 16
	17	+#define PERF_MAX_STACK_DEPTH 127
	18	+
	19	+#define STROBE_TYPE_INVALID 0
	20	+#define STROBE_TYPE_INT 1
	21	+#define STROBE_TYPE_STR 2
	22	+#define STROBE_TYPE_MAP 3
	23	+
	24	+#define STACK_TABLE_EPOCH_SHIFT 20
	25	+#define STROBE_MAX_STR_LEN 1
	26	+#define STROBE_MAX_CFGS 32
	27	+#define STROBE_MAX_PAYLOAD \
	28	+ (STROBE_MAX_STRS * STROBE_MAX_STR_LEN + \
	29	+ STROBE_MAX_MAPS * (1 + STROBE_MAX_MAP_ENTRIES * 2) * STROBE_MAX_STR_LEN)
	30	+
	31	+struct strobe_value_header {
	32	+ /*
	33	+ * meaning depends on type:
	34	+ * 1. int: 0, if value not set, 1 otherwise
	35	+ * 2. str: 1 always, whether value is set or not is determined by ptr
	36	+ * 3. map: 1 always, pointer points to additional struct with number
	37	+ * of entries (up to STROBE_MAX_MAP_ENTRIES)
	38	+ */
	39	+ uint16_t len;
	40	+ /*
	41	+ * _reserved might be used for some future fields/flags, but we always
	42	+ * want to keep strobe_value_header to be 8 bytes, so BPF can read 16
	43	+ * bytes in one go and get both header and value
	44	+ */
	45	+ uint8_t _reserved[6];
	46	+};
	47	+
	48	+/*
	49	+ * strobe_value_generic is used from BPF probe only, but needs to be a union
	50	+ * of strobe_value_int/strobe_value_str/strobe_value_map
	51	+ */
	52	+struct strobe_value_generic {
	53	+ struct strobe_value_header header;
	54	+ union {
	55	+ int64_t val;
	56	+ void *ptr;
	57	+ };
	58	+};
	59	+
	60	+struct strobe_value_int {
	61	+ struct strobe_value_header header;
	62	+ int64_t value;
	63	+};
	64	+
	65	+struct strobe_value_str {
	66	+ struct strobe_value_header header;
	67	+ const char* value;
	68	+};
	69	+
	70	+struct strobe_value_map {
	71	+ struct strobe_value_header header;
	72	+ const struct strobe_map_raw* value;
	73	+};
	74	+
	75	+struct strobe_map_entry {
	76	+ const char* key;
	77	+ const char* val;
	78	+};
	79	+
	80	+/*
	81	+ * Map of C-string key/value pairs with fixed maximum capacity. Each map has
	82	+ * corresponding int64 ID, which application can use (or ignore) in whatever
	83	+ * way appropriate. Map is "write-only", there is no way to get data out of
	84	+ * map. Map is intended to be used to provide metadata for profilers and is
	85	+ * not to be used for internal in-app communication. All methods are
	86	+ * thread-safe.
	87	+ */
	88	+struct strobe_map_raw {
	89	+ /*
	90	+ * general purpose unique ID that's up to application to decide
	91	+ * whether and how to use; for request metadata use case id is unique
	92	+ * request ID that's used to match metadata with stack traces on
	93	+ * Strobelight backend side
	94	+ */
	95	+ int64_t id;
	96	+ /* number of used entries in map */
	97	+ int64_t cnt;
	98	+ /*
	99	+ * having volatile doesn't change anything on BPF side, but clang
	100	+ * emits warnings for passing `volatile const char *` into
	101	+ * bpf_probe_read_str that expects just `const char *`
	102	+ */
	103	+ const char* tag;
	104	+ /*
	105	+ * key/value entries, each consisting of 2 pointers to key and value
	106	+ * C strings
	107	+ */
	108	+ struct strobe_map_entry entries[STROBE_MAX_MAP_ENTRIES];
	109	+};
	110	+
	111	+/* Following values define supported values of TLS mode */
	112	+#define TLS_NOT_SET -1
	113	+#define TLS_LOCAL_EXEC 0
	114	+#define TLS_IMM_EXEC 1
	115	+#define TLS_GENERAL_DYN 2
	116	+
	117	+/*
	118	+ * structure that universally represents TLS location (both for static
	119	+ * executables and shared libraries)
	120	+ */
	121	+struct strobe_value_loc {
	122	+ /*
	123	+ * tls_mode defines what TLS mode was used for particular metavariable:
	124	+ * - -1 (TLS_NOT_SET) - no metavariable;
	125	+ * - 0 (TLS_LOCAL_EXEC) - Local Executable mode;
	126	+ * - 1 (TLS_IMM_EXEC) - Immediate Executable mode;
	127	+ * - 2 (TLS_GENERAL_DYN) - General Dynamic mode;
	128	+ * Local Dynamic mode is not yet supported, because never seen in
	129	+ * practice. Mode defines how offset field is interpreted. See
	130	+ * calc_location() in below for details.
	131	+ */
	132	+ int64_t tls_mode;
	133	+ /*
	134	+ * TLS_LOCAL_EXEC: offset from thread pointer (fs:0 for x86-64,
	135	+ * tpidr_el0 for aarch64).
	136	+ * TLS_IMM_EXEC: absolute address of GOT entry containing offset
	137	+ * from thread pointer;
	138	+ * TLS_GENERAL_DYN: absolute addres of double GOT entry
	139	+ * containing tls_index_t struct;
	140	+ */
	141	+ int64_t offset;
	142	+};
	143	+
	144	+struct strobemeta_cfg {
	145	+ int64_t req_meta_idx;
	146	+ struct strobe_value_loc int_locs[STROBE_MAX_INTS];
	147	+ struct strobe_value_loc str_locs[STROBE_MAX_STRS];
	148	+ struct strobe_value_loc map_locs[STROBE_MAX_MAPS];
	149	+};
	150	+
	151	+struct strobe_map_descr {
	152	+ uint64_t id;
	153	+ int16_t tag_len;
	154	+ /*
	155	+ * cnt <0 - map value isn't set;
	156	+ * 0 - map has id set, but no key/value entries
	157	+ */
	158	+ int16_t cnt;
	159	+ /*
	160	+ * both key_lens[i] and val_lens[i] should be >0 for present key/value
	161	+ * entry
	162	+ */
	163	+ uint16_t key_lens[STROBE_MAX_MAP_ENTRIES];
	164	+ uint16_t val_lens[STROBE_MAX_MAP_ENTRIES];
	165	+};
	166	+
	167	+struct strobemeta_payload {
	168	+ /* req_id has valid request ID, if req_meta_valid == 1 */
	169	+ int64_t req_id;
	170	+ uint8_t req_meta_valid;
	171	+ /*
	172	+ * mask has Nth bit set to 1, if Nth metavar was present and
	173	+ * successfully read
	174	+ */
	175	+ uint64_t int_vals_set_mask;
	176	+ int64_t int_vals[STROBE_MAX_INTS];
	177	+ /* len is >0 for present values */
	178	+ uint16_t str_lens[STROBE_MAX_STRS];
	179	+ /* if map_descrs[i].cnt == -1, metavar is not present/set */
	180	+ struct strobe_map_descr map_descrs[STROBE_MAX_MAPS];
	181	+ /*
	182	+ * payload has compactly packed values of str and map variables in the
	183	+ * form: strval1\0strval2\0map1key1\0map1val1\0map2key1\0map2val1\0
	184	+ * (and so on); str_lens[i], key_lens[i] and val_lens[i] determines
	185	+ * value length
	186	+ */
	187	+ char payload[STROBE_MAX_PAYLOAD];
	188	+};
	189	+
	190	+struct strobelight_bpf_sample {
	191	+ uint64_t ktime;
	192	+ char comm[TASK_COMM_LEN];
	193	+ pid_t pid;
	194	+ int user_stack_id;
	195	+ int kernel_stack_id;
	196	+ int has_meta;
	197	+ struct strobemeta_payload metadata;
	198	+ /*
	199	+ * makes it possible to pass (<real payload size> + 1) as data size to
	200	+ * perf_submit() to avoid perf_submit's paranoia about passing zero as
	201	+ * size, as it deduces that <real payload size> might be
	202	+ * theoretically zero
	203	+ */
	204	+ char dummy_safeguard;
	205	+};
	206	+
	207	+struct bpf_map_def SEC("maps") samples = {
	208	+ .type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
	209	+ .key_size = sizeof(int),
	210	+ .value_size = sizeof(int),
	211	+ .max_entries = 32,
	212	+};
	213	+
	214	+struct bpf_map_def SEC("maps") stacks_0 = {
	215	+ .type = BPF_MAP_TYPE_STACK_TRACE,
	216	+ .key_size = sizeof(uint32_t),
	217	+ .value_size = sizeof(uint64_t) * PERF_MAX_STACK_DEPTH,
	218	+ .max_entries = 16,
	219	+};
	220	+
	221	+struct bpf_map_def SEC("maps") stacks_1 = {
	222	+ .type = BPF_MAP_TYPE_STACK_TRACE,
	223	+ .key_size = sizeof(uint32_t),
	224	+ .value_size = sizeof(uint64_t) * PERF_MAX_STACK_DEPTH,
	225	+ .max_entries = 16,
	226	+};
	227	+
	228	+struct bpf_map_def SEC("maps") sample_heap = {
	229	+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
	230	+ .key_size = sizeof(uint32_t),
	231	+ .value_size = sizeof(struct strobelight_bpf_sample),
	232	+ .max_entries = 1,
	233	+};
	234	+
	235	+struct bpf_map_def SEC("maps") strobemeta_cfgs = {
	236	+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
	237	+ .key_size = sizeof(pid_t),
	238	+ .value_size = sizeof(struct strobemeta_cfg),
	239	+ .max_entries = STROBE_MAX_CFGS,
	240	+};
	241	+
	242	+/* Type for the dtv. */
	243	+/* https://github.com/lattera/glibc/blob/master/nptl/sysdeps/x86_64/tls.h#L34 */
	244	+typedef union dtv {
	245	+ size_t counter;
	246	+ struct {
	247	+ void* val;
	248	+ bool is_static;
	249	+ } pointer;
	250	+} dtv_t;
	251	+
	252	+/* Partial definition for tcbhead_t */
	253	+/* https://github.com/bminor/glibc/blob/master/sysdeps/x86_64/nptl/tls.h#L42 */
	254	+struct tcbhead {
	255	+ void* tcb;
	256	+ dtv_t* dtv;
	257	+};
	258	+
	259	+/*
	260	+ * TLS module/offset information for shared library case.
	261	+ * For x86-64, this is mapped onto two entries in GOT.
	262	+ * For aarch64, this is pointed to by second GOT entry.
	263	+ */
	264	+struct tls_index {
	265	+ uint64_t module;
	266	+ uint64_t offset;
	267	+};
	268	+
	269	+static inline __attribute__((always_inline))
	270	+void calc_location(struct strobe_value_loc loc, void *tls_base)
	271	+{
	272	+ /*
	273	+ * tls_mode value is:
	274	+ * - -1 (TLS_NOT_SET), if no metavar is present;
	275	+ * - 0 (TLS_LOCAL_EXEC), if metavar uses Local Executable mode of TLS
	276	+ * (offset from fs:0 for x86-64 or tpidr_el0 for aarch64);
	277	+ * - 1 (TLS_IMM_EXEC), if metavar uses Immediate Executable mode of TLS;
	278	+ * - 2 (TLS_GENERAL_DYN), if metavar uses General Dynamic mode of TLS;
	279	+ * This schema allows to use something like:
	280	+ * (tls_mode + 1) * (tls_base + offset)
	281	+ * to get NULL for "no metavar" location, or correct pointer for local
	282	+ * executable mode without doing extra ifs.
	283	+ */
	284	+ if (loc->tls_mode <= TLS_LOCAL_EXEC) {
	285	+ /* static executable is simple, we just have offset from
	286	+ * tls_base */
	287	+ void *addr = tls_base + loc->offset;
	288	+ /* multiply by (tls_mode + 1) to get NULL, if we have no
	289	+ * metavar in this slot */
	290	+ return (void )((loc->tls_mode + 1) (int64_t)addr);
	291	+ }
	292	+ /*
	293	+ * Other modes are more complicated, we need to jump through few hoops.
	294	+ *
	295	+ * For immediate executable mode (currently supported only for aarch64):
	296	+ * - loc->offset is pointing to a GOT entry containing fixed offset
	297	+ * relative to tls_base;
	298	+ *
	299	+ * For general dynamic mode:
	300	+ * - loc->offset is pointing to a beginning of double GOT entries;
	301	+ * - (for aarch64 only) second entry points to tls_index_t struct;
	302	+ * - (for x86-64 only) two GOT entries are already tls_index_t;
	303	+ * - tls_index_t->module is used to find start of TLS section in
	304	+ * which variable resides;
	305	+ * - tls_index_t->offset provides offset within that TLS section,
	306	+ * pointing to value of variable.
	307	+ */
	308	+ struct tls_index tls_index;
	309	+ dtv_t *dtv;
	310	+ void *tls_ptr;
	311	+
	312	+ bpf_probe_read(&tls_index, sizeof(struct tls_index),
	313	+ (void *)loc->offset);
	314	+ /* valid module index is always positive */
	315	+ if (tls_index.module > 0) {
	316	+ /* dtv = ((struct tcbhead )tls_base)->dtv[tls_index.module] /
	317	+ bpf_probe_read(&dtv, sizeof(dtv),
	318	+ &((struct tcbhead *)tls_base)->dtv);
	319	+ dtv += tls_index.module;
	320	+ } else {
	321	+ dtv = NULL;
	322	+ }
	323	+ bpf_probe_read(&tls_ptr, sizeof(void *), dtv);
	324	+ /* if pointer has (void )-1 value, then TLS wasn't initialized yet /
	325	+ return tls_ptr && tls_ptr != (void *)-1
	326	+ ? tls_ptr + tls_index.offset
	327	+ : NULL;
	328	+}
	329	+
	330	+static inline __attribute__((always_inline))
	331	+void read_int_var(struct strobemeta_cfg cfg, size_t idx, void tls_base,
	332	+ struct strobe_value_generic *value,
	333	+ struct strobemeta_payload *data)
	334	+{
	335	+ void *location = calc_location(&cfg->int_locs[idx], tls_base);
	336	+ if (!location)
	337	+ return;
	338	+
	339	+ bpf_probe_read(value, sizeof(struct strobe_value_generic), location);
	340	+ data->int_vals[idx] = value->val;
	341	+ if (value->header.len)
	342	+ data->int_vals_set_mask \|= (1 << idx);
	343	+}
	344	+
	345	+static inline __attribute__((always_inline))
	346	+uint64_t read_str_var(struct strobemeta_cfg* cfg, size_t idx, void *tls_base,
	347	+ struct strobe_value_generic *value,
	348	+ struct strobemeta_payload data, void payload)
	349	+{
	350	+ void *location;
	351	+ uint32_t len;
	352	+
	353	+ data->str_lens[idx] = 0;
	354	+ location = calc_location(&cfg->str_locs[idx], tls_base);
	355	+ if (!location)
	356	+ return 0;
	357	+
	358	+ bpf_probe_read(value, sizeof(struct strobe_value_generic), location);
	359	+ len = bpf_probe_read_str(payload, STROBE_MAX_STR_LEN, value->ptr);
	360	+ /*
	361	+ * if bpf_probe_read_str returns error (<0), due to casting to
	362	+ * unsinged int, it will become big number, so next check is
	363	+ * sufficient to check for errors AND prove to BPF verifier, that
	364	+ * bpf_probe_read_str won't return anything bigger than
	365	+ * STROBE_MAX_STR_LEN
	366	+ */
	367	+ if (len > STROBE_MAX_STR_LEN)
	368	+ return 0;
	369	+
	370	+ data->str_lens[idx] = len;
	371	+ return len;
	372	+}
	373	+
	374	+static inline __attribute__((always_inline))
	375	+void read_map_var(struct strobemeta_cfg cfg, size_t idx, void *tls_base,
	376	+ struct strobe_value_generic *value,
	377	+ struct strobemeta_payload* data, void *payload)
	378	+{
	379	+ struct strobe_map_descr* descr = &data->map_descrs[idx];
	380	+ struct strobe_map_raw map;
	381	+ void *location;
	382	+ uint32_t len;
	383	+ int i;
	384	+
	385	+ descr->tag_len = 0; /* presume no tag is set */
	386	+ descr->cnt = -1; /* presume no value is set */
	387	+
	388	+ location = calc_location(&cfg->map_locs[idx], tls_base);
	389	+ if (!location)
	390	+ return payload;
	391	+
	392	+ bpf_probe_read(value, sizeof(struct strobe_value_generic), location);
	393	+ if (bpf_probe_read(&map, sizeof(struct strobe_map_raw), value->ptr))
	394	+ return payload;
	395	+
	396	+ descr->id = map.id;
	397	+ descr->cnt = map.cnt;
	398	+ if (cfg->req_meta_idx == idx) {
	399	+ data->req_id = map.id;
	400	+ data->req_meta_valid = 1;
	401	+ }
	402	+
	403	+ len = bpf_probe_read_str(payload, STROBE_MAX_STR_LEN, map.tag);
	404	+ if (len <= STROBE_MAX_STR_LEN) {
	405	+ descr->tag_len = len;
	406	+ payload += len;
	407	+ }
	408	+
	409	+#ifdef NO_UNROLL
	410	+#pragma clang loop unroll(disable)
	411	+#else
	412	+#pragma unroll
	413	+#endif
	414	+ for (int i = 0; i < STROBE_MAX_MAP_ENTRIES && i < map.cnt; ++i) {
	415	+ descr->key_lens[i] = 0;
	416	+ len = bpf_probe_read_str(payload, STROBE_MAX_STR_LEN,
	417	+ map.entries[i].key);
	418	+ if (len <= STROBE_MAX_STR_LEN) {
	419	+ descr->key_lens[i] = len;
	420	+ payload += len;
	421	+ }
	422	+ descr->val_lens[i] = 0;
	423	+ len = bpf_probe_read_str(payload, STROBE_MAX_STR_LEN,
	424	+ map.entries[i].val);
	425	+ if (len <= STROBE_MAX_STR_LEN) {
	426	+ descr->val_lens[i] = len;
	427	+ payload += len;
	428	+ }
	429	+ }
	430	+
	431	+ return payload;
	432	+}
	433	+
	434	+/*
	435	+ * read_strobe_meta returns NULL, if no metadata was read; otherwise returns
	436	+ * pointer to right after payload ends
	437	+ */
	438	+static inline __attribute__((always_inline))
	439	+void read_strobe_meta(struct task_struct task,
	440	+ struct strobemeta_payload* data) {
	441	+ pid_t pid = bpf_get_current_pid_tgid() >> 32;
	442	+ struct strobe_value_generic value = {0};
	443	+ struct strobemeta_cfg *cfg;
	444	+ void tls_base, payload;
	445	+
	446	+ cfg = bpf_map_lookup_elem(&strobemeta_cfgs, &pid);
	447	+ if (!cfg)
	448	+ return NULL;
	449	+
	450	+ data->int_vals_set_mask = 0;
	451	+ data->req_meta_valid = 0;
	452	+ payload = data->payload;
	453	+ /*
	454	+ * we don't have struct task_struct definition, it should be:
	455	+ * tls_base = (void *)task->thread.fsbase;
	456	+ */
	457	+ tls_base = (void *)task;
	458	+
	459	+#ifdef NO_UNROLL
	460	+#pragma clang loop unroll(disable)
	461	+#else
	462	+#pragma unroll
	463	+#endif
	464	+ for (int i = 0; i < STROBE_MAX_INTS; ++i) {
	465	+ read_int_var(cfg, i, tls_base, &value, data);
	466	+ }
	467	+#ifdef NO_UNROLL
	468	+#pragma clang loop unroll(disable)
	469	+#else
	470	+#pragma unroll
	471	+#endif
	472	+ for (int i = 0; i < STROBE_MAX_STRS; ++i) {
	473	+ payload += read_str_var(cfg, i, tls_base, &value, data, payload);
	474	+ }
	475	+#ifdef NO_UNROLL
	476	+#pragma clang loop unroll(disable)
	477	+#else
	478	+#pragma unroll
	479	+#endif
	480	+ for (int i = 0; i < STROBE_MAX_MAPS; ++i) {
	481	+ payload = read_map_var(cfg, i, tls_base, &value, data, payload);
	482	+ }
	483	+ /*
	484	+ * return pointer right after end of payload, so it's possible to
	485	+ * calculate exact amount of useful data that needs to be sent
	486	+ */
	487	+ return payload;
	488	+}
	489	+
	490	+SEC("raw_tracepoint/kfree_skb")
	491	+int on_event(struct pt_regs *ctx) {
	492	+ pid_t pid = bpf_get_current_pid_tgid() >> 32;
	493	+ struct strobelight_bpf_sample* sample;
	494	+ struct task_struct *task;
	495	+ uint32_t zero = 0;
	496	+ uint64_t ktime_ns;
	497	+ void *sample_end;
	498	+
	499	+ sample = bpf_map_lookup_elem(&sample_heap, &zero);
	500	+ if (!sample)
	501	+ return 0; /* this will never happen */
	502	+
	503	+ sample->pid = pid;
	504	+ bpf_get_current_comm(&sample->comm, TASK_COMM_LEN);
	505	+ ktime_ns = bpf_ktime_get_ns();
	506	+ sample->ktime = ktime_ns;
	507	+
	508	+ task = (struct task_struct *)bpf_get_current_task();
	509	+ sample_end = read_strobe_meta(task, &sample->metadata);
	510	+ sample->has_meta = sample_end != NULL;
	511	+ sample_end = sample_end ? : &sample->metadata;
	512	+
	513	+ if ((ktime_ns >> STACK_TABLE_EPOCH_SHIFT) & 1) {
	514	+ sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_1, 0);
	515	+ sample->user_stack_id = bpf_get_stackid(ctx, &stacks_1, BPF_F_USER_STACK);
	516	+ } else {
	517	+ sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_0, 0);
	518	+ sample->user_stack_id = bpf_get_stackid(ctx, &stacks_0, BPF_F_USER_STACK);
	519	+ }
	520	+
	521	+ uint64_t sample_size = sample_end - (void *)sample;
	522	+ /* should always be true */
	523	+ if (sample_size < sizeof(struct strobelight_bpf_sample))
	524	+ bpf_perf_event_output(ctx, &samples, 0, sample, 1 + sample_size);
	525	+ return 0;
	526	+}
	527	+
	528	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/strobemeta_nounroll1.c+9−0

@@ -0,0 +1,9 @@
	1	+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
	2	+// Copyright (c) 2019 Facebook
	3	+
	4	+#define STROBE_MAX_INTS 2
	5	+#define STROBE_MAX_STRS 25
	6	+#define STROBE_MAX_MAPS 13
	7	+#define STROBE_MAX_MAP_ENTRIES 20
	8	+#define NO_UNROLL
	9	+#include "strobemeta.h"

tools/testing/selftests/bpf/progs/strobemeta_nounroll2.c+9−0

@@ -0,0 +1,9 @@
	1	+// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
	2	+// Copyright (c) 2019 Facebook
	3	+
	4	+#define STROBE_MAX_INTS 2
	5	+#define STROBE_MAX_STRS 25
	6	+#define STROBE_MAX_MAPS 30
	7	+#define STROBE_MAX_MAP_ENTRIES 20
	8	+#define NO_UNROLL
	9	+#include "strobemeta.h"

tools/testing/selftests/bpf/progs/test_btf_newkv.c+73−0

@@ -0,0 +1,73 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/* Copyright (c) 2018 Facebook */
	3	+#include <linux/bpf.h>
	4	+#include "bpf_helpers.h"
	5	+
	6	+int _version SEC("version") = 1;
	7	+
	8	+struct ipv_counts {
	9	+ unsigned int v4;
	10	+ unsigned int v6;
	11	+};
	12	+
	13	+/* just to validate we can handle maps in multiple sections */
	14	+struct bpf_map_def SEC("maps") btf_map_legacy = {
	15	+ .type = BPF_MAP_TYPE_ARRAY,
	16	+ .key_size = sizeof(int),
	17	+ .value_size = sizeof(long long),
	18	+ .max_entries = 4,
	19	+};
	20	+
	21	+BPF_ANNOTATE_KV_PAIR(btf_map_legacy, int, struct ipv_counts);
	22	+
	23	+struct {
	24	+ int *key;
	25	+ struct ipv_counts *value;
	26	+ unsigned int type;
	27	+ unsigned int max_entries;
	28	+} btf_map SEC(".maps") = {
	29	+ .type = BPF_MAP_TYPE_ARRAY,
	30	+ .max_entries = 4,
	31	+};
	32	+
	33	+struct dummy_tracepoint_args {
	34	+ unsigned long long pad;
	35	+ struct sock *sock;
	36	+};
	37	+
	38	+__attribute__((noinline))
	39	+static int test_long_fname_2(struct dummy_tracepoint_args *arg)
	40	+{
	41	+ struct ipv_counts *counts;
	42	+ int key = 0;
	43	+
	44	+ if (!arg->sock)
	45	+ return 0;
	46	+
	47	+ counts = bpf_map_lookup_elem(&btf_map, &key);
	48	+ if (!counts)
	49	+ return 0;
	50	+
	51	+ counts->v6++;
	52	+
	53	+ /* just verify we can reference both maps */
	54	+ counts = bpf_map_lookup_elem(&btf_map_legacy, &key);
	55	+ if (!counts)
	56	+ return 0;
	57	+
	58	+ return 0;
	59	+}
	60	+
	61	+__attribute__((noinline))
	62	+static int test_long_fname_1(struct dummy_tracepoint_args *arg)
	63	+{
	64	+ return test_long_fname_2(arg);
	65	+}
	66	+
	67	+SEC("dummy_tracepoint")
	68	+int _dummy_tracepoint(struct dummy_tracepoint_args *arg)
	69	+{
	70	+ return test_long_fname_1(arg);
	71	+}
	72	+
	73	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_get_stack_rawtp.c+19−8

@@ -15,17 +15,25 @@ struct stack_trace_t { · 14 unmodified lines
15	15	struct bpf_stack_build_id user_stack_buildid[MAX_STACK_RAWTP];
16	16	};
17	17
18		−struct bpf_map_def SEC("maps") perfmap = {
	18	+struct {
	19	+ __u32 type;
	20	+ __u32 max_entries;
	21	+ __u32 key_size;
	22	+ __u32 value_size;
	23	+} perfmap SEC(".maps") = {
19	24	.type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
	25	+ .max_entries = 2,
20	26	.key_size = sizeof(int),
21	27	.value_size = sizeof(__u32),
22		− .max_entries = 2,
23	28	};
24	29
25		−struct bpf_map_def SEC("maps") stackdata_map = {
	30	+struct {
	31	+ __u32 type;
	32	+ __u32 max_entries;
	33	+ __u32 *key;
	34	+ struct stack_trace_t *value;
	35	+} stackdata_map SEC(".maps") = {
26	36	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
27		− .key_size = sizeof(__u32),
28		− .value_size = sizeof(struct stack_trace_t),
29	37	.max_entries = 1,
30	38	};
31	39
@@ -47,10 +55,13 @@ struct bpf_map_def SEC("maps") stackdata_map = { · 15 unmodified lines
47	55	* issue and avoid complicated C programming massaging.
48	56	* This is an acceptable workaround since there is one entry here.
49	57	*/
50		−struct bpf_map_def SEC("maps") rawdata_map = {
	58	+struct {
	59	+ __u32 type;
	60	+ __u32 max_entries;
	61	+ __u32 *key;
	62	+ __u64 (value)[2 MAX_STACK_RAWTP];
	63	+} rawdata_map SEC(".maps") = {
51	64	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
52		− .key_size = sizeof(__u32),
53		− .value_size = MAX_STACK_RAWTP * sizeof(__u64) * 2,
54	65	.max_entries = 1,
55	66	};
56	67

tools/testing/selftests/bpf/progs/test_global_data.c+18−9

@@ -7,17 +7,23 @@ · 6 unmodified lines
7	7
8	8	#include "bpf_helpers.h"
9	9
10		−struct bpf_map_def SEC("maps") result_number = {
	10	+struct {
	11	+ __u32 type;
	12	+ __u32 max_entries;
	13	+ __u32 *key;
	14	+ __u64 *value;
	15	+} result_number SEC(".maps") = {
11	16	.type = BPF_MAP_TYPE_ARRAY,
12		− .key_size = sizeof(__u32),
13		− .value_size = sizeof(__u64),
14	17	.max_entries = 11,
15	18	};
16	19
17		−struct bpf_map_def SEC("maps") result_string = {
	20	+struct {
	21	+ __u32 type;
	22	+ __u32 max_entries;
	23	+ __u32 *key;
	24	+ const char (*value)[32];
	25	+} result_string SEC(".maps") = {
18	26	.type = BPF_MAP_TYPE_ARRAY,
19		− .key_size = sizeof(__u32),
20		− .value_size = 32,
21	27	.max_entries = 5,
22	28	};
23	29
@@ -27,10 +33,13 @@ struct foo { · 3 unmodified lines
27	33	__u64 c;
28	34	};
29	35
30		−struct bpf_map_def SEC("maps") result_struct = {
	36	+struct {
	37	+ __u32 type;
	38	+ __u32 max_entries;
	39	+ __u32 *key;
	40	+ struct foo *value;
	41	+} result_struct SEC(".maps") = {
31	42	.type = BPF_MAP_TYPE_ARRAY,
32		− .key_size = sizeof(__u32),
33		− .value_size = sizeof(struct foo),
34	43	.max_entries = 5,
35	44	};
36	45

tools/testing/selftests/bpf/progs/test_l4lb.c+30−15

@@ -169,38 +169,53 @@ struct eth_hdr { · 168 unmodified lines
169	169	unsigned short eth_proto;
170	170	};
171	171
172		−struct bpf_map_def SEC("maps") vip_map = {
	172	+struct {
	173	+ __u32 type;
	174	+ __u32 max_entries;
	175	+ struct vip *key;
	176	+ struct vip_meta *value;
	177	+} vip_map SEC(".maps") = {
173	178	.type = BPF_MAP_TYPE_HASH,
174		− .key_size = sizeof(struct vip),
175		− .value_size = sizeof(struct vip_meta),
176	179	.max_entries = MAX_VIPS,
177	180	};
178	181
179		−struct bpf_map_def SEC("maps") ch_rings = {
	182	+struct {
	183	+ __u32 type;
	184	+ __u32 max_entries;
	185	+ __u32 *key;
	186	+ __u32 *value;
	187	+} ch_rings SEC(".maps") = {
180	188	.type = BPF_MAP_TYPE_ARRAY,
181		− .key_size = sizeof(__u32),
182		− .value_size = sizeof(__u32),
183	189	.max_entries = CH_RINGS_SIZE,
184	190	};
185	191
186		−struct bpf_map_def SEC("maps") reals = {
	192	+struct {
	193	+ __u32 type;
	194	+ __u32 max_entries;
	195	+ __u32 *key;
	196	+ struct real_definition *value;
	197	+} reals SEC(".maps") = {
187	198	.type = BPF_MAP_TYPE_ARRAY,
188		− .key_size = sizeof(__u32),
189		− .value_size = sizeof(struct real_definition),
190	199	.max_entries = MAX_REALS,
191	200	};
192	201
193		−struct bpf_map_def SEC("maps") stats = {
	202	+struct {
	203	+ __u32 type;
	204	+ __u32 max_entries;
	205	+ __u32 *key;
	206	+ struct vip_stats *value;
	207	+} stats SEC(".maps") = {
194	208	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
195		− .key_size = sizeof(__u32),
196		− .value_size = sizeof(struct vip_stats),
197	209	.max_entries = MAX_VIPS,
198	210	};
199	211
200		−struct bpf_map_def SEC("maps") ctl_array = {
	212	+struct {
	213	+ __u32 type;
	214	+ __u32 max_entries;
	215	+ __u32 *key;
	216	+ struct ctl_value *value;
	217	+} ctl_array SEC(".maps") = {
201	218	.type = BPF_MAP_TYPE_ARRAY,
202		− .key_size = sizeof(__u32),
203		− .value_size = sizeof(struct ctl_value),
204	219	.max_entries = CTL_MAP_SIZE,
205	220	};
206	221

tools/testing/selftests/bpf/progs/test_l4lb_noinline.c+30−15

@@ -165,38 +165,53 @@ struct eth_hdr { · 164 unmodified lines
165	165	unsigned short eth_proto;
166	166	};
167	167
168		−struct bpf_map_def SEC("maps") vip_map = {
	168	+struct {
	169	+ __u32 type;
	170	+ __u32 max_entries;
	171	+ struct vip *key;
	172	+ struct vip_meta *value;
	173	+} vip_map SEC(".maps") = {
169	174	.type = BPF_MAP_TYPE_HASH,
170		− .key_size = sizeof(struct vip),
171		− .value_size = sizeof(struct vip_meta),
172	175	.max_entries = MAX_VIPS,
173	176	};
174	177
175		−struct bpf_map_def SEC("maps") ch_rings = {
	178	+struct {
	179	+ __u32 type;
	180	+ __u32 max_entries;
	181	+ __u32 *key;
	182	+ __u32 *value;
	183	+} ch_rings SEC(".maps") = {
176	184	.type = BPF_MAP_TYPE_ARRAY,
177		− .key_size = sizeof(__u32),
178		− .value_size = sizeof(__u32),
179	185	.max_entries = CH_RINGS_SIZE,
180	186	};
181	187
182		−struct bpf_map_def SEC("maps") reals = {
	188	+struct {
	189	+ __u32 type;
	190	+ __u32 max_entries;
	191	+ __u32 *key;
	192	+ struct real_definition *value;
	193	+} reals SEC(".maps") = {
183	194	.type = BPF_MAP_TYPE_ARRAY,
184		− .key_size = sizeof(__u32),
185		− .value_size = sizeof(struct real_definition),
186	195	.max_entries = MAX_REALS,
187	196	};
188	197
189		−struct bpf_map_def SEC("maps") stats = {
	198	+struct {
	199	+ __u32 type;
	200	+ __u32 max_entries;
	201	+ __u32 *key;
	202	+ struct vip_stats *value;
	203	+} stats SEC(".maps") = {
190	204	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
191		− .key_size = sizeof(__u32),
192		− .value_size = sizeof(struct vip_stats),
193	205	.max_entries = MAX_VIPS,
194	206	};
195	207
196		−struct bpf_map_def SEC("maps") ctl_array = {
	208	+struct {
	209	+ __u32 type;
	210	+ __u32 max_entries;
	211	+ __u32 *key;
	212	+ struct ctl_value *value;
	213	+} ctl_array SEC(".maps") = {
197	214	.type = BPF_MAP_TYPE_ARRAY,
198		− .key_size = sizeof(__u32),
199		− .value_size = sizeof(struct ctl_value),
200	215	.max_entries = CTL_MAP_SIZE,
201	216	};
202	217

tools/testing/selftests/bpf/progs/test_map_lock.c+12−10

@@ -11,29 +11,31 @@ struct hmap_elem { · 10 unmodified lines
11	11	int var[VAR_NUM];
12	12	};
13	13
14		−struct bpf_map_def SEC("maps") hash_map = {
	14	+struct {
	15	+ __u32 type;
	16	+ __u32 max_entries;
	17	+ __u32 *key;
	18	+ struct hmap_elem *value;
	19	+} hash_map SEC(".maps") = {
15	20	.type = BPF_MAP_TYPE_HASH,
16		− .key_size = sizeof(int),
17		− .value_size = sizeof(struct hmap_elem),
18	21	.max_entries = 1,
19	22	};
20	23
21		−BPF_ANNOTATE_KV_PAIR(hash_map, int, struct hmap_elem);
22		−
23	24	struct array_elem {
24	25	struct bpf_spin_lock lock;
25	26	int var[VAR_NUM];
26	27	};
27	28
28		−struct bpf_map_def SEC("maps") array_map = {
	29	+struct {
	30	+ __u32 type;
	31	+ __u32 max_entries;
	32	+ int *key;
	33	+ struct array_elem *value;
	34	+} array_map SEC(".maps") = {
29	35	.type = BPF_MAP_TYPE_ARRAY,
30		− .key_size = sizeof(int),
31		− .value_size = sizeof(struct array_elem),
32	36	.max_entries = 1,
33	37	};
34	38
35		−BPF_ANNOTATE_KV_PAIR(array_map, int, struct array_elem);
36		−
37	39	SEC("map_lock_demo")
38	40	int bpf_map_lock_test(struct __sk_buff *skb)
39	41	{

tools/testing/selftests/bpf/progs/test_seg6_loop.c+261−0

@@ -0,0 +1,261 @@
	1	+#include <stddef.h>
	2	+#include <inttypes.h>
	3	+#include <errno.h>
	4	+#include <linux/seg6_local.h>
	5	+#include <linux/bpf.h>
	6	+#include "bpf_helpers.h"
	7	+#include "bpf_endian.h"
	8	+
	9	+/* Packet parsing state machine helpers. */
	10	+#define cursor_advance(_cursor, _len) \
	11	+ ({ void *_tmp = _cursor; _cursor += _len; _tmp; })
	12	+
	13	+#define SR6_FLAG_ALERT (1 << 4)
	14	+
	15	+#define htonll(x) ((bpf_htonl(1)) == 1 ? (x) : ((uint64_t)bpf_htonl((x) & \
	16	+ 0xFFFFFFFF) << 32) \| bpf_htonl((x) >> 32))
	17	+#define ntohll(x) ((bpf_ntohl(1)) == 1 ? (x) : ((uint64_t)bpf_ntohl((x) & \
	18	+ 0xFFFFFFFF) << 32) \| bpf_ntohl((x) >> 32))
	19	+#define BPF_PACKET_HEADER __attribute__((packed))
	20	+
	21	+struct ip6_t {
	22	+ unsigned int ver:4;
	23	+ unsigned int priority:8;
	24	+ unsigned int flow_label:20;
	25	+ unsigned short payload_len;
	26	+ unsigned char next_header;
	27	+ unsigned char hop_limit;
	28	+ unsigned long long src_hi;
	29	+ unsigned long long src_lo;
	30	+ unsigned long long dst_hi;
	31	+ unsigned long long dst_lo;
	32	+} BPF_PACKET_HEADER;
	33	+
	34	+struct ip6_addr_t {
	35	+ unsigned long long hi;
	36	+ unsigned long long lo;
	37	+} BPF_PACKET_HEADER;
	38	+
	39	+struct ip6_srh_t {
	40	+ unsigned char nexthdr;
	41	+ unsigned char hdrlen;
	42	+ unsigned char type;
	43	+ unsigned char segments_left;
	44	+ unsigned char first_segment;
	45	+ unsigned char flags;
	46	+ unsigned short tag;
	47	+
	48	+ struct ip6_addr_t segments[0];
	49	+} BPF_PACKET_HEADER;
	50	+
	51	+struct sr6_tlv_t {
	52	+ unsigned char type;
	53	+ unsigned char len;
	54	+ unsigned char value[0];
	55	+} BPF_PACKET_HEADER;
	56	+
	57	+static __attribute__((always_inline)) struct ip6_srh_t get_srh(struct __sk_buff skb)
	58	+{
	59	+ void cursor, data_end;
	60	+ struct ip6_srh_t *srh;
	61	+ struct ip6_t *ip;
	62	+ uint8_t *ipver;
	63	+
	64	+ data_end = (void *)(long)skb->data_end;
	65	+ cursor = (void *)(long)skb->data;
	66	+ ipver = (uint8_t *)cursor;
	67	+
	68	+ if ((void )ipver + sizeof(ipver) > data_end)
	69	+ return NULL;
	70	+
	71	+ if ((*ipver >> 4) != 6)
	72	+ return NULL;
	73	+
	74	+ ip = cursor_advance(cursor, sizeof(*ip));
	75	+ if ((void )ip + sizeof(ip) > data_end)
	76	+ return NULL;
	77	+
	78	+ if (ip->next_header != 43)
	79	+ return NULL;
	80	+
	81	+ srh = cursor_advance(cursor, sizeof(*srh));
	82	+ if ((void )srh + sizeof(srh) > data_end)
	83	+ return NULL;
	84	+
	85	+ if (srh->type != 4)
	86	+ return NULL;
	87	+
	88	+ return srh;
	89	+}
	90	+
	91	+static __attribute__((always_inline))
	92	+int update_tlv_pad(struct __sk_buff *skb, uint32_t new_pad,
	93	+ uint32_t old_pad, uint32_t pad_off)
	94	+{
	95	+ int err;
	96	+
	97	+ if (new_pad != old_pad) {
	98	+ err = bpf_lwt_seg6_adjust_srh(skb, pad_off,
	99	+ (int) new_pad - (int) old_pad);
	100	+ if (err)
	101	+ return err;
	102	+ }
	103	+
	104	+ if (new_pad > 0) {
	105	+ char pad_tlv_buf[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
	106	+ 0, 0, 0};
	107	+ struct sr6_tlv_t pad_tlv = (struct sr6_tlv_t ) pad_tlv_buf;
	108	+
	109	+ pad_tlv->type = SR6_TLV_PADDING;
	110	+ pad_tlv->len = new_pad - 2;
	111	+
	112	+ err = bpf_lwt_seg6_store_bytes(skb, pad_off,
	113	+ (void *)pad_tlv_buf, new_pad);
	114	+ if (err)
	115	+ return err;
	116	+ }
	117	+
	118	+ return 0;
	119	+}
	120	+
	121	+static __attribute__((always_inline))
	122	+int is_valid_tlv_boundary(struct __sk_buff skb, struct ip6_srh_t srh,
	123	+ uint32_t tlv_off, uint32_t pad_size,
	124	+ uint32_t *pad_off)
	125	+{
	126	+ uint32_t srh_off, cur_off;
	127	+ int offset_valid = 0;
	128	+ int err;
	129	+
	130	+ srh_off = (char )srh - (char )(long)skb->data;
	131	+ // cur_off = end of segments, start of possible TLVs
	132	+ cur_off = srh_off + sizeof(*srh) +
	133	+ sizeof(struct ip6_addr_t) * (srh->first_segment + 1);
	134	+
	135	+ *pad_off = 0;
	136	+
	137	+ // we can only go as far as ~10 TLVs due to the BPF max stack size
	138	+ #pragma clang loop unroll(disable)
	139	+ for (int i = 0; i < 100; i++) {
	140	+ struct sr6_tlv_t tlv;
	141	+
	142	+ if (cur_off == *tlv_off)
	143	+ offset_valid = 1;
	144	+
	145	+ if (cur_off >= srh_off + ((srh->hdrlen + 1) << 3))
	146	+ break;
	147	+
	148	+ err = bpf_skb_load_bytes(skb, cur_off, &tlv, sizeof(tlv));
	149	+ if (err)
	150	+ return err;
	151	+
	152	+ if (tlv.type == SR6_TLV_PADDING) {
	153	+ *pad_size = tlv.len + sizeof(tlv);
	154	+ *pad_off = cur_off;
	155	+
	156	+ if (*tlv_off == srh_off) {
	157	+ *tlv_off = cur_off;
	158	+ offset_valid = 1;
	159	+ }
	160	+ break;
	161	+
	162	+ } else if (tlv.type == SR6_TLV_HMAC) {
	163	+ break;
	164	+ }
	165	+
	166	+ cur_off += sizeof(tlv) + tlv.len;
	167	+ } // we reached the padding or HMAC TLVs, or the end of the SRH
	168	+
	169	+ if (*pad_off == 0)
	170	+ *pad_off = cur_off;
	171	+
	172	+ if (*tlv_off == -1)
	173	+ *tlv_off = cur_off;
	174	+ else if (!offset_valid)
	175	+ return -EINVAL;
	176	+
	177	+ return 0;
	178	+}
	179	+
	180	+static __attribute__((always_inline))
	181	+int add_tlv(struct __sk_buff skb, struct ip6_srh_t srh, uint32_t tlv_off,
	182	+ struct sr6_tlv_t *itlv, uint8_t tlv_size)
	183	+{
	184	+ uint32_t srh_off = (char )srh - (char )(long)skb->data;
	185	+ uint8_t len_remaining, new_pad;
	186	+ uint32_t pad_off = 0;
	187	+ uint32_t pad_size = 0;
	188	+ uint32_t partial_srh_len;
	189	+ int err;
	190	+
	191	+ if (tlv_off != -1)
	192	+ tlv_off += srh_off;
	193	+
	194	+ if (itlv->type == SR6_TLV_PADDING \|\| itlv->type == SR6_TLV_HMAC)
	195	+ return -EINVAL;
	196	+
	197	+ err = is_valid_tlv_boundary(skb, srh, &tlv_off, &pad_size, &pad_off);
	198	+ if (err)
	199	+ return err;
	200	+
	201	+ err = bpf_lwt_seg6_adjust_srh(skb, tlv_off, sizeof(*itlv) + itlv->len);
	202	+ if (err)
	203	+ return err;
	204	+
	205	+ err = bpf_lwt_seg6_store_bytes(skb, tlv_off, (void *)itlv, tlv_size);
	206	+ if (err)
	207	+ return err;
	208	+
	209	+ // the following can't be moved inside update_tlv_pad because the
	210	+ // bpf verifier has some issues with it
	211	+ pad_off += sizeof(*itlv) + itlv->len;
	212	+ partial_srh_len = pad_off - srh_off;
	213	+ len_remaining = partial_srh_len % 8;
	214	+ new_pad = 8 - len_remaining;
	215	+
	216	+ if (new_pad == 1) // cannot pad for 1 byte only
	217	+ new_pad = 9;
	218	+ else if (new_pad == 8)
	219	+ new_pad = 0;
	220	+
	221	+ return update_tlv_pad(skb, new_pad, pad_size, pad_off);
	222	+}
	223	+
	224	+// Add an Egress TLV fc00::4, add the flag A,
	225	+// and apply End.X action to fc42::1
	226	+SEC("lwt_seg6local")
	227	+int __add_egr_x(struct __sk_buff *skb)
	228	+{
	229	+ unsigned long long hi = 0xfc42000000000000;
	230	+ unsigned long long lo = 0x1;
	231	+ struct ip6_srh_t *srh = get_srh(skb);
	232	+ uint8_t new_flags = SR6_FLAG_ALERT;
	233	+ struct ip6_addr_t addr;
	234	+ int err, offset;
	235	+
	236	+ if (srh == NULL)
	237	+ return BPF_DROP;
	238	+
	239	+ uint8_t tlv[20] = {2, 18, 0, 0, 0xfd, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
	240	+ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4};
	241	+
	242	+ err = add_tlv(skb, srh, (srh->hdrlen+1) << 3,
	243	+ (struct sr6_tlv_t *)&tlv, 20);
	244	+ if (err)
	245	+ return BPF_DROP;
	246	+
	247	+ offset = sizeof(struct ip6_t) + offsetof(struct ip6_srh_t, flags);
	248	+ err = bpf_lwt_seg6_store_bytes(skb, offset,
	249	+ (void *)&new_flags, sizeof(new_flags));
	250	+ if (err)
	251	+ return BPF_DROP;
	252	+
	253	+ addr.lo = htonll(lo);
	254	+ addr.hi = htonll(hi);
	255	+ err = bpf_lwt_seg6_action(skb, SEG6_LOCAL_ACTION_END_X,
	256	+ (void *)&addr, sizeof(addr));
	257	+ if (err)
	258	+ return BPF_DROP;
	259	+ return BPF_REDIRECT;
	260	+}
	261	+char __license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_select_reuseport_kern.c+31−14

@@ -21,38 +21,55 @@ int _version SEC("version") = 1; · 20 unmodified lines
21	21	#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
22	22	#endif
23	23
24		−struct bpf_map_def SEC("maps") outer_map = {
	24	+struct {
	25	+ __u32 type;
	26	+ __u32 max_entries;
	27	+ __u32 key_size;
	28	+ __u32 value_size;
	29	+} outer_map SEC(".maps") = {
25	30	.type = BPF_MAP_TYPE_ARRAY_OF_MAPS,
	31	+ .max_entries = 1,
26	32	.key_size = sizeof(__u32),
27	33	.value_size = sizeof(__u32),
28		− .max_entries = 1,
29	34	};
30	35
31		−struct bpf_map_def SEC("maps") result_map = {
	36	+struct {
	37	+ __u32 type;
	38	+ __u32 max_entries;
	39	+ __u32 *key;
	40	+ __u32 *value;
	41	+} result_map SEC(".maps") = {
32	42	.type = BPF_MAP_TYPE_ARRAY,
33		− .key_size = sizeof(__u32),
34		− .value_size = sizeof(__u32),
35	43	.max_entries = NR_RESULTS,
36	44	};
37	45
38		−struct bpf_map_def SEC("maps") tmp_index_ovr_map = {
	46	+struct {
	47	+ __u32 type;
	48	+ __u32 max_entries;
	49	+ __u32 *key;
	50	+ int *value;
	51	+} tmp_index_ovr_map SEC(".maps") = {
39	52	.type = BPF_MAP_TYPE_ARRAY,
40		− .key_size = sizeof(__u32),
41		− .value_size = sizeof(int),
42	53	.max_entries = 1,
43	54	};
44	55
45		−struct bpf_map_def SEC("maps") linum_map = {
	56	+struct {
	57	+ __u32 type;
	58	+ __u32 max_entries;
	59	+ __u32 *key;
	60	+ __u32 *value;
	61	+} linum_map SEC(".maps") = {
46	62	.type = BPF_MAP_TYPE_ARRAY,
47		− .key_size = sizeof(__u32),
48		− .value_size = sizeof(__u32),
49	63	.max_entries = 1,
50	64	};
51	65
52		−struct bpf_map_def SEC("maps") data_check_map = {
	66	+struct {
	67	+ __u32 type;
	68	+ __u32 max_entries;
	69	+ __u32 *key;
	70	+ struct data_check *value;
	71	+} data_check_map SEC(".maps") = {
53	72	.type = BPF_MAP_TYPE_ARRAY,
54		− .key_size = sizeof(__u32),
55		− .value_size = sizeof(struct data_check),
56	73	.max_entries = 1,
57	74	};
58	75

tools/testing/selftests/bpf/progs/test_send_signal_kern.c+12−10

@@ -4,24 +4,26 @@ · 3 unmodified lines
4	4	#include <linux/version.h>
5	5	#include "bpf_helpers.h"
6	6
7		−struct bpf_map_def SEC("maps") info_map = {
	7	+struct {
	8	+ __u32 type;
	9	+ __u32 max_entries;
	10	+ __u32 *key;
	11	+ __u64 *value;
	12	+} info_map SEC(".maps") = {
8	13	.type = BPF_MAP_TYPE_ARRAY,
9		− .key_size = sizeof(__u32),
10		− .value_size = sizeof(__u64),
11	14	.max_entries = 1,
12	15	};
13	16
14		−BPF_ANNOTATE_KV_PAIR(info_map, __u32, __u64);
15		−
16		−struct bpf_map_def SEC("maps") status_map = {
	17	+struct {
	18	+ __u32 type;
	19	+ __u32 max_entries;
	20	+ __u32 *key;
	21	+ __u64 *value;
	22	+} status_map SEC(".maps") = {
17	23	.type = BPF_MAP_TYPE_ARRAY,
18		− .key_size = sizeof(__u32),
19		− .value_size = sizeof(__u64),
20	24	.max_entries = 1,
21	25	};
22	26
23		−BPF_ANNOTATE_KV_PAIR(status_map, __u32, __u64);
24		−
25	27	SEC("send_signal_demo")
26	28	int bpf_send_signal_test(void *ctx)
27	29	{

tools/testing/selftests/bpf/progs/test_sock_fields_kern.c+36−24

@@ -27,31 +27,43 @@ enum bpf_linum_array_idx { · 26 unmodified lines
27	27	__NR_BPF_LINUM_ARRAY_IDX,
28	28	};
29	29
30		−struct bpf_map_def SEC("maps") addr_map = {
	30	+struct {
	31	+ __u32 type;
	32	+ __u32 max_entries;
	33	+ __u32 *key;
	34	+ struct sockaddr_in6 *value;
	35	+} addr_map SEC(".maps") = {
31	36	.type = BPF_MAP_TYPE_ARRAY,
32		− .key_size = sizeof(__u32),
33		− .value_size = sizeof(struct sockaddr_in6),
34	37	.max_entries = __NR_BPF_ADDR_ARRAY_IDX,
35	38	};
36	39
37		−struct bpf_map_def SEC("maps") sock_result_map = {
	40	+struct {
	41	+ __u32 type;
	42	+ __u32 max_entries;
	43	+ __u32 *key;
	44	+ struct bpf_sock *value;
	45	+} sock_result_map SEC(".maps") = {
38	46	.type = BPF_MAP_TYPE_ARRAY,
39		− .key_size = sizeof(__u32),
40		− .value_size = sizeof(struct bpf_sock),
41	47	.max_entries = __NR_BPF_RESULT_ARRAY_IDX,
42	48	};
43	49
44		−struct bpf_map_def SEC("maps") tcp_sock_result_map = {
	50	+struct {
	51	+ __u32 type;
	52	+ __u32 max_entries;
	53	+ __u32 *key;
	54	+ struct bpf_tcp_sock *value;
	55	+} tcp_sock_result_map SEC(".maps") = {
45	56	.type = BPF_MAP_TYPE_ARRAY,
46		− .key_size = sizeof(__u32),
47		− .value_size = sizeof(struct bpf_tcp_sock),
48	57	.max_entries = __NR_BPF_RESULT_ARRAY_IDX,
49	58	};
50	59
51		−struct bpf_map_def SEC("maps") linum_map = {
	60	+struct {
	61	+ __u32 type;
	62	+ __u32 max_entries;
	63	+ __u32 *key;
	64	+ __u32 *value;
	65	+} linum_map SEC(".maps") = {
52	66	.type = BPF_MAP_TYPE_ARRAY,
53		− .key_size = sizeof(__u32),
54		− .value_size = sizeof(__u32),
55	67	.max_entries = __NR_BPF_LINUM_ARRAY_IDX,
56	68	};
57	69
@@ -60,26 +72,26 @@ struct bpf_spinlock_cnt { · 2 unmodified lines
60	72	__u32 cnt;
61	73	};
62	74
63		−struct bpf_map_def SEC("maps") sk_pkt_out_cnt = {
	75	+struct {
	76	+ __u32 type;
	77	+ __u32 map_flags;
	78	+ int *key;
	79	+ struct bpf_spinlock_cnt *value;
	80	+} sk_pkt_out_cnt SEC(".maps") = {
64	81	.type = BPF_MAP_TYPE_SK_STORAGE,
65		− .key_size = sizeof(int),
66		− .value_size = sizeof(struct bpf_spinlock_cnt),
67		− .max_entries = 0,
68	82	.map_flags = BPF_F_NO_PREALLOC,
69	83	};
70	84
71		−BPF_ANNOTATE_KV_PAIR(sk_pkt_out_cnt, int, struct bpf_spinlock_cnt);
72		−
73		−struct bpf_map_def SEC("maps") sk_pkt_out_cnt10 = {
	85	+struct {
	86	+ __u32 type;
	87	+ __u32 map_flags;
	88	+ int *key;
	89	+ struct bpf_spinlock_cnt *value;
	90	+} sk_pkt_out_cnt10 SEC(".maps") = {
74	91	.type = BPF_MAP_TYPE_SK_STORAGE,
75		− .key_size = sizeof(int),
76		− .value_size = sizeof(struct bpf_spinlock_cnt),
77		− .max_entries = 0,
78	92	.map_flags = BPF_F_NO_PREALLOC,
79	93	};
80	94
81		−BPF_ANNOTATE_KV_PAIR(sk_pkt_out_cnt10, int, struct bpf_spinlock_cnt);
82		−
83	95	static bool is_loopback6(__u32 *a6)
84	96	{
85	97	return !a6[0] && !a6[1] && !a6[2] && a6[3] == bpf_htonl(1);

tools/testing/selftests/bpf/progs/test_spin_lock.c+17−16

@@ -10,30 +10,29 @@ struct hmap_elem { · 9 unmodified lines
10	10	int test_padding;
11	11	};
12	12
13		−struct bpf_map_def SEC("maps") hmap = {
	13	+struct {
	14	+ __u32 type;
	15	+ __u32 max_entries;
	16	+ int *key;
	17	+ struct hmap_elem *value;
	18	+} hmap SEC(".maps") = {
14	19	.type = BPF_MAP_TYPE_HASH,
15		− .key_size = sizeof(int),
16		− .value_size = sizeof(struct hmap_elem),
17	20	.max_entries = 1,
18	21	};
19	22
20		−BPF_ANNOTATE_KV_PAIR(hmap, int, struct hmap_elem);
21		−
22		−
23	23	struct cls_elem {
24	24	struct bpf_spin_lock lock;
25	25	volatile int cnt;
26	26	};
27	27
28		−struct bpf_map_def SEC("maps") cls_map = {
	28	+struct {
	29	+ __u32 type;
	30	+ struct bpf_cgroup_storage_key *key;
	31	+ struct cls_elem *value;
	32	+} cls_map SEC(".maps") = {
29	33	.type = BPF_MAP_TYPE_CGROUP_STORAGE,
30		− .key_size = sizeof(struct bpf_cgroup_storage_key),
31		− .value_size = sizeof(struct cls_elem),
32	34	};
33	35
34		−BPF_ANNOTATE_KV_PAIR(cls_map, struct bpf_cgroup_storage_key,
35		− struct cls_elem);
36		−
37	36	struct bpf_vqueue {
38	37	struct bpf_spin_lock lock;
39	38	/* 4 byte hole */
@@ -42,14 +41,16 @@ struct bpf_vqueue { · 2 unmodified lines
42	41	unsigned int rate;
43	42	};
44	43
45		−struct bpf_map_def SEC("maps") vqueue = {
	44	+struct {
	45	+ __u32 type;
	46	+ __u32 max_entries;
	47	+ int *key;
	48	+ struct bpf_vqueue *value;
	49	+} vqueue SEC(".maps") = {
46	50	.type = BPF_MAP_TYPE_ARRAY,
47		− .key_size = sizeof(int),
48		− .value_size = sizeof(struct bpf_vqueue),
49	51	.max_entries = 1,
50	52	};
51	53
52		−BPF_ANNOTATE_KV_PAIR(vqueue, int, struct bpf_vqueue);
53	54	#define CREDIT_PER_NS(delta, rate) (((delta) * rate) >> 20)
54	55
55	56	SEC("spin_lock_demo")

tools/testing/selftests/bpf/progs/test_stacktrace_build_id.c+30−14

@@ -8,34 +8,50 @@ · 7 unmodified lines
8	8	#define PERF_MAX_STACK_DEPTH 127
9	9	#endif
10	10
11		−struct bpf_map_def SEC("maps") control_map = {
	11	+struct {
	12	+ __u32 type;
	13	+ __u32 max_entries;
	14	+ __u32 *key;
	15	+ __u32 *value;
	16	+} control_map SEC(".maps") = {
12	17	.type = BPF_MAP_TYPE_ARRAY,
13		− .key_size = sizeof(__u32),
14		− .value_size = sizeof(__u32),
15	18	.max_entries = 1,
16	19	};
17	20
18		−struct bpf_map_def SEC("maps") stackid_hmap = {
	21	+struct {
	22	+ __u32 type;
	23	+ __u32 max_entries;
	24	+ __u32 *key;
	25	+ __u32 *value;
	26	+} stackid_hmap SEC(".maps") = {
19	27	.type = BPF_MAP_TYPE_HASH,
20		− .key_size = sizeof(__u32),
21		− .value_size = sizeof(__u32),
22	28	.max_entries = 16384,
23	29	};
24	30
25		−struct bpf_map_def SEC("maps") stackmap = {
	31	+typedef struct bpf_stack_build_id stack_trace_t[PERF_MAX_STACK_DEPTH];
	32	+
	33	+struct {
	34	+ __u32 type;
	35	+ __u32 max_entries;
	36	+ __u32 map_flags;
	37	+ __u32 key_size;
	38	+ __u32 value_size;
	39	+} stackmap SEC(".maps") = {
26	40	.type = BPF_MAP_TYPE_STACK_TRACE,
27		− .key_size = sizeof(__u32),
28		− .value_size = sizeof(struct bpf_stack_build_id)
29		− * PERF_MAX_STACK_DEPTH,
30	41	.max_entries = 128,
31	42	.map_flags = BPF_F_STACK_BUILD_ID,
	43	+ .key_size = sizeof(__u32),
	44	+ .value_size = sizeof(stack_trace_t),
32	45	};
33	46
34		−struct bpf_map_def SEC("maps") stack_amap = {
	47	+struct {
	48	+ __u32 type;
	49	+ __u32 max_entries;
	50	+ __u32 *key;
	51	+ /* there seems to be a bug in kernel not handling typedef properly */
	52	+ struct bpf_stack_build_id (*value)[PERF_MAX_STACK_DEPTH];
	53	+} stack_amap SEC(".maps") = {
35	54	.type = BPF_MAP_TYPE_ARRAY,
36		− .key_size = sizeof(__u32),
37		− .value_size = sizeof(struct bpf_stack_build_id)
38		− * PERF_MAX_STACK_DEPTH,
39	55	.max_entries = 128,
40	56	};
41	57

tools/testing/selftests/bpf/progs/test_stacktrace_map.c+28−12

@@ -8,31 +8,47 @@ · 7 unmodified lines
8	8	#define PERF_MAX_STACK_DEPTH 127
9	9	#endif
10	10
11		−struct bpf_map_def SEC("maps") control_map = {
	11	+struct {
	12	+ __u32 type;
	13	+ __u32 max_entries;
	14	+ __u32 *key;
	15	+ __u32 *value;
	16	+} control_map SEC(".maps") = {
12	17	.type = BPF_MAP_TYPE_ARRAY,
13		− .key_size = sizeof(__u32),
14		− .value_size = sizeof(__u32),
15	18	.max_entries = 1,
16	19	};
17	20
18		−struct bpf_map_def SEC("maps") stackid_hmap = {
	21	+struct {
	22	+ __u32 type;
	23	+ __u32 max_entries;
	24	+ __u32 *key;
	25	+ __u32 *value;
	26	+} stackid_hmap SEC(".maps") = {
19	27	.type = BPF_MAP_TYPE_HASH,
20		− .key_size = sizeof(__u32),
21		− .value_size = sizeof(__u32),
22	28	.max_entries = 16384,
23	29	};
24	30
25		−struct bpf_map_def SEC("maps") stackmap = {
	31	+typedef __u64 stack_trace_t[PERF_MAX_STACK_DEPTH];
	32	+
	33	+struct {
	34	+ __u32 type;
	35	+ __u32 max_entries;
	36	+ __u32 key_size;
	37	+ __u32 value_size;
	38	+} stackmap SEC(".maps") = {
26	39	.type = BPF_MAP_TYPE_STACK_TRACE,
27		− .key_size = sizeof(__u32),
28		− .value_size = sizeof(__u64) * PERF_MAX_STACK_DEPTH,
29	40	.max_entries = 16384,
	41	+ .key_size = sizeof(__u32),
	42	+ .value_size = sizeof(stack_trace_t),
30	43	};
31	44
32		−struct bpf_map_def SEC("maps") stack_amap = {
	45	+struct {
	46	+ __u32 type;
	47	+ __u32 max_entries;
	48	+ __u32 *key;
	49	+ __u64 (*value)[PERF_MAX_STACK_DEPTH];
	50	+} stack_amap SEC(".maps") = {
33	51	.type = BPF_MAP_TYPE_ARRAY,
34		− .key_size = sizeof(__u32),
35		− .value_size = sizeof(__u64) * PERF_MAX_STACK_DEPTH,
36	52	.max_entries = 16384,
37	53	};
38	54

tools/testing/selftests/bpf/progs/test_sysctl_loop1.c+71−0

@@ -0,0 +1,71 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+
	4	+#include <stdint.h>
	5	+#include <string.h>
	6	+
	7	+#include <linux/stddef.h>
	8	+#include <linux/bpf.h>
	9	+
	10	+#include "bpf_helpers.h"
	11	+
	12	+#ifndef ARRAY_SIZE
	13	+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
	14	+#endif
	15	+
	16	+/* tcp_mem sysctl has only 3 ints, but this test is doing TCP_MEM_LOOPS */
	17	+#define TCP_MEM_LOOPS 28 /* because 30 doesn't fit into 512 bytes of stack */
	18	+#define MAX_ULONG_STR_LEN 7
	19	+#define MAX_VALUE_STR_LEN (TCP_MEM_LOOPS * MAX_ULONG_STR_LEN)
	20	+
	21	+static __always_inline int is_tcp_mem(struct bpf_sysctl *ctx)
	22	+{
	23	+ volatile char tcp_mem_name[] = "net/ipv4/tcp_mem/very_very_very_very_long_pointless_string";
	24	+ unsigned char i;
	25	+ char name[64];
	26	+ int ret;
	27	+
	28	+ memset(name, 0, sizeof(name));
	29	+ ret = bpf_sysctl_get_name(ctx, name, sizeof(name), 0);
	30	+ if (ret < 0 \|\| ret != sizeof(tcp_mem_name) - 1)
	31	+ return 0;
	32	+
	33	+#pragma clang loop unroll(disable)
	34	+ for (i = 0; i < sizeof(tcp_mem_name); ++i)
	35	+ if (name[i] != tcp_mem_name[i])
	36	+ return 0;
	37	+
	38	+ return 1;
	39	+}
	40	+
	41	+SEC("cgroup/sysctl")
	42	+int sysctl_tcp_mem(struct bpf_sysctl *ctx)
	43	+{
	44	+ unsigned long tcp_mem[TCP_MEM_LOOPS] = {};
	45	+ char value[MAX_VALUE_STR_LEN];
	46	+ unsigned char i, off = 0;
	47	+ int ret;
	48	+
	49	+ if (ctx->write)
	50	+ return 0;
	51	+
	52	+ if (!is_tcp_mem(ctx))
	53	+ return 0;
	54	+
	55	+ ret = bpf_sysctl_get_current_value(ctx, value, MAX_VALUE_STR_LEN);
	56	+ if (ret < 0 \|\| ret >= MAX_VALUE_STR_LEN)
	57	+ return 0;
	58	+
	59	+#pragma clang loop unroll(disable)
	60	+ for (i = 0; i < ARRAY_SIZE(tcp_mem); ++i) {
	61	+ ret = bpf_strtoul(value + off, MAX_ULONG_STR_LEN, 0,
	62	+ tcp_mem + i);
	63	+ if (ret <= 0 \|\| ret > MAX_ULONG_STR_LEN)
	64	+ return 0;
	65	+ off += ret & MAX_ULONG_STR_LEN;
	66	+ }
	67	+
	68	+ return tcp_mem[0] < tcp_mem[1] && tcp_mem[1] < tcp_mem[2];
	69	+}
	70	+
	71	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_sysctl_loop2.c+72−0

@@ -0,0 +1,72 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+
	4	+#include <stdint.h>
	5	+#include <string.h>
	6	+
	7	+#include <linux/stddef.h>
	8	+#include <linux/bpf.h>
	9	+
	10	+#include "bpf_helpers.h"
	11	+
	12	+#ifndef ARRAY_SIZE
	13	+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
	14	+#endif
	15	+
	16	+/* tcp_mem sysctl has only 3 ints, but this test is doing TCP_MEM_LOOPS */
	17	+#define TCP_MEM_LOOPS 20 /* because 30 doesn't fit into 512 bytes of stack */
	18	+#define MAX_ULONG_STR_LEN 7
	19	+#define MAX_VALUE_STR_LEN (TCP_MEM_LOOPS * MAX_ULONG_STR_LEN)
	20	+
	21	+static __attribute__((noinline)) int is_tcp_mem(struct bpf_sysctl *ctx)
	22	+{
	23	+ volatile char tcp_mem_name[] = "net/ipv4/tcp_mem/very_very_very_very_long_pointless_string_to_stress_byte_loop";
	24	+ unsigned char i;
	25	+ char name[64];
	26	+ int ret;
	27	+
	28	+ memset(name, 0, sizeof(name));
	29	+ ret = bpf_sysctl_get_name(ctx, name, sizeof(name), 0);
	30	+ if (ret < 0 \|\| ret != sizeof(tcp_mem_name) - 1)
	31	+ return 0;
	32	+
	33	+#pragma clang loop unroll(disable)
	34	+ for (i = 0; i < sizeof(tcp_mem_name); ++i)
	35	+ if (name[i] != tcp_mem_name[i])
	36	+ return 0;
	37	+
	38	+ return 1;
	39	+}
	40	+
	41	+
	42	+SEC("cgroup/sysctl")
	43	+int sysctl_tcp_mem(struct bpf_sysctl *ctx)
	44	+{
	45	+ unsigned long tcp_mem[TCP_MEM_LOOPS] = {};
	46	+ char value[MAX_VALUE_STR_LEN];
	47	+ unsigned char i, off = 0;
	48	+ int ret;
	49	+
	50	+ if (ctx->write)
	51	+ return 0;
	52	+
	53	+ if (!is_tcp_mem(ctx))
	54	+ return 0;
	55	+
	56	+ ret = bpf_sysctl_get_current_value(ctx, value, MAX_VALUE_STR_LEN);
	57	+ if (ret < 0 \|\| ret >= MAX_VALUE_STR_LEN)
	58	+ return 0;
	59	+
	60	+#pragma clang loop unroll(disable)
	61	+ for (i = 0; i < ARRAY_SIZE(tcp_mem); ++i) {
	62	+ ret = bpf_strtoul(value + off, MAX_ULONG_STR_LEN, 0,
	63	+ tcp_mem + i);
	64	+ if (ret <= 0 \|\| ret > MAX_ULONG_STR_LEN)
	65	+ return 0;
	66	+ off += ret & MAX_ULONG_STR_LEN;
	67	+ }
	68	+
	69	+ return tcp_mem[0] < tcp_mem[1] && tcp_mem[1] < tcp_mem[2];
	70	+}
	71	+
	72	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_sysctl_prog.c+4−1

@@ -8,7 +8,6 @@ · 7 unmodified lines
8	8	#include <linux/bpf.h>
9	9
10	10	#include "bpf_helpers.h"
11		−#include "bpf_util.h"
12	11
13	12	/* Max supported length of a string with unsigned long in base 10 (pow2 - 1). */
14	13	#define MAX_ULONG_STR_LEN 0xF
@@ -16,6 +15,10 @@ · 1 unmodified line
16	15	/* Max supported length of sysctl value string (pow2). */
17	16	#define MAX_VALUE_STR_LEN 0x40
18	17
	18	+#ifndef ARRAY_SIZE
	19	+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
	20	+#endif
	21	+
19	22	static __always_inline int is_tcp_mem(struct bpf_sysctl *ctx)
20	23	{
21	24	char tcp_mem_name[] = "net/ipv4/tcp_mem";

tools/testing/selftests/bpf/progs/test_tcp_estats.c+6−3

@@ -148,10 +148,13 @@ struct tcp_estats_basic_event { · 147 unmodified lines
148	148	struct tcp_estats_conn_id conn_id;
149	149	};
150	150
151		−struct bpf_map_def SEC("maps") ev_record_map = {
	151	+struct {
	152	+ __u32 type;
	153	+ __u32 max_entries;
	154	+ __u32 *key;
	155	+ struct tcp_estats_basic_event *value;
	156	+} ev_record_map SEC(".maps") = {
152	157	.type = BPF_MAP_TYPE_HASH,
153		− .key_size = sizeof(__u32),
154		− .value_size = sizeof(struct tcp_estats_basic_event),
155	158	.max_entries = 1024,
156	159	};
157	160

tools/testing/selftests/bpf/progs/test_tcpbpf_kern.c+12−6

@@ -14,17 +14,23 @@ · 13 unmodified lines
14	14	#include "bpf_endian.h"
15	15	#include "test_tcpbpf.h"
16	16
17		−struct bpf_map_def SEC("maps") global_map = {
	17	+struct {
	18	+ __u32 type;
	19	+ __u32 max_entries;
	20	+ __u32 *key;
	21	+ struct tcpbpf_globals *value;
	22	+} global_map SEC(".maps") = {
18	23	.type = BPF_MAP_TYPE_ARRAY,
19		− .key_size = sizeof(__u32),
20		− .value_size = sizeof(struct tcpbpf_globals),
21	24	.max_entries = 4,
22	25	};
23	26
24		−struct bpf_map_def SEC("maps") sockopt_results = {
	27	+struct {
	28	+ __u32 type;
	29	+ __u32 max_entries;
	30	+ __u32 *key;
	31	+ int *value;
	32	+} sockopt_results SEC(".maps") = {
25	33	.type = BPF_MAP_TYPE_ARRAY,
26		− .key_size = sizeof(__u32),
27		− .value_size = sizeof(int),
28	34	.max_entries = 2,
29	35	};
30	36

tools/testing/selftests/bpf/progs/test_tcpnotify_kern.c+13−5

@@ -14,18 +14,26 @@ · 13 unmodified lines
14	14	#include "bpf_endian.h"
15	15	#include "test_tcpnotify.h"
16	16
17		−struct bpf_map_def SEC("maps") global_map = {
	17	+struct {
	18	+ __u32 type;
	19	+ __u32 max_entries;
	20	+ __u32 *key;
	21	+ struct tcpnotify_globals *value;
	22	+} global_map SEC(".maps") = {
18	23	.type = BPF_MAP_TYPE_ARRAY,
19		− .key_size = sizeof(__u32),
20		− .value_size = sizeof(struct tcpnotify_globals),
21	24	.max_entries = 4,
22	25	};
23	26
24		−struct bpf_map_def SEC("maps") perf_event_map = {
	27	+struct {
	28	+ __u32 type;
	29	+ __u32 max_entries;
	30	+ __u32 key_size;
	31	+ __u32 value_size;
	32	+} perf_event_map SEC(".maps") = {
25	33	.type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
	34	+ .max_entries = 2,
26	35	.key_size = sizeof(int),
27	36	.value_size = sizeof(__u32),
28		− .max_entries = 2,
29	37	};
30	38
31	39	int _version SEC("version") = 1;

tools/testing/selftests/bpf/progs/test_xdp.c+12−6

@@ -22,17 +22,23 @@ · 21 unmodified lines
22	22
23	23	int _version SEC("version") = 1;
24	24
25		−struct bpf_map_def SEC("maps") rxcnt = {
	25	+struct {
	26	+ __u32 type;
	27	+ __u32 max_entries;
	28	+ __u32 *key;
	29	+ __u64 *value;
	30	+} rxcnt SEC(".maps") = {
26	31	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
27		− .key_size = sizeof(__u32),
28		− .value_size = sizeof(__u64),
29	32	.max_entries = 256,
30	33	};
31	34
32		−struct bpf_map_def SEC("maps") vip2tnl = {
	35	+struct {
	36	+ __u32 type;
	37	+ __u32 max_entries;
	38	+ struct vip *key;
	39	+ struct iptnl_info *value;
	40	+} vip2tnl SEC(".maps") = {
33	41	.type = BPF_MAP_TYPE_HASH,
34		− .key_size = sizeof(struct vip),
35		− .value_size = sizeof(struct iptnl_info),
36	42	.max_entries = MAX_IPTNL_ENTRIES,
37	43	};
38	44

tools/testing/selftests/bpf/progs/test_xdp_loop.c+231−0

@@ -0,0 +1,231 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+// Copyright (c) 2019 Facebook
	3	+#include <stddef.h>
	4	+#include <string.h>
	5	+#include <linux/bpf.h>
	6	+#include <linux/if_ether.h>
	7	+#include <linux/if_packet.h>
	8	+#include <linux/ip.h>
	9	+#include <linux/ipv6.h>
	10	+#include <linux/in.h>
	11	+#include <linux/udp.h>
	12	+#include <linux/tcp.h>
	13	+#include <linux/pkt_cls.h>
	14	+#include <sys/socket.h>
	15	+#include "bpf_helpers.h"
	16	+#include "bpf_endian.h"
	17	+#include "test_iptunnel_common.h"
	18	+
	19	+int _version SEC("version") = 1;
	20	+
	21	+struct bpf_map_def SEC("maps") rxcnt = {
	22	+ .type = BPF_MAP_TYPE_PERCPU_ARRAY,
	23	+ .key_size = sizeof(__u32),
	24	+ .value_size = sizeof(__u64),
	25	+ .max_entries = 256,
	26	+};
	27	+
	28	+struct bpf_map_def SEC("maps") vip2tnl = {
	29	+ .type = BPF_MAP_TYPE_HASH,
	30	+ .key_size = sizeof(struct vip),
	31	+ .value_size = sizeof(struct iptnl_info),
	32	+ .max_entries = MAX_IPTNL_ENTRIES,
	33	+};
	34	+
	35	+static __always_inline void count_tx(__u32 protocol)
	36	+{
	37	+ __u64 *rxcnt_count;
	38	+
	39	+ rxcnt_count = bpf_map_lookup_elem(&rxcnt, &protocol);
	40	+ if (rxcnt_count)
	41	+ *rxcnt_count += 1;
	42	+}
	43	+
	44	+static __always_inline int get_dport(void trans_data, void data_end,
	45	+ __u8 protocol)
	46	+{
	47	+ struct tcphdr *th;
	48	+ struct udphdr *uh;
	49	+
	50	+ switch (protocol) {
	51	+ case IPPROTO_TCP:
	52	+ th = (struct tcphdr *)trans_data;
	53	+ if (th + 1 > data_end)
	54	+ return -1;
	55	+ return th->dest;
	56	+ case IPPROTO_UDP:
	57	+ uh = (struct udphdr *)trans_data;
	58	+ if (uh + 1 > data_end)
	59	+ return -1;
	60	+ return uh->dest;
	61	+ default:
	62	+ return 0;
	63	+ }
	64	+}
	65	+
	66	+static __always_inline void set_ethhdr(struct ethhdr *new_eth,
	67	+ const struct ethhdr *old_eth,
	68	+ const struct iptnl_info *tnl,
	69	+ __be16 h_proto)
	70	+{
	71	+ memcpy(new_eth->h_source, old_eth->h_dest, sizeof(new_eth->h_source));
	72	+ memcpy(new_eth->h_dest, tnl->dmac, sizeof(new_eth->h_dest));
	73	+ new_eth->h_proto = h_proto;
	74	+}
	75	+
	76	+static __always_inline int handle_ipv4(struct xdp_md *xdp)
	77	+{
	78	+ void data_end = (void )(long)xdp->data_end;
	79	+ void data = (void )(long)xdp->data;
	80	+ struct iptnl_info *tnl;
	81	+ struct ethhdr *new_eth;
	82	+ struct ethhdr *old_eth;
	83	+ struct iphdr *iph = data + sizeof(struct ethhdr);
	84	+ __u16 *next_iph;
	85	+ __u16 payload_len;
	86	+ struct vip vip = {};
	87	+ int dport;
	88	+ __u32 csum = 0;
	89	+ int i;
	90	+
	91	+ if (iph + 1 > data_end)
	92	+ return XDP_DROP;
	93	+
	94	+ dport = get_dport(iph + 1, data_end, iph->protocol);
	95	+ if (dport == -1)
	96	+ return XDP_DROP;
	97	+
	98	+ vip.protocol = iph->protocol;
	99	+ vip.family = AF_INET;
	100	+ vip.daddr.v4 = iph->daddr;
	101	+ vip.dport = dport;
	102	+ payload_len = bpf_ntohs(iph->tot_len);
	103	+
	104	+ tnl = bpf_map_lookup_elem(&vip2tnl, &vip);
	105	+ /* It only does v4-in-v4 */
	106	+ if (!tnl \|\| tnl->family != AF_INET)
	107	+ return XDP_PASS;
	108	+
	109	+ if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct iphdr)))
	110	+ return XDP_DROP;
	111	+
	112	+ data = (void *)(long)xdp->data;
	113	+ data_end = (void *)(long)xdp->data_end;
	114	+
	115	+ new_eth = data;
	116	+ iph = data + sizeof(*new_eth);
	117	+ old_eth = data + sizeof(*iph);
	118	+
	119	+ if (new_eth + 1 > data_end \|\|
	120	+ old_eth + 1 > data_end \|\|
	121	+ iph + 1 > data_end)
	122	+ return XDP_DROP;
	123	+
	124	+ set_ethhdr(new_eth, old_eth, tnl, bpf_htons(ETH_P_IP));
	125	+
	126	+ iph->version = 4;
	127	+ iph->ihl = sizeof(*iph) >> 2;
	128	+ iph->frag_off = 0;
	129	+ iph->protocol = IPPROTO_IPIP;
	130	+ iph->check = 0;
	131	+ iph->tos = 0;
	132	+ iph->tot_len = bpf_htons(payload_len + sizeof(*iph));
	133	+ iph->daddr = tnl->daddr.v4;
	134	+ iph->saddr = tnl->saddr.v4;
	135	+ iph->ttl = 8;
	136	+
	137	+ next_iph = (__u16 *)iph;
	138	+#pragma clang loop unroll(disable)
	139	+ for (i = 0; i < sizeof(*iph) >> 1; i++)
	140	+ csum += *next_iph++;
	141	+
	142	+ iph->check = ~((csum & 0xffff) + (csum >> 16));
	143	+
	144	+ count_tx(vip.protocol);
	145	+
	146	+ return XDP_TX;
	147	+}
	148	+
	149	+static __always_inline int handle_ipv6(struct xdp_md *xdp)
	150	+{
	151	+ void data_end = (void )(long)xdp->data_end;
	152	+ void data = (void )(long)xdp->data;
	153	+ struct iptnl_info *tnl;
	154	+ struct ethhdr *new_eth;
	155	+ struct ethhdr *old_eth;
	156	+ struct ipv6hdr *ip6h = data + sizeof(struct ethhdr);
	157	+ __u16 payload_len;
	158	+ struct vip vip = {};
	159	+ int dport;
	160	+
	161	+ if (ip6h + 1 > data_end)
	162	+ return XDP_DROP;
	163	+
	164	+ dport = get_dport(ip6h + 1, data_end, ip6h->nexthdr);
	165	+ if (dport == -1)
	166	+ return XDP_DROP;
	167	+
	168	+ vip.protocol = ip6h->nexthdr;
	169	+ vip.family = AF_INET6;
	170	+ memcpy(vip.daddr.v6, ip6h->daddr.s6_addr32, sizeof(vip.daddr));
	171	+ vip.dport = dport;
	172	+ payload_len = ip6h->payload_len;
	173	+
	174	+ tnl = bpf_map_lookup_elem(&vip2tnl, &vip);
	175	+ /* It only does v6-in-v6 */
	176	+ if (!tnl \|\| tnl->family != AF_INET6)
	177	+ return XDP_PASS;
	178	+
	179	+ if (bpf_xdp_adjust_head(xdp, 0 - (int)sizeof(struct ipv6hdr)))
	180	+ return XDP_DROP;
	181	+
	182	+ data = (void *)(long)xdp->data;
	183	+ data_end = (void *)(long)xdp->data_end;
	184	+
	185	+ new_eth = data;
	186	+ ip6h = data + sizeof(*new_eth);
	187	+ old_eth = data + sizeof(*ip6h);
	188	+
	189	+ if (new_eth + 1 > data_end \|\| old_eth + 1 > data_end \|\|
	190	+ ip6h + 1 > data_end)
	191	+ return XDP_DROP;
	192	+
	193	+ set_ethhdr(new_eth, old_eth, tnl, bpf_htons(ETH_P_IPV6));
	194	+
	195	+ ip6h->version = 6;
	196	+ ip6h->priority = 0;
	197	+ memset(ip6h->flow_lbl, 0, sizeof(ip6h->flow_lbl));
	198	+ ip6h->payload_len = bpf_htons(bpf_ntohs(payload_len) + sizeof(*ip6h));
	199	+ ip6h->nexthdr = IPPROTO_IPV6;
	200	+ ip6h->hop_limit = 8;
	201	+ memcpy(ip6h->saddr.s6_addr32, tnl->saddr.v6, sizeof(tnl->saddr.v6));
	202	+ memcpy(ip6h->daddr.s6_addr32, tnl->daddr.v6, sizeof(tnl->daddr.v6));
	203	+
	204	+ count_tx(vip.protocol);
	205	+
	206	+ return XDP_TX;
	207	+}
	208	+
	209	+SEC("xdp_tx_iptunnel")
	210	+int _xdp_tx_iptunnel(struct xdp_md *xdp)
	211	+{
	212	+ void data_end = (void )(long)xdp->data_end;
	213	+ void data = (void )(long)xdp->data;
	214	+ struct ethhdr *eth = data;
	215	+ __u16 h_proto;
	216	+
	217	+ if (eth + 1 > data_end)
	218	+ return XDP_DROP;
	219	+
	220	+ h_proto = eth->h_proto;
	221	+
	222	+ if (h_proto == bpf_htons(ETH_P_IP))
	223	+ return handle_ipv4(xdp);
	224	+ else if (h_proto == bpf_htons(ETH_P_IPV6))
	225	+
	226	+ return handle_ipv6(xdp);
	227	+ else
	228	+ return XDP_DROP;
	229	+}
	230	+
	231	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_xdp_noinline.c+37−23

@@ -163,52 +163,66 @@ struct lb_stats { · 162 unmodified lines
163	163	__u64 v1;
164	164	};
165	165
166		−struct bpf_map_def __attribute__ ((section("maps"), used)) vip_map = {
	166	+struct {
	167	+ __u32 type;
	168	+ __u32 max_entries;
	169	+ struct vip_definition *key;
	170	+ struct vip_meta *value;
	171	+} vip_map SEC(".maps") = {
167	172	.type = BPF_MAP_TYPE_HASH,
168		− .key_size = sizeof(struct vip_definition),
169		− .value_size = sizeof(struct vip_meta),
170	173	.max_entries = 512,
171		− .map_flags = 0,
172	174	};
173	175
174		−struct bpf_map_def __attribute__ ((section("maps"), used)) lru_cache = {
	176	+struct {
	177	+ __u32 type;
	178	+ __u32 max_entries;
	179	+ __u32 map_flags;
	180	+ struct flow_key *key;
	181	+ struct real_pos_lru *value;
	182	+} lru_cache SEC(".maps") = {
175	183	.type = BPF_MAP_TYPE_LRU_HASH,
176		− .key_size = sizeof(struct flow_key),
177		− .value_size = sizeof(struct real_pos_lru),
178	184	.max_entries = 300,
179	185	.map_flags = 1U << 1,
180	186	};
181	187
182		−struct bpf_map_def __attribute__ ((section("maps"), used)) ch_rings = {
	188	+struct {
	189	+ __u32 type;
	190	+ __u32 max_entries;
	191	+ __u32 *key;
	192	+ __u32 *value;
	193	+} ch_rings SEC(".maps") = {
183	194	.type = BPF_MAP_TYPE_ARRAY,
184		− .key_size = sizeof(__u32),
185		− .value_size = sizeof(__u32),
186	195	.max_entries = 12 * 655,
187		− .map_flags = 0,
188	196	};
189	197
190		−struct bpf_map_def __attribute__ ((section("maps"), used)) reals = {
	198	+struct {
	199	+ __u32 type;
	200	+ __u32 max_entries;
	201	+ __u32 *key;
	202	+ struct real_definition *value;
	203	+} reals SEC(".maps") = {
191	204	.type = BPF_MAP_TYPE_ARRAY,
192		− .key_size = sizeof(__u32),
193		− .value_size = sizeof(struct real_definition),
194	205	.max_entries = 40,
195		− .map_flags = 0,
196	206	};
197	207
198		−struct bpf_map_def __attribute__ ((section("maps"), used)) stats = {
	208	+struct {
	209	+ __u32 type;
	210	+ __u32 max_entries;
	211	+ __u32 *key;
	212	+ struct lb_stats *value;
	213	+} stats SEC(".maps") = {
199	214	.type = BPF_MAP_TYPE_PERCPU_ARRAY,
200		− .key_size = sizeof(__u32),
201		− .value_size = sizeof(struct lb_stats),
202	215	.max_entries = 515,
203		− .map_flags = 0,
204	216	};
205	217
206		−struct bpf_map_def __attribute__ ((section("maps"), used)) ctl_array = {
	218	+struct {
	219	+ __u32 type;
	220	+ __u32 max_entries;
	221	+ __u32 *key;
	222	+ struct ctl_value *value;
	223	+} ctl_array SEC(".maps") = {
207	224	.type = BPF_MAP_TYPE_ARRAY,
208		− .key_size = sizeof(__u32),
209		− .value_size = sizeof(struct ctl_value),
210	225	.max_entries = 16,
211		− .map_flags = 0,
212	226	};
213	227
214	228	struct eth_hdr {

tools/testing/selftests/bpf/test_btf.c+3−7

@@ -4016,13 +4016,9 @@ struct btf_file_test { · 4015 unmodified lines
4016	4016	};
4017	4017
4018	4018	static struct btf_file_test file_tests[] = {
4019		−{
4020		− .file = "test_btf_haskv.o",
4021		−},
4022		−{
4023		− .file = "test_btf_nokv.o",
4024		− .btf_kv_notfound = true,
4025		−},
	4019	+ { .file = "test_btf_haskv.o", },
	4020	+ { .file = "test_btf_newkv.o", },
	4021	+ { .file = "test_btf_nokv.o", .btf_kv_notfound = true, },
4026	4022	};
4027	4023
4028	4024	static int do_test_file(unsigned int test_num)

tools/testing/selftests/bpf/test_select_reuseport.c+54−0

@@ -523,6 +523,58 @@ static void test_pass_on_err(int type, sa_family_t family) · 522 unmodified lines
523	523	printf("OK\n");
524	524	}
525	525
	526	+static void test_detach_bpf(int type, sa_family_t family)
	527	+{
	528	+#ifdef SO_DETACH_REUSEPORT_BPF
	529	+ __u32 nr_run_before = 0, nr_run_after = 0, tmp, i;
	530	+ struct epoll_event ev;
	531	+ int cli_fd, err, nev;
	532	+ struct cmd cmd = {};
	533	+ int optvalue = 0;
	534	+
	535	+ printf("%s: ", __func__);
	536	+ err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
	537	+ &optvalue, sizeof(optvalue));
	538	+ CHECK(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
	539	+ "err:%d errno:%d\n", err, errno);
	540	+
	541	+ err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
	542	+ &optvalue, sizeof(optvalue));
	543	+ CHECK(err == 0 \|\| errno != ENOENT, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
	544	+ "err:%d errno:%d\n", err, errno);
	545	+
	546	+ for (i = 0; i < NR_RESULTS; i++) {
	547	+ err = bpf_map_lookup_elem(result_map, &i, &tmp);
	548	+ CHECK(err == -1, "lookup_elem(result_map)",
	549	+ "i:%u err:%d errno:%d\n", i, err, errno);
	550	+ nr_run_before += tmp;
	551	+ }
	552	+
	553	+ cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS);
	554	+ nev = epoll_wait(epfd, &ev, 1, 5);
	555	+ CHECK(nev <= 0, "nev <= 0",
	556	+ "nev:%d expected:1 type:%d family:%d data:(0, 0)\n",
	557	+ nev, type, family);
	558	+
	559	+ for (i = 0; i < NR_RESULTS; i++) {
	560	+ err = bpf_map_lookup_elem(result_map, &i, &tmp);
	561	+ CHECK(err == -1, "lookup_elem(result_map)",
	562	+ "i:%u err:%d errno:%d\n", i, err, errno);
	563	+ nr_run_after += tmp;
	564	+ }
	565	+
	566	+ CHECK(nr_run_before != nr_run_after,
	567	+ "nr_run_before != nr_run_after",
	568	+ "nr_run_before:%u nr_run_after:%u\n",
	569	+ nr_run_before, nr_run_after);
	570	+
	571	+ printf("OK\n");
	572	+ close(cli_fd);
	573	+#else
	574	+ printf("%s: SKIP\n", __func__);
	575	+#endif
	576	+}
	577	+
526	578	static void prepare_sk_fds(int type, sa_family_t family, bool inany)
527	579	{
528	580	const int first = REUSEPORT_ARRAY_SIZE - 1;
@@ -664,6 +716,8 @@ static void test_all(void) · 135 unmodified lines
664	716	test_pass(type, family);
665	717	test_syncookie(type, family);
666	718	test_pass_on_err(type, family);
	719	+ /* Must be the last test */
	720	+ test_detach_bpf(type, family);
667	721
668	722	cleanup_per_test();
669	723	printf("\n");

tools/testing/selftests/bpf/test_socket_cookie.c+9−15

@@ -18,6 +18,11 @@ · 17 unmodified lines
18	18	#define CG_PATH "/foo"
19	19	#define SOCKET_COOKIE_PROG "./socket_cookie_prog.o"
20	20
	21	+struct socket_cookie {
	22	+ __u64 cookie_key;
	23	+ __u32 cookie_value;
	24	+};
	25	+
21	26	static int start_server(void)
22	27	{
23	28	struct sockaddr_in6 addr;
@@ -89,8 +94,7 @@ static int validate_map(struct bpf_map *map, int client_fd) · 65 unmodified lines
89	94	__u32 cookie_expected_value;
90	95	struct sockaddr_in6 addr;
91	96	socklen_t len = sizeof(addr);
92		− __u32 cookie_value;
93		− __u64 cookie_key;
	97	+ struct socket_cookie val;
94	98	int err = 0;
95	99	int map_fd;
96	100
@@ -101,17 +105,7 @@ static int validate_map(struct bpf_map *map, int client_fd) · 4 unmodified lines
101	105
102	106	map_fd = bpf_map__fd(map);
103	107
104		− err = bpf_map_get_next_key(map_fd, NULL, &cookie_key);
105		− if (err) {
106		− log_err("Can't get cookie key from map");
107		− goto out;
108		− }
109		−
110		− err = bpf_map_lookup_elem(map_fd, &cookie_key, &cookie_value);
111		− if (err) {
112		− log_err("Can't get cookie value from map");
113		− goto out;
114		− }
	108	+ err = bpf_map_lookup_elem(map_fd, &client_fd, &val);
115	109
116	110	err = getsockname(client_fd, (struct sockaddr *)&addr, &len);
117	111	if (err) {
@@ -120,8 +114,8 @@ static int validate_map(struct bpf_map *map, int client_fd) · 2 unmodified lines
120	114	}
121	115
122	116	cookie_expected_value = (ntohs(addr.sin6_port) << 8) \| 0xFF;
123		− if (cookie_value != cookie_expected_value) {
124		− log_err("Unexpected value in map: %x != %x", cookie_value,
	117	+ if (val.cookie_value != cookie_expected_value) {
	118	+ log_err("Unexpected value in map: %x != %x", val.cookie_value,
125	119	cookie_expected_value);
126	120	goto err;
127	121	}

tools/testing/selftests/bpf/test_verifier.c+4−7

@@ -234,10 +234,10 @@ static void bpf_fill_scale1(struct bpf_test *self) · 233 unmodified lines
234	234	insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6,
235	235	-8 * (k % 64 + 1));
236	236	}
237		− /* every jump adds 1 step to insn_processed, so to stay exactly
238		− * within 1m limit add MAX_TEST_INSNS - MAX_JMP_SEQ - 1 MOVs and 1 EXIT
	237	+ /* is_state_visited() doesn't allocate state for pruning for every jump.
	238	+ * Hence multiply jmps by 4 to accommodate that heuristic
239	239	*/
240		− while (i < MAX_TEST_INSNS - MAX_JMP_SEQ - 1)
	240	+ while (i < MAX_TEST_INSNS - MAX_JMP_SEQ * 4)
241	241	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 42);
242	242	insn[i] = BPF_EXIT_INSN();
243	243	self->prog_len = i + 1;
@@ -266,10 +266,7 @@ static void bpf_fill_scale2(struct bpf_test *self) · 22 unmodified lines
266	266	insn[i++] = BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_6,
267	267	-8 * (k % (64 - 4 * FUNC_NEST) + 1));
268	268	}
269		− /* every jump adds 1 step to insn_processed, so to stay exactly
270		− * within 1m limit add MAX_TEST_INSNS - MAX_JMP_SEQ - 1 MOVs and 1 EXIT
271		− */
272		− while (i < MAX_TEST_INSNS - MAX_JMP_SEQ - 1)
	269	+ while (i < MAX_TEST_INSNS - MAX_JMP_SEQ * 4)
273	270	insn[i++] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 42);
274	271	insn[i] = BPF_EXIT_INSN();
275	272	self->prog_len = i + 1;

tools/testing/selftests/bpf/verifier/calls.c+13−9

@@ -215,9 +215,11 @@ · 214 unmodified lines
215	215	BPF_MOV64_IMM(BPF_REG_0, 3),
216	216	BPF_JMP_IMM(BPF_JA, 0, 0, -6),
217	217	},
218		− .prog_type = BPF_PROG_TYPE_TRACEPOINT,
219		− .errstr = "back-edge from insn",
220		− .result = REJECT,
	218	+ .prog_type = BPF_PROG_TYPE_SOCKET_FILTER,
	219	+ .errstr_unpriv = "back-edge from insn",
	220	+ .result_unpriv = REJECT,
	221	+ .result = ACCEPT,
	222	+ .retval = 1,
221	223	},
222	224	{
223	225	"calls: conditional call 4",
@@ -250,22 +252,24 @@ · 26 unmodified lines
250	252	BPF_MOV64_IMM(BPF_REG_0, 3),
251	253	BPF_EXIT_INSN(),
252	254	},
253		− .prog_type = BPF_PROG_TYPE_TRACEPOINT,
254		− .errstr = "back-edge from insn",
255		− .result = REJECT,
	255	+ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
	256	+ .result = ACCEPT,
	257	+ .retval = 1,
256	258	},
257	259	{
258	260	"calls: conditional call 6",
259	261	.insns = {
	262	+ BPF_MOV64_REG(BPF_REG_6, BPF_REG_1),
	263	+ BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
260	264	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, 2),
261		− BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -2),
	265	+ BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -3),
262	266	BPF_EXIT_INSN(),
263	267	BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
264	268	offsetof(struct __sk_buff, mark)),
265	269	BPF_EXIT_INSN(),
266	270	},
267		− .prog_type = BPF_PROG_TYPE_TRACEPOINT,
268		− .errstr = "back-edge from insn",
	271	+ .prog_type = BPF_PROG_TYPE_SCHED_CLS,
	272	+ .errstr = "infinite loop detected",
269	273	.result = REJECT,
270	274	},
271	275	{

tools/testing/selftests/bpf/verifier/cfg.c+7−4

@@ -41,7 +41,8 @@ · 40 unmodified lines
41	41	BPF_JMP_IMM(BPF_JA, 0, 0, -1),
42	42	BPF_EXIT_INSN(),
43	43	},
44		− .errstr = "back-edge",
	44	+ .errstr = "unreachable insn 1",
	45	+ .errstr_unpriv = "back-edge",
45	46	.result = REJECT,
46	47	},
47	48	{
@@ -53,18 +54,20 @@ · 5 unmodified lines
53	54	BPF_JMP_IMM(BPF_JA, 0, 0, -4),
54	55	BPF_EXIT_INSN(),
55	56	},
56		− .errstr = "back-edge",
	57	+ .errstr = "unreachable insn 4",
	58	+ .errstr_unpriv = "back-edge",
57	59	.result = REJECT,
58	60	},
59	61	{
60	62	"conditional loop",
61	63	.insns = {
62		− BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
	64	+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
63	65	BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
64	66	BPF_MOV64_REG(BPF_REG_3, BPF_REG_0),
65	67	BPF_JMP_IMM(BPF_JEQ, BPF_REG_1, 0, -3),
66	68	BPF_EXIT_INSN(),
67	69	},
68		− .errstr = "back-edge",
	70	+ .errstr = "infinite loop detected",
	71	+ .errstr_unpriv = "back-edge",
69	72	.result = REJECT,
70	73	},

tools/testing/selftests/bpf/verifier/direct_packet_access.c+2−1

@@ -511,7 +511,8 @@ · 510 unmodified lines
511	511	offsetof(struct __sk_buff, data)),
512	512	BPF_LDX_MEM(BPF_W, BPF_REG_3, BPF_REG_1,
513	513	offsetof(struct __sk_buff, data_end)),
514		− BPF_MOV64_IMM(BPF_REG_0, 0xffffffff),
	514	+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1,
	515	+ offsetof(struct __sk_buff, mark)),
515	516	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
516	517	BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_10, -8),
517	518	BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xffff),

tools/testing/selftests/bpf/verifier/helper_access_var_len.c+15−13

@@ -29,9 +29,9 @@ · 28 unmodified lines
29	29	{
30	30	"helper access to variable memory: stack, bitwise AND, zero included",
31	31	.insns = {
	32	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
32	33	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
33	34	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
34		− BPF_MOV64_IMM(BPF_REG_2, 16),
35	35	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_2, -128),
36	36	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, -128),
37	37	BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 64),
@@ -46,9 +46,9 @@ · 8 unmodified lines
46	46	{
47	47	"helper access to variable memory: stack, bitwise AND + JMP, wrong max",
48	48	.insns = {
	49	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
49	50	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
50	51	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
51		− BPF_MOV64_IMM(BPF_REG_2, 16),
52	52	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_2, -128),
53	53	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, -128),
54	54	BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 65),
@@ -122,9 +122,9 @@ · 67 unmodified lines
122	122	{
123	123	"helper access to variable memory: stack, JMP, bounds + offset",
124	124	.insns = {
	125	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
125	126	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
126	127	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
127		− BPF_MOV64_IMM(BPF_REG_2, 16),
128	128	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_2, -128),
129	129	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, -128),
130	130	BPF_JMP_IMM(BPF_JGT, BPF_REG_2, 64, 5),
@@ -143,9 +143,9 @@ · 12 unmodified lines
143	143	{
144	144	"helper access to variable memory: stack, JMP, wrong max",
145	145	.insns = {
	146	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
146	147	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
147	148	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
148		− BPF_MOV64_IMM(BPF_REG_2, 16),
149	149	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_2, -128),
150	150	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, -128),
151	151	BPF_JMP_IMM(BPF_JGT, BPF_REG_2, 65, 4),
@@ -163,9 +163,9 @@ · 11 unmodified lines
163	163	{
164	164	"helper access to variable memory: stack, JMP, no max check",
165	165	.insns = {
	166	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
166	167	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
167	168	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
168		− BPF_MOV64_IMM(BPF_REG_2, 16),
169	169	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_2, -128),
170	170	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, -128),
171	171	BPF_MOV64_IMM(BPF_REG_4, 0),
@@ -183,9 +183,9 @@ · 11 unmodified lines
183	183	{
184	184	"helper access to variable memory: stack, JMP, no min check",
185	185	.insns = {
	186	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
186	187	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
187	188	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
188		− BPF_MOV64_IMM(BPF_REG_2, 16),
189	189	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_2, -128),
190	190	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, -128),
191	191	BPF_JMP_IMM(BPF_JGT, BPF_REG_2, 64, 3),
@@ -201,9 +201,9 @@ · 9 unmodified lines
201	201	{
202	202	"helper access to variable memory: stack, JMP (signed), no min check",
203	203	.insns = {
	204	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
204	205	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
205	206	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
206		− BPF_MOV64_IMM(BPF_REG_2, 16),
207	207	BPF_STX_MEM(BPF_DW, BPF_REG_1, BPF_REG_2, -128),
208	208	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, -128),
209	209	BPF_JMP_IMM(BPF_JSGT, BPF_REG_2, 64, 3),
@@ -244,6 +244,7 @@ · 34 unmodified lines
244	244	{
245	245	"helper access to variable memory: map, JMP, wrong max",
246	246	.insns = {
	247	+ BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 8),
247	248	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
248	249	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
249	250	BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
@@ -251,7 +252,7 @@ · 1 unmodified line
251	252	BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
252	253	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 10),
253	254	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
254		− BPF_MOV64_IMM(BPF_REG_2, sizeof(struct test_val)),
	255	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_6),
255	256	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128),
256	257	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128),
257	258	BPF_JMP_IMM(BPF_JSGT, BPF_REG_2, sizeof(struct test_val) + 1, 4),
@@ -262,7 +263,7 @@ · 4 unmodified lines
262	263	BPF_MOV64_IMM(BPF_REG_0, 0),
263	264	BPF_EXIT_INSN(),
264	265	},
265		− .fixup_map_hash_48b = { 3 },
	266	+ .fixup_map_hash_48b = { 4 },
266	267	.errstr = "invalid access to map value, value_size=48 off=0 size=49",
267	268	.result = REJECT,
268	269	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
@@ -296,6 +297,7 @@ · 27 unmodified lines
296	297	{
297	298	"helper access to variable memory: map adjusted, JMP, wrong max",
298	299	.insns = {
	300	+ BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_1, 8),
299	301	BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
300	302	BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
301	303	BPF_ST_MEM(BPF_DW, BPF_REG_2, 0, 0),
@@ -304,7 +306,7 @@ · 2 unmodified lines
304	306	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 11),
305	307	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
306	308	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 20),
307		− BPF_MOV64_IMM(BPF_REG_2, sizeof(struct test_val)),
	309	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_6),
308	310	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128),
309	311	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128),
310	312	BPF_JMP_IMM(BPF_JSGT, BPF_REG_2, sizeof(struct test_val) - 19, 4),
@@ -315,7 +317,7 @@ · 4 unmodified lines
315	317	BPF_MOV64_IMM(BPF_REG_0, 0),
316	318	BPF_EXIT_INSN(),
317	319	},
318		− .fixup_map_hash_48b = { 3 },
	320	+ .fixup_map_hash_48b = { 4 },
319	321	.errstr = "R1 min value is outside of the array range",
320	322	.result = REJECT,
321	323	.prog_type = BPF_PROG_TYPE_TRACEPOINT,
@@ -337,8 +339,8 @@ · 15 unmodified lines
337	339	{
338	340	"helper access to variable memory: size > 0 not allowed on NULL (ARG_PTR_TO_MEM_OR_NULL)",
339	341	.insns = {
	342	+ BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, 0),
340	343	BPF_MOV64_IMM(BPF_REG_1, 0),
341		− BPF_MOV64_IMM(BPF_REG_2, 1),
342	344	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128),
343	345	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128),
344	346	BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 64),
@@ -562,6 +564,7 @@ · 217 unmodified lines
562	564	{
563	565	"helper access to variable memory: 8 bytes leak",
564	566	.insns = {
	567	+ BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_1, 8),
565	568	BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
566	569	BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -64),
567	570	BPF_MOV64_IMM(BPF_REG_0, 0),
@@ -572,7 +575,6 @@ · 4 unmodified lines
572	575	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -24),
573	576	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -16),
574	577	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_0, -8),
575		− BPF_MOV64_IMM(BPF_REG_2, 1),
576	578	BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -128),
577	579	BPF_LDX_MEM(BPF_DW, BPF_REG_2, BPF_REG_10, -128),
578	580	BPF_ALU64_IMM(BPF_AND, BPF_REG_2, 63),

tools/testing/selftests/bpf/verifier/loops1.c+161−0

@@ -0,0 +1,161 @@
	1	+{
	2	+ "bounded loop, count to 4",
	3	+ .insns = {
	4	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	5	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	6	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 4, -2),
	7	+ BPF_EXIT_INSN(),
	8	+ },
	9	+ .result = ACCEPT,
	10	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	11	+ .retval = 4,
	12	+},
	13	+{
	14	+ "bounded loop, count to 20",
	15	+ .insns = {
	16	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	17	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 3),
	18	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 20, -2),
	19	+ BPF_EXIT_INSN(),
	20	+ },
	21	+ .result = ACCEPT,
	22	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	23	+},
	24	+{
	25	+ "bounded loop, count from positive unknown to 4",
	26	+ .insns = {
	27	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_get_prandom_u32),
	28	+ BPF_JMP_IMM(BPF_JSLT, BPF_REG_0, 0, 2),
	29	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	30	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 4, -2),
	31	+ BPF_EXIT_INSN(),
	32	+ },
	33	+ .result = ACCEPT,
	34	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	35	+ .retval = 4,
	36	+},
	37	+{
	38	+ "bounded loop, count from totally unknown to 4",
	39	+ .insns = {
	40	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_get_prandom_u32),
	41	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	42	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 4, -2),
	43	+ BPF_EXIT_INSN(),
	44	+ },
	45	+ .result = ACCEPT,
	46	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	47	+},
	48	+{
	49	+ "bounded loop, count to 4 with equality",
	50	+ .insns = {
	51	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	52	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	53	+ BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 4, -2),
	54	+ BPF_EXIT_INSN(),
	55	+ },
	56	+ .result = ACCEPT,
	57	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	58	+},
	59	+{
	60	+ "bounded loop, start in the middle",
	61	+ .insns = {
	62	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	63	+ BPF_JMP_A(1),
	64	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	65	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 4, -2),
	66	+ BPF_EXIT_INSN(),
	67	+ },
	68	+ .result = REJECT,
	69	+ .errstr = "back-edge",
	70	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	71	+ .retval = 4,
	72	+},
	73	+{
	74	+ "bounded loop containing a forward jump",
	75	+ .insns = {
	76	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	77	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	78	+ BPF_JMP_REG(BPF_JEQ, BPF_REG_0, BPF_REG_0, 0),
	79	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 4, -3),
	80	+ BPF_EXIT_INSN(),
	81	+ },
	82	+ .result = ACCEPT,
	83	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	84	+ .retval = 4,
	85	+},
	86	+{
	87	+ "bounded loop that jumps out rather than in",
	88	+ .insns = {
	89	+ BPF_MOV64_IMM(BPF_REG_6, 0),
	90	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_6, 1),
	91	+ BPF_JMP_IMM(BPF_JGT, BPF_REG_6, 10000, 2),
	92	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_get_prandom_u32),
	93	+ BPF_JMP_A(-4),
	94	+ BPF_EXIT_INSN(),
	95	+ },
	96	+ .result = ACCEPT,
	97	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	98	+},
	99	+{
	100	+ "infinite loop after a conditional jump",
	101	+ .insns = {
	102	+ BPF_MOV64_IMM(BPF_REG_0, 5),
	103	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 4, 2),
	104	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	105	+ BPF_JMP_A(-2),
	106	+ BPF_EXIT_INSN(),
	107	+ },
	108	+ .result = REJECT,
	109	+ .errstr = "program is too large",
	110	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	111	+},
	112	+{
	113	+ "bounded recursion",
	114	+ .insns = {
	115	+ BPF_MOV64_IMM(BPF_REG_1, 0),
	116	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, 1),
	117	+ BPF_EXIT_INSN(),
	118	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, 1),
	119	+ BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
	120	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_1, 4, 1),
	121	+ BPF_EXIT_INSN(),
	122	+ BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 1, 0, -5),
	123	+ BPF_EXIT_INSN(),
	124	+ },
	125	+ .result = REJECT,
	126	+ .errstr = "back-edge",
	127	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	128	+},
	129	+{
	130	+ "infinite loop in two jumps",
	131	+ .insns = {
	132	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	133	+ BPF_JMP_A(0),
	134	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 4, -2),
	135	+ BPF_EXIT_INSN(),
	136	+ },
	137	+ .result = REJECT,
	138	+ .errstr = "loop detected",
	139	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	140	+},
	141	+{
	142	+ "infinite loop: three-jump trick",
	143	+ .insns = {
	144	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	145	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	146	+ BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1),
	147	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 2, 1),
	148	+ BPF_EXIT_INSN(),
	149	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	150	+ BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1),
	151	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 2, 1),
	152	+ BPF_EXIT_INSN(),
	153	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
	154	+ BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 1),
	155	+ BPF_JMP_IMM(BPF_JLT, BPF_REG_0, 2, -11),
	156	+ BPF_EXIT_INSN(),
	157	+ },
	158	+ .result = REJECT,
	159	+ .errstr = "loop detected",
	160	+ .prog_type = BPF_PROG_TYPE_TRACEPOINT,
	161	+},

tools/testing/selftests/bpf/verifier/prevent_map_lookup.c+0−15

@@ -28,21 +28,6 @@ · 27 unmodified lines
28	28	.errstr = "cannot pass map_type 18 into func bpf_map_lookup_elem",
29	29	.prog_type = BPF_PROG_TYPE_SOCK_OPS,
30	30	},
31		−{
32		− "prevent map lookup in xskmap",
33		− .insns = {
34		− BPF_ST_MEM(BPF_DW, BPF_REG_10, -8, 0),
35		− BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
36		− BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
37		− BPF_LD_MAP_FD(BPF_REG_1, 0),
38		− BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
39		− BPF_EXIT_INSN(),
40		− },
41		− .fixup_map_xskmap = { 3 },
42		− .result = REJECT,
43		− .errstr = "cannot pass map_type 17 into func bpf_map_lookup_elem",
44		− .prog_type = BPF_PROG_TYPE_XDP,
45		−},
46	31	{
47	32	"prevent map lookup in stack trace",
48	33	.insns = {

tools/testing/selftests/bpf/verifier/sock.c+18−0

@@ -498,3 +498,21 @@ · 497 unmodified lines
498	498	.result = REJECT,
499	499	.errstr = "cannot pass map_type 24 into func bpf_map_lookup_elem",
500	500	},
	501	+{
	502	+ "bpf_map_lookup_elem(xskmap, &key); xs->queue_id",
	503	+ .insns = {
	504	+ BPF_ST_MEM(BPF_W, BPF_REG_10, -8, 0),
	505	+ BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
	506	+ BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -8),
	507	+ BPF_LD_MAP_FD(BPF_REG_1, 0),
	508	+ BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
	509	+ BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
	510	+ BPF_EXIT_INSN(),
	511	+ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_0, offsetof(struct bpf_xdp_sock, queue_id)),
	512	+ BPF_MOV64_IMM(BPF_REG_0, 0),
	513	+ BPF_EXIT_INSN(),
	514	+ },
	515	+ .fixup_map_xskmap = { 3 },
	516	+ .prog_type = BPF_PROG_TYPE_XDP,
	517	+ .result = ACCEPT,
	518	+},