lael-labs/linux

arch/x86/net/bpf_jit_comp.c+23−12

 	EMIT1(0x55);             /* push rbp */
 	EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */
 	/* sub rsp, rounded_stack_depth */
-	EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
+	if (stack_depth)
+		EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
 	if (tail_call_reachable)
 		EMIT1(0x50);         /* push rax */
 	*pprog = prog;
 	int tcc_off = -4 - round_up(stack_depth, 8);
 	u8 *prog = *pprog;
 	int pop_bytes = 0;
-	int off1 = 49;
+	int off1 = 42;
-	int off2 = 38;
+	int off2 = 31;
-	int off3 = 16;
+	int off3 = 9;
 	int cnt = 0;
 	/* count the additional bytes used for popping callee regs from stack
 	off2 += pop_bytes;
 	off3 += pop_bytes;
+	if (stack_depth) {
+		off1 += 7;
+		off2 += 7;
+		off3 += 7;
+	}
 	/*
 	 * rdi - pointer to ctx
 	 * rsi - pointer to bpf_array
 	prog = *pprog;
 	EMIT1(0x58);                              /* pop rax */
-	EMIT3_off32(0x48, 0x81, 0xC4,             /* add rsp, sd */
+	if (stack_depth)
-		    round_up(stack_depth, 8));
+		EMIT3_off32(0x48, 0x81, 0xC4,     /* add rsp, sd */
+			    round_up(stack_depth, 8));
 	/* goto *(prog->bpf_func + X86_TAIL_CALL_OFFSET); */
 	EMIT4(0x48, 0x8B, 0x49,                   /* mov rcx, qword ptr [rcx + 32] */
 	int tcc_off = -4 - round_up(stack_depth, 8);
 	u8 *prog = *pprog;
 	int pop_bytes = 0;
-	int off1 = 27;
+	int off1 = 20;
 	int poke_off;
 	int cnt = 0;
 	 * total bytes for:
 	 * - nop5/ jmpq $off
 	 * - pop callee regs
-	 * - sub rsp, $val
+	 * - sub rsp, $val if depth > 0
 	 * - pop rax
 	 */
-	poke_off = X86_PATCH_SIZE + pop_bytes + 7 + 1;
+	poke_off = X86_PATCH_SIZE + pop_bytes + 1;
+	if (stack_depth) {
+		poke_off += 7;
+		off1 += 7;
+	}
 	/*
 	 * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
 	pop_callee_regs(pprog, callee_regs_used);
 	prog = *pprog;
 	EMIT1(0x58);                                  /* pop rax */
-	EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8));
+	if (stack_depth)
+		EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8));
 	memcpy(prog, ideal_nops[NOP_ATOMIC5], X86_PATCH_SIZE);
 	prog += X86_PATCH_SIZE;
 			/* Update cleanup_addr */
 			ctx->cleanup_addr = proglen;
 			pop_callee_regs(&prog, callee_regs_used);
-			if (tail_call_reachable)
-				EMIT1(0x59); /* pop rcx, get rid of tail_call_cnt */
 			EMIT1(0xC9);         /* leave */
 			EMIT1(0xC3);         /* ret */
 			break;

include/linux/bpf.h+48−13

 	ARG_PTR_TO_ALLOC_MEM,	/* pointer to dynamically allocated memory */
 	ARG_PTR_TO_ALLOC_MEM_OR_NULL,	/* pointer to dynamically allocated memory or NULL */
 	ARG_CONST_ALLOC_SIZE_OR_ZERO,	/* number of allocated bytes requested */
+	ARG_PTR_TO_BTF_ID_SOCK_COMMON,	/* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */
 	__BPF_ARG_TYPE_MAX,
 };
 	PTR_TO_TCP_SOCK_OR_NULL, /* reg points to struct tcp_sock or NULL */
 	PTR_TO_TP_BUFFER,	 /* reg points to a writable raw tp's buffer */
 	PTR_TO_XDP_SOCK,	 /* reg points to struct xdp_sock */
-	PTR_TO_BTF_ID,		 /* reg points to kernel struct */
+	/* PTR_TO_BTF_ID points to a kernel struct that does not need
-	PTR_TO_BTF_ID_OR_NULL,	 /* reg points to kernel struct or NULL */
+	 * to be null checked by the BPF program. This does not imply the
+	 * pointer is _not_ null and in practice this can easily be a null
+	 * pointer when reading pointer chains. The assumption is program
+	 * context will handle null pointer dereference typically via fault
+	 * handling. The verifier must keep this in mind and can make no
+	 * assumptions about null or non-null when doing branch analysis.
+	 * Further, when passed into helpers the helpers can not, without
+	 * additional context, assume the value is non-null.
+	 */
+	PTR_TO_BTF_ID,
+	/* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not
+	 * been checked for null. Used primarily to inform the verifier
+	 * an explicit null check is required for this struct.
+	 */
+	PTR_TO_BTF_ID_OR_NULL,
 	PTR_TO_MEM,		 /* reg points to valid memory region */
 	PTR_TO_MEM_OR_NULL,	 /* reg points to valid memory region or NULL */
 	PTR_TO_RDONLY_BUF,	 /* reg points to a readonly buffer */
 	struct bpf_ksym ksym;
 };
+struct bpf_attach_target_info {
+	struct btf_func_model fmodel;
+	long tgt_addr;
+	const char *tgt_name;
+	const struct btf_type *tgt_type;
+};
 #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */
 struct bpf_dispatcher_prog {
 	return bpf_func(ctx, insnsi);
 }
 #ifdef CONFIG_BPF_JIT
-struct bpf_trampoline *bpf_trampoline_lookup(u64 key);
+int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_link_prog(struct bpf_prog *prog);
+int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr);
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog);
+struct bpf_trampoline *bpf_trampoline_get(u64 key,
+					  struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
 #define BPF_DISPATCHER_INIT(_name) {				\
 	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\
 void bpf_ksym_add(struct bpf_ksym *ksym);
 void bpf_ksym_del(struct bpf_ksym *ksym);
 #else
-static inline struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
+static inline int bpf_trampoline_link_prog(struct bpf_prog *prog,
+					   struct bpf_trampoline *tr)
 {
-	return NULL;
+	return -ENOTSUPP;
 }
-static inline int bpf_trampoline_link_prog(struct bpf_prog *prog)
+static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog,
+					     struct bpf_trampoline *tr)
 {
 	return -ENOTSUPP;
 }
-static inline int bpf_trampoline_unlink_prog(struct bpf_prog *prog)
+static inline struct bpf_trampoline *bpf_trampoline_get(u64 key,
+							struct bpf_attach_target_info *tgt_info)
 {
-	return -ENOTSUPP;
+	return ERR_PTR(-EOPNOTSUPP);
 }
 static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {}
 #define DEFINE_BPF_DISPATCHER(name)
 	u32 max_rdonly_access;
 	u32 max_rdwr_access;
 	const struct bpf_ctx_arg_aux *ctx_arg_info;
-	struct bpf_prog *linked_prog;
+	struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */
+	struct bpf_prog *dst_prog;
+	struct bpf_trampoline *dst_trampoline;
+	enum bpf_prog_type saved_dst_prog_type;
+	enum bpf_attach_type saved_dst_attach_type;
 	bool verifier_zext; /* Zero extensions has been inserted by verifier. */
 	bool offload_requested;
 	bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */
 	bool sleepable;
 	bool tail_call_reachable;
 	enum bpf_tramp_prog_type trampoline_prog_type;
-	struct bpf_trampoline *trampoline;
 	struct hlist_node tramp_hlist;
 	/* BTF_KIND_FUNC_PROTO for valid attach_btf_id */
 	const struct btf_type *attach_func_proto;
 	      union bpf_attr __user *uattr);
 void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+struct btf *bpf_get_btf_vmlinux(void);
 /* Map specifics */
 struct xdp_buff;
 struct sk_buff;
 int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog,
 				     const union bpf_attr *kattr,
 				     union bpf_attr __user *uattr);
+int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
+			     const union bpf_attr *kattr,
+			     union bpf_attr __user *uattr);
 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 		    const struct bpf_prog *prog,
 		    struct bpf_insn_access_aux *info);
 			     struct bpf_reg_state *regs);
 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
 			  struct bpf_reg_state *reg);
-int btf_check_type_match(struct bpf_verifier_env *env, struct bpf_prog *prog,
+int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
 			 struct btf *btf, const struct btf_type *t);
 struct bpf_prog *bpf_prog_by_id(u32 id);
 extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto;
 extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto;
 extern const struct bpf_func_proto bpf_copy_from_user_proto;
+extern const struct bpf_func_proto bpf_snprintf_btf_proto;
 const struct bpf_func_proto *bpf_tracing_func_proto(
 	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_verifier.h+16−2

@@ -347,8 +347,9 @@ static inline bool bpf_verifier_log_full(const struct bpf_verifier_log *log) · 346 unmodified lines
347		347
348	static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)	348	static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
349	{	349	{
350	return (log->level && log->ubuf && !bpf_verifier_log_full(log)) \|\|	350	return log &&
351	log->level == BPF_LOG_KERNEL;	351	((log->level && log->ubuf && !bpf_verifier_log_full(log)) \|\|
		352	log->level == BPF_LOG_KERNEL);
352	}	353	}
353		354
354	#define BPF_MAX_SUBPROGS 256	355	#define BPF_MAX_SUBPROGS 256
@@ -449,4 +450,17 @@ bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt); · 94 unmodified lines
449	int check_ctx_reg(struct bpf_verifier_env *env,	450	int check_ctx_reg(struct bpf_verifier_env *env,
450	const struct bpf_reg_state *reg, int regno);	451	const struct bpf_reg_state *reg, int regno);
451		452
		453	/* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
		454	static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
		455	u32 btf_id)
		456	{
		457	return tgt_prog ? (((u64)tgt_prog->aux->id) << 32 \| btf_id) : btf_id;
		458	}
		459
		460	int bpf_check_attach_target(struct bpf_verifier_log *log,
		461	const struct bpf_prog *prog,
		462	const struct bpf_prog *tgt_prog,
		463	u32 btf_id,
		464	struct bpf_attach_target_info *tgt_info);
		465
452	#endif /* _LINUX_BPF_VERIFIER_H */	466	#endif /* _LINUX_BPF_VERIFIER_H */

include/linux/btf.h+39−0

@@ -6,6 +6,7 @@ · 5 unmodified lines
6		6
7	#include <linux/types.h>	7	#include <linux/types.h>
8	#include <uapi/linux/btf.h>	8	#include <uapi/linux/btf.h>
		9	#include <uapi/linux/bpf.h>
9		10
10	#define BTF_TYPE_EMIT(type) ((void)(type *)0)	11	#define BTF_TYPE_EMIT(type) ((void)(type *)0)
11		12
@@ -13,6 +14,7 @@ struct btf; · 1 unmodified line
13	struct btf_member;	14	struct btf_member;
14	struct btf_type;	15	struct btf_type;
15	union bpf_attr;	16	union bpf_attr;
		17	struct btf_show;
16		18
17	extern const struct file_operations btf_fops;	19	extern const struct file_operations btf_fops;
18		20
@@ -46,8 +48,45 @@ int btf_get_info_by_fd(const struct btf *btf, · 27 unmodified lines
46	const struct btf_type btf_type_id_size(const struct btf btf,	48	const struct btf_type btf_type_id_size(const struct btf btf,
47	u32 *type_id,	49	u32 *type_id,
48	u32 *ret_size);	50	u32 *ret_size);
		51
		52	/*
		53	* Options to control show behaviour.
		54	* - BTF_SHOW_COMPACT: no formatting around type information
		55	* - BTF_SHOW_NONAME: no struct/union member names/types
		56	* - BTF_SHOW_PTR_RAW: show raw (unobfuscated) pointer values;
		57	* equivalent to %px.
		58	* - BTF_SHOW_ZERO: show zero-valued struct/union members; they
		59	* are not displayed by default
		60	* - BTF_SHOW_UNSAFE: skip use of bpf_probe_read() to safely read
		61	* data before displaying it.
		62	*/
		63	#define BTF_SHOW_COMPACT BTF_F_COMPACT
		64	#define BTF_SHOW_NONAME BTF_F_NONAME
		65	#define BTF_SHOW_PTR_RAW BTF_F_PTR_RAW
		66	#define BTF_SHOW_ZERO BTF_F_ZERO
		67	#define BTF_SHOW_UNSAFE (1ULL << 4)
		68
49	void btf_type_seq_show(const struct btf btf, u32 type_id, void obj,	69	void btf_type_seq_show(const struct btf btf, u32 type_id, void obj,
50	struct seq_file *m);	70	struct seq_file *m);
		71	int btf_type_seq_show_flags(const struct btf btf, u32 type_id, void obj,
		72	struct seq_file *m, u64 flags);
		73
		74	/*
		75	* Copy len bytes of string representation of obj of BTF type_id into buf.
		76	*
		77	* @btf: struct btf object
		78	* @type_id: type id of type obj points to
		79	* @obj: pointer to typed data
		80	* @buf: buffer to write to
		81	* @len: maximum length to write to buf
		82	* @flags: show options (see above)
		83	*
		84	* Return: length that would have been/was copied as per snprintf, or
		85	* negative error.
		86	*/
		87	int btf_type_snprintf_show(const struct btf btf, u32 type_id, void obj,
		88	char *buf, int len, u64 flags);
		89
51	int btf_get_fd_by_id(u32 id);	90	int btf_get_fd_by_id(u32 id);
52	u32 btf_id(const struct btf *btf);	91	u32 btf_id(const struct btf *btf);
53	bool btf_member_is_reg_int(const struct btf btf, const struct btf_type s,	92	bool btf_member_is_reg_int(const struct btf btf, const struct btf_type s,

include/linux/cookie.h+51−0

@@ -0,0 +1,51 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __LINUX_COOKIE_H
	3	#define __LINUX_COOKIE_H
	4
	5	#include <linux/atomic.h>
	6	#include <linux/percpu.h>
	7	#include <asm/local.h>
	8
	9	struct pcpu_gen_cookie {
	10	local_t nesting;
	11	u64 last;
	12	} __aligned(16);
	13
	14	struct gen_cookie {
	15	struct pcpu_gen_cookie __percpu *local;
	16	atomic64_t forward_last ____cacheline_aligned_in_smp;
	17	atomic64_t reverse_last;
	18	};
	19
	20	#define COOKIE_LOCAL_BATCH 4096
	21
	22	#define DEFINE_COOKIE(name) \
	23	static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name); \
	24	static struct gen_cookie name = { \
	25	.local = &__##name, \
	26	.forward_last = ATOMIC64_INIT(0), \
	27	.reverse_last = ATOMIC64_INIT(0), \
	28	}
	29
	30	static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
	31	{
	32	struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
	33	u64 val;
	34
	35	if (likely(local_inc_return(&local->nesting) == 1)) {
	36	val = local->last;
	37	if (__is_defined(CONFIG_SMP) &&
	38	unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
	39	s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
	40	&gc->forward_last);
	41	val = next - COOKIE_LOCAL_BATCH;
	42	}
	43	local->last = ++val;
	44	} else {
	45	val = atomic64_dec_return(&gc->reverse_last);
	46	}
	47	local_dec(&local->nesting);
	48	return val;
	49	}
	50
	51	#endif /* __LINUX_COOKIE_H */

include/linux/rcupdate_trace.h+4−0

 	struct task_struct *t = current;
 	WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1);
+	barrier();
 	if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) &&
 	    t->trc_reader_special.b.need_mb)
 		smp_mb(); // Pairs with update-side barriers
 	rcu_lock_release(&rcu_trace_lock_map);
 	nesting = READ_ONCE(t->trc_reader_nesting) - 1;
+	barrier(); // Critical section before disabling.
+	// Disable IPI-based setting of .need_qs.
+	WRITE_ONCE(t->trc_reader_nesting, INT_MIN);
 	if (likely(!READ_ONCE(t->trc_reader_special.s)) || nesting) {
 		WRITE_ONCE(t->trc_reader_nesting, nesting);
 		return;  // We assume shallow reader nesting.

include/linux/skbuff.h+5−0

 	return skb->mac_header != (typeof(skb->mac_header))~0U;
 }
+static inline void skb_unset_mac_header(struct sk_buff *skb)
+{
+	skb->mac_header = (typeof(skb->mac_header))~0U;
+}
 static inline void skb_reset_mac_header(struct sk_buff *skb)
 {
 	skb->mac_header = skb->data - skb->head;

include/linux/sock_diag.h+13−1

@@ -25,7 +25,19 @@ void sock_diag_unregister(const struct sock_diag_handler *h); · 24 unmodified lines
25	void sock_diag_register_inet_compat(int (fn)(struct sk_buff skb, struct nlmsghdr *nlh));	25	void sock_diag_register_inet_compat(int (fn)(struct sk_buff skb, struct nlmsghdr *nlh));
26	void sock_diag_unregister_inet_compat(int (fn)(struct sk_buff skb, struct nlmsghdr *nlh));	26	void sock_diag_unregister_inet_compat(int (fn)(struct sk_buff skb, struct nlmsghdr *nlh));
27		27
28	u64 sock_gen_cookie(struct sock *sk);	28	u64 __sock_gen_cookie(struct sock *sk);
		29
		30	static inline u64 sock_gen_cookie(struct sock *sk)
		31	{
		32	u64 cookie;
		33
		34	preempt_disable();
		35	cookie = __sock_gen_cookie(sk);
		36	preempt_enable();
		37
		38	return cookie;
		39	}
		40
29	int sock_diag_check_cookie(struct sock sk, const __u32 cookie);	41	int sock_diag_check_cookie(struct sock sk, const __u32 cookie);
30	void sock_diag_save_cookie(struct sock sk, __u32 cookie);	42	void sock_diag_save_cookie(struct sock sk, __u32 cookie);
31		43

include/net/bpf_sk_storage.h+0−2

@@ -20,8 +20,6 @@ void bpf_sk_storage_free(struct sock *sk); · 19 unmodified lines
20		20
21	extern const struct bpf_func_proto bpf_sk_storage_get_proto;	21	extern const struct bpf_func_proto bpf_sk_storage_get_proto;
22	extern const struct bpf_func_proto bpf_sk_storage_delete_proto;	22	extern const struct bpf_func_proto bpf_sk_storage_delete_proto;
23	extern const struct bpf_func_proto sk_storage_get_btf_proto;
24	extern const struct bpf_func_proto sk_storage_delete_btf_proto;
25		23
26	struct bpf_local_storage_elem;	24	struct bpf_local_storage_elem;
27	struct bpf_sk_storage_diag;	25	struct bpf_sk_storage_diag;

include/net/net_namespace.h+1−1

@@ -230,7 +230,7 @@ extern struct list_head net_namespace_list; · 229 unmodified lines
230	struct net *get_net_ns_by_pid(pid_t pid);	230	struct net *get_net_ns_by_pid(pid_t pid);
231	struct net *get_net_ns_by_fd(int fd);	231	struct net *get_net_ns_by_fd(int fd);
232		232
233	u64 net_gen_cookie(struct net *net);	233	u64 __net_gen_cookie(struct net *net);
234		234
235	#ifdef CONFIG_SYSCTL	235	#ifdef CONFIG_SYSCTL
236	void ipx_register_sysctl(void);	236	void ipx_register_sysctl(void);

include/uapi/linux/bpf.h+125−9

@@ -414,6 +414,9 @@ enum { · 413 unmodified lines
414		414
415	/* Enable memory-mapping BPF map */	415	/* Enable memory-mapping BPF map */
416	BPF_F_MMAPABLE = (1U << 10),	416	BPF_F_MMAPABLE = (1U << 10),
		417
		418	/* Share perf_event among processes */
		419	BPF_F_PRESERVE_ELEMS = (1U << 11),
417	};	420	};
418		421
419	/* Flags for BPF_PROG_QUERY. */	422	/* Flags for BPF_PROG_QUERY. */
@@ -424,6 +427,11 @@ enum { · 4 unmodified lines
424	*/	427	*/
425	#define BPF_F_QUERY_EFFECTIVE (1U << 0)	428	#define BPF_F_QUERY_EFFECTIVE (1U << 0)
426		429
		430	/* Flags for BPF_PROG_TEST_RUN */
		431
		432	/* If set, run the test on the cpu specified by bpf_attr.test.cpu */
		433	#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
		434
427	/* type for BPF_ENABLE_STATS */	435	/* type for BPF_ENABLE_STATS */
428	enum bpf_stats_type {	436	enum bpf_stats_type {
429	/* enabled run_time_ns and run_cnt */	437	/* enabled run_time_ns and run_cnt */
@@ -566,6 +574,8 @@ union bpf_attr { · 136 unmodified lines
566	*/	574	*/
567	__aligned_u64 ctx_in;	575	__aligned_u64 ctx_in;
568	__aligned_u64 ctx_out;	576	__aligned_u64 ctx_out;
		577	__u32 flags;
		578	__u32 cpu;
569	} test;	579	} test;
570		580
571	struct { /* anonymous struct used by BPF__GET__ID */	581	struct { /* anonymous struct used by BPF__GET__ID */
@@ -632,8 +642,13 @@ union bpf_attr { · 60 unmodified lines
632	};	642	};
633	__u32 attach_type; /* attach type */	643	__u32 attach_type; /* attach type */
634	__u32 flags; /* extra flags */	644	__u32 flags; /* extra flags */
635	__aligned_u64 iter_info; /* extra bpf_iter_link_info */	645	union {
636	__u32 iter_info_len; /* iter_info length */	646	__u32 target_btf_id; /* btf_id of target to attach to */
		647	struct {
		648	__aligned_u64 iter_info; /* extra bpf_iter_link_info */
		649	__u32 iter_info_len; /* iter_info length */
		650	};
		651	};
637	} link_create;	652	} link_create;
638		653
639	struct { /* struct used by BPF_LINK_UPDATE command */	654	struct { /* struct used by BPF_LINK_UPDATE command */
@@ -2512,7 +2527,7 @@ union bpf_attr { · 1872 unmodified lines
2512	* result is from reuse\ ->socks\ [] using the hash of the	2527	* result is from reuse\ ->socks\ [] using the hash of the
2513	* tuple.	2528	* tuple.
2514	*	2529	*
2515	* long bpf_sk_release(struct bpf_sock *sock)	2530	* long bpf_sk_release(void *sock)
2516	* Description	2531	* Description
2517	* Release the reference held by sock. sock must be a	2532	* Release the reference held by sock. sock must be a
2518	* non-NULL pointer that was returned from	2533	* non-NULL pointer that was returned from
@@ -2692,7 +2707,7 @@ union bpf_attr { · 173 unmodified lines
2692	* result is from reuse\ ->socks\ [] using the hash of the	2707	* result is from reuse\ ->socks\ [] using the hash of the
2693	* tuple.	2708	* tuple.
2694	*	2709	*
2695	* long bpf_tcp_check_syncookie(struct bpf_sock sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)	2710	* long bpf_tcp_check_syncookie(void sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)
2696	* Description	2711	* Description
2697	* Check whether iph and th contain a valid SYN cookie ACK for	2712	* Check whether iph and th contain a valid SYN cookie ACK for
2698	* the listening socket in sk.	2713	* the listening socket in sk.
@@ -2861,6 +2876,7 @@ union bpf_attr { · 162 unmodified lines
2861	* 0 on success.	2876	* 0 on success.
2862	*	2877	*
2863	* -ENOENT if the bpf-local-storage cannot be found.	2878	* -ENOENT if the bpf-local-storage cannot be found.
		2879	* -EINVAL if sk is not a fullsock (e.g. a request_sock).
2864	*	2880	*
2865	* long bpf_send_signal(u32 sig)	2881	* long bpf_send_signal(u32 sig)
2866	* Description	2882	* Description
@@ -2877,7 +2893,7 @@ union bpf_attr { · 10 unmodified lines
2877	*	2893	*
2878	* -EAGAIN if bpf program can try again.	2894	* -EAGAIN if bpf program can try again.
2879	*	2895	*
2880	* s64 bpf_tcp_gen_syncookie(struct bpf_sock sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)	2896	* s64 bpf_tcp_gen_syncookie(void sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)
2881	* Description	2897	* Description
2882	* Try to issue a SYN cookie for the packet with corresponding	2898	* Try to issue a SYN cookie for the packet with corresponding
2883	* IP/TCP headers, iph and th, on the listening socket in sk.	2899	* IP/TCP headers, iph and th, on the listening socket in sk.
@@ -3106,7 +3122,7 @@ union bpf_attr { · 222 unmodified lines
3106	* Return	3122	* Return
3107	* The id is returned or 0 in case the id could not be retrieved.	3123	* The id is returned or 0 in case the id could not be retrieved.
3108	*	3124	*
3109	* long bpf_sk_assign(struct sk_buff skb, struct bpf_sock sk, u64 flags)	3125	* long bpf_sk_assign(struct sk_buff skb, void sk, u64 flags)
3110	* Description	3126	* Description
3111	* Helper is overloaded depending on BPF program type. This	3127	* Helper is overloaded depending on BPF program type. This
3112	* description applies to BPF_PROG_TYPE_SCHED_CLS and	3128	* description applies to BPF_PROG_TYPE_SCHED_CLS and
@@ -3234,11 +3250,11 @@ union bpf_attr { · 121 unmodified lines
3234	*	3250	*
3235	* -EOVERFLOW if an overflow happened: The same object will be tried again.	3251	* -EOVERFLOW if an overflow happened: The same object will be tried again.
3236	*	3252	*
3237	* u64 bpf_sk_cgroup_id(struct bpf_sock *sk)	3253	* u64 bpf_sk_cgroup_id(void *sk)
3238	* Description	3254	* Description
3239	* Return the cgroup v2 id of the socket sk.	3255	* Return the cgroup v2 id of the socket sk.
3240	*	3256	*
3241	* sk must be a non-NULL pointer to a full socket, e.g. one	3257	* sk must be a non-NULL pointer to a socket, e.g. one
3242	* returned from bpf_sk_lookup_xxx\ (),	3258	* returned from bpf_sk_lookup_xxx\ (),
3243	* bpf_sk_fullsock\ (), etc. The format of returned id is	3259	* bpf_sk_fullsock\ (), etc. The format of returned id is
3244	* same as in bpf_skb_cgroup_id\ ().	3260	* same as in bpf_skb_cgroup_id\ ().
@@ -3248,7 +3264,7 @@ union bpf_attr { · 3 unmodified lines
3248	* Return	3264	* Return
3249	* The id is returned or 0 in case the id could not be retrieved.	3265	* The id is returned or 0 in case the id could not be retrieved.
3250	*	3266	*
3251	* u64 bpf_sk_ancestor_cgroup_id(struct bpf_sock *sk, int ancestor_level)	3267	* u64 bpf_sk_ancestor_cgroup_id(void *sk, int ancestor_level)
3252	* Description	3268	* Description
3253	* Return id of cgroup v2 that is ancestor of cgroup associated	3269	* Return id of cgroup v2 that is ancestor of cgroup associated
3254	* with the sk at the ancestor_level. The root cgroup is at	3270	* with the sk at the ancestor_level. The root cgroup is at
@@ -3586,6 +3602,72 @@ union bpf_attr { · 331 unmodified lines
3586	* the data in dst. This is a wrapper of copy_from_user\ ().	3602	* the data in dst. This is a wrapper of copy_from_user\ ().
3587	* Return	3603	* Return
3588	* 0 on success, or a negative error in case of failure.	3604	* 0 on success, or a negative error in case of failure.
		3605	*
		3606	* long bpf_snprintf_btf(char str, u32 str_size, struct btf_ptr ptr, u32 btf_ptr_size, u64 flags)
		3607	* Description
		3608	* Use BTF to store a string representation of ptr->ptr in str,
		3609	* using ptr->type_id. This value should specify the type
		3610	* that ptr->ptr points to. LLVM __builtin_btf_type_id(type, 1)
		3611	* can be used to look up vmlinux BTF type ids. Traversing the
		3612	* data structure using BTF, the type information and values are
		3613	* stored in the first str_size - 1 bytes of str. Safe copy of
		3614	* the pointer data is carried out to avoid kernel crashes during
		3615	* operation. Smaller types can use string space on the stack;
		3616	* larger programs can use map data to store the string
		3617	* representation.
		3618	*
		3619	* The string can be subsequently shared with userspace via
		3620	* bpf_perf_event_output() or ring buffer interfaces.
		3621	* bpf_trace_printk() is to be avoided as it places too small
		3622	* a limit on string size to be useful.
		3623	*
		3624	* flags is a combination of
		3625	*
		3626	* BTF_F_COMPACT
		3627	* no formatting around type information
		3628	* BTF_F_NONAME
		3629	* no struct/union member names/types
		3630	* BTF_F_PTR_RAW
		3631	* show raw (unobfuscated) pointer values;
		3632	* equivalent to printk specifier %px.
		3633	* BTF_F_ZERO
		3634	* show zero-valued struct/union members; they
		3635	* are not displayed by default
		3636	*
		3637	* Return
		3638	* The number of bytes that were written (or would have been
		3639	* written if output had to be truncated due to string size),
		3640	* or a negative error in cases of failure.
		3641	*
		3642	* long bpf_seq_printf_btf(struct seq_file m, struct btf_ptr ptr, u32 ptr_size, u64 flags)
		3643	* Description
		3644	* Use BTF to write to seq_write a string representation of
		3645	* ptr->ptr, using ptr->type_id as per bpf_snprintf_btf().
		3646	* flags are identical to those used for bpf_snprintf_btf.
		3647	* Return
		3648	* 0 on success or a negative error in case of failure.
		3649	*
		3650	* u64 bpf_skb_cgroup_classid(struct sk_buff *skb)
		3651	* Description
		3652	* See bpf_get_cgroup_classid\ () for the main description.
		3653	* This helper differs from bpf_get_cgroup_classid\ () in that
		3654	* the cgroup v1 net_cls class is retrieved only from the skb's
		3655	* associated socket instead of the current process.
		3656	* Return
		3657	* The id is returned or 0 in case the id could not be retrieved.
		3658	*
		3659	* long bpf_redirect_neigh(u32 ifindex, u64 flags)
		3660	* Description
		3661	* Redirect the packet to another net device of index ifindex
		3662	* and fill in L2 addresses from neighboring subsystem. This helper
		3663	* is somewhat similar to bpf_redirect\ (), except that it
		3664	* fills in e.g. MAC addresses based on the L3 information from
		3665	* the packet. This helper is supported for IPv4 and IPv6 protocols.
		3666	* The flags argument is reserved and must be 0. The helper is
		3667	* currently only supported for tc BPF program types.
		3668	* Return
		3669	* The helper returns TC_ACT_REDIRECT on success or
		3670	* TC_ACT_SHOT on error.
3589	*/	3671	*/
3590	#define __BPF_FUNC_MAPPER(FN) \	3672	#define __BPF_FUNC_MAPPER(FN) \
3591	FN(unspec), \	3673	FN(unspec), \
@@ -3737,6 +3819,10 @@ union bpf_attr { · 145 unmodified lines
3737	FN(inode_storage_delete), \	3819	FN(inode_storage_delete), \
3738	FN(d_path), \	3820	FN(d_path), \
3739	FN(copy_from_user), \	3821	FN(copy_from_user), \
		3822	FN(snprintf_btf), \
		3823	FN(seq_printf_btf), \
		3824	FN(skb_cgroup_classid), \
		3825	FN(redirect_neigh), \
3740	/* */	3826	/* */
3741		3827
3742	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	3828	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -4845,4 +4931,34 @@ struct bpf_sk_lookup { · 1102 unmodified lines
4845	__u32 local_port; /* Host byte order */	4931	__u32 local_port; /* Host byte order */
4846	};	4932	};
4847		4933
		4934	/*
		4935	* struct btf_ptr is used for typed pointer representation; the
		4936	* type id is used to render the pointer data as the appropriate type
		4937	* via the bpf_snprintf_btf() helper described above. A flags field -
		4938	* potentially to specify additional details about the BTF pointer
		4939	* (rather than its mode of display) - is included for future use.
		4940	* Display flags - BTF_F_* - are passed to bpf_snprintf_btf separately.
		4941	*/
		4942	struct btf_ptr {
		4943	void *ptr;
		4944	__u32 type_id;
		4945	__u32 flags; /* BTF ptr flags; unused at present. */
		4946	};
		4947
		4948	/*
		4949	* Flags to control bpf_snprintf_btf() behaviour.
		4950	* - BTF_F_COMPACT: no formatting around type information
		4951	* - BTF_F_NONAME: no struct/union member names/types
		4952	* - BTF_F_PTR_RAW: show raw (unobfuscated) pointer values;
		4953	* equivalent to %px.
		4954	* - BTF_F_ZERO: show zero-valued struct/union members; they
		4955	* are not displayed by default
		4956	*/
		4957	enum {
		4958	BTF_F_COMPACT = (1ULL << 0),
		4959	BTF_F_NONAME = (1ULL << 1),
		4960	BTF_F_PTR_RAW = (1ULL << 2),
		4961	BTF_F_ZERO = (1ULL << 3),
		4962	};
		4963
4848	#endif /* _UAPI__LINUX_BPF_H__ */	4964	#endif /* _UAPI__LINUX_BPF_H__ */

kernel/bpf/arraymap.c+17−2

 #include "map_in_map.h"
 #define ARRAY_CREATE_FLAG_MASK \
-	(BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK)
+	(BPF_F_NUMA_NODE | BPF_F_MMAPABLE | BPF_F_ACCESS_MASK | \
+	 BPF_F_PRESERVE_ELEMS)
 static void bpf_array_free_percpu(struct bpf_array *array)
 {
 	    attr->map_flags & BPF_F_MMAPABLE)
 		return -EINVAL;
+	if (attr->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY &&
+	    attr->map_flags & BPF_F_PRESERVE_ELEMS)
+		return -EINVAL;
 	if (attr->value_size > KMALLOC_MAX_SIZE)
 		/* if value_size is bigger, the user space won't be able to
 		 * access the elements.
 	struct bpf_event_entry *ee;
 	int i;
+	if (map->map_flags & BPF_F_PRESERVE_ELEMS)
+		return;
 	rcu_read_lock();
 	for (i = 0; i < array->map.max_entries; i++) {
 		ee = READ_ONCE(array->ptrs[i]);
 	rcu_read_unlock();
 }
+static void perf_event_fd_array_map_free(struct bpf_map *map)
+{
+	if (map->map_flags & BPF_F_PRESERVE_ELEMS)
+		bpf_fd_array_map_clear(map);
+	fd_array_map_free(map);
+}
 static int perf_event_array_map_btf_id;
 const struct bpf_map_ops perf_event_array_map_ops = {
 	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc_check = fd_array_map_alloc_check,
 	.map_alloc = array_map_alloc,
-	.map_free = fd_array_map_free,
+	.map_free = perf_event_fd_array_map_free,
 	.map_get_next_key = array_map_get_next_key,
 	.map_lookup_elem = fd_array_map_lookup_elem,
 	.map_delete_elem = fd_array_map_delete_elem,

kernel/bpf/bpf_iter.c+2−2

@@ -88,8 +88,8 @@ static ssize_t bpf_seq_read(struct file file, char __user buf, size_t size, · 87 unmodified lines
88	mutex_lock(&seq->lock);	88	mutex_lock(&seq->lock);
89		89
90	if (!seq->buf) {	90	if (!seq->buf) {
91	seq->size = PAGE_SIZE;	91	seq->size = PAGE_SIZE << 3;
92	seq->buf = kmalloc(seq->size, GFP_KERNEL);	92	seq->buf = kvmalloc(seq->size, GFP_KERNEL);
93	if (!seq->buf) {	93	if (!seq->buf) {
94	err = -ENOMEM;	94	err = -ENOMEM;
95	goto done;	95	goto done;

kernel/bpf/bpf_lsm.c+2−2

@@ -56,9 +56,9 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 55 unmodified lines
56	case BPF_FUNC_inode_storage_delete:	56	case BPF_FUNC_inode_storage_delete:
57	return &bpf_inode_storage_delete_proto;	57	return &bpf_inode_storage_delete_proto;
58	case BPF_FUNC_sk_storage_get:	58	case BPF_FUNC_sk_storage_get:
59	return &sk_storage_get_btf_proto;	59	return &bpf_sk_storage_get_proto;
60	case BPF_FUNC_sk_storage_delete:	60	case BPF_FUNC_sk_storage_delete:
61	return &sk_storage_delete_btf_proto;	61	return &bpf_sk_storage_delete_proto;
62	default:	62	default:
63	return tracing_prog_func_proto(func_id, prog);	63	return tracing_prog_func_proto(func_id, prog);
64	}	64	}

kernel/bpf/btf.c+919−109

@@ -284,6 +284,91 @@ static const char btf_type_str(const struct btf_type t) · 283 unmodified lines
284	return btf_kind_str[BTF_INFO_KIND(t->info)];	284	return btf_kind_str[BTF_INFO_KIND(t->info)];
285	}	285	}
286		286
		287	/* Chunk size we use in safe copy of data to be shown. */
		288	#define BTF_SHOW_OBJ_SAFE_SIZE 32
		289
		290	/*
		291	* This is the maximum size of a base type value (equivalent to a
		292	* 128-bit int); if we are at the end of our safe buffer and have
		293	* less than 16 bytes space we can't be assured of being able
		294	* to copy the next type safely, so in such cases we will initiate
		295	* a new copy.
		296	*/
		297	#define BTF_SHOW_OBJ_BASE_TYPE_SIZE 16
		298
		299	/* Type name size */
		300	#define BTF_SHOW_NAME_SIZE 80
		301
		302	/*
		303	* Common data to all BTF show operations. Private show functions can add
		304	* their own data to a structure containing a struct btf_show and consult it
		305	* in the show callback. See btf_type_show() below.
		306	*
		307	* One challenge with showing nested data is we want to skip 0-valued
		308	* data, but in order to figure out whether a nested object is all zeros
		309	* we need to walk through it. As a result, we need to make two passes
		310	* when handling structs, unions and arrays; the first path simply looks
		311	* for nonzero data, while the second actually does the display. The first
		312	* pass is signalled by show->state.depth_check being set, and if we
		313	* encounter a non-zero value we set show->state.depth_to_show to
		314	* the depth at which we encountered it. When we have completed the
		315	* first pass, we will know if anything needs to be displayed if
		316	* depth_to_show > depth. See btf_[struct,array]_show() for the
		317	* implementation of this.
		318	*
		319	* Another problem is we want to ensure the data for display is safe to
		320	* access. To support this, the anonymous "struct {} obj" tracks the data
		321	* object and our safe copy of it. We copy portions of the data needed
		322	* to the object "copy" buffer, but because its size is limited to
		323	* BTF_SHOW_OBJ_COPY_LEN bytes, multiple copies may be required as we
		324	* traverse larger objects for display.
		325	*
		326	* The various data type show functions all start with a call to
		327	* btf_show_start_type() which returns a pointer to the safe copy
		328	* of the data needed (or if BTF_SHOW_UNSAFE is specified, to the
		329	* raw data itself). btf_show_obj_safe() is responsible for
		330	* using copy_from_kernel_nofault() to update the safe data if necessary
		331	* as we traverse the object's data. skbuff-like semantics are
		332	* used:
		333	*
		334	* - obj.head points to the start of the toplevel object for display
		335	* - obj.size is the size of the toplevel object
		336	* - obj.data points to the current point in the original data at
		337	* which our safe data starts. obj.data will advance as we copy
		338	* portions of the data.
		339	*
		340	* In most cases a single copy will suffice, but larger data structures
		341	* such as "struct task_struct" will require many copies. The logic in
		342	* btf_show_obj_safe() handles the logic that determines if a new
		343	* copy_from_kernel_nofault() is needed.
		344	*/
		345	struct btf_show {
		346	u64 flags;
		347	void target; / target of show operation (seq file, buffer) */
		348	void (showfn)(struct btf_show show, const char *fmt, va_list args);
		349	const struct btf *btf;
		350	/* below are used during iteration */
		351	struct {
		352	u8 depth;
		353	u8 depth_to_show;
		354	u8 depth_check;
		355	u8 array_member:1,
		356	array_terminated:1;
		357	u16 array_encoding;
		358	u32 type_id;
		359	int status; /* non-zero for error */
		360	const struct btf_type *type;
		361	const struct btf_member *member;
		362	char name[BTF_SHOW_NAME_SIZE]; /* space for member name/type */
		363	} state;
		364	struct {
		365	u32 size;
		366	void *head;
		367	void *data;
		368	u8 safe[BTF_SHOW_OBJ_SAFE_SIZE];
		369	} obj;
		370	};
		371
287	struct btf_kind_operations {	372	struct btf_kind_operations {
288	s32 (check_meta)(struct btf_verifier_env env,	373	s32 (check_meta)(struct btf_verifier_env env,
289	const struct btf_type *t,	374	const struct btf_type *t,
@@ -300,9 +385,9 @@ struct btf_kind_operations { · 10 unmodified lines
300	const struct btf_type *member_type);	385	const struct btf_type *member_type);
301	void (log_details)(struct btf_verifier_env env,	386	void (log_details)(struct btf_verifier_env env,
302	const struct btf_type *t);	387	const struct btf_type *t);
303	void (seq_show)(const struct btf btf, const struct btf_type *t,	388	void (show)(const struct btf btf, const struct btf_type *t,
304	u32 type_id, void *data, u8 bits_offsets,	389	u32 type_id, void *data, u8 bits_offsets,
305	struct seq_file *m);	390	struct btf_show *show);
306	};	391	};
307		392
308	static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];	393	static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS];
@@ -679,6 +764,488 @@ bool btf_member_is_reg_int(const struct btf btf, const struct btf_type s, · 370 unmodified lines
679	return true;	764	return true;
680	}	765	}
681		766
		767	/* Similar to btf_type_skip_modifiers() but does not skip typedefs. */
		768	static const struct btf_type btf_type_skip_qualifiers(const struct btf btf,
		769	u32 id)
		770	{
		771	const struct btf_type *t = btf_type_by_id(btf, id);
		772
		773	while (btf_type_is_modifier(t) &&
		774	BTF_INFO_KIND(t->info) != BTF_KIND_TYPEDEF) {
		775	id = t->type;
		776	t = btf_type_by_id(btf, t->type);
		777	}
		778
		779	return t;
		780	}
		781
		782	#define BTF_SHOW_MAX_ITER 10
		783
		784	#define BTF_KIND_BIT(kind) (1ULL << kind)
		785
		786	/*
		787	* Populate show->state.name with type name information.
		788	* Format of type name is
		789	*
		790	* [.member_name = ] (type_name)
		791	*/
		792	static const char btf_show_name(struct btf_show show)
		793	{
		794	/* BTF_MAX_ITER array suffixes "[]" */
		795	const char *array_suffixes = "[][][][][][][][][][]";
		796	const char *array_suffix = &array_suffixes[strlen(array_suffixes)];
		797	/* BTF_MAX_ITER pointer suffixes "" /
		798	const char ptr_suffixes = "*********";
		799	const char *ptr_suffix = &ptr_suffixes[strlen(ptr_suffixes)];
		800	const char name = NULL, prefix = "", *parens = "";
		801	const struct btf_member *m = show->state.member;
		802	const struct btf_type *t = show->state.type;
		803	const struct btf_array *array;
		804	u32 id = show->state.type_id;
		805	const char *member = NULL;
		806	bool show_member = false;
		807	u64 kinds = 0;
		808	int i;
		809
		810	show->state.name[0] = '\0';
		811
		812	/*
		813	* Don't show type name if we're showing an array member;
		814	* in that case we show the array type so don't need to repeat
		815	* ourselves for each member.
		816	*/
		817	if (show->state.array_member)
		818	return "";
		819
		820	/* Retrieve member name, if any. */
		821	if (m) {
		822	member = btf_name_by_offset(show->btf, m->name_off);
		823	show_member = strlen(member) > 0;
		824	id = m->type;
		825	}
		826
		827	/*
		828	* Start with type_id, as we have resolved the struct btf_type *
		829	* via btf_modifier_show() past the parent typedef to the child
		830	* struct, int etc it is defined as. In such cases, the type_id
		831	* still represents the starting type while the struct btf_type *
		832	* in our show->state points at the resolved type of the typedef.
		833	*/
		834	t = btf_type_by_id(show->btf, id);
		835	if (!t)
		836	return "";
		837
		838	/*
		839	* The goal here is to build up the right number of pointer and
		840	* array suffixes while ensuring the type name for a typedef
		841	* is represented. Along the way we accumulate a list of
		842	* BTF kinds we have encountered, since these will inform later
		843	* display; for example, pointer types will not require an
		844	* opening "{" for struct, we will just display the pointer value.
		845	*
		846	* We also want to accumulate the right number of pointer or array
		847	* indices in the format string while iterating until we get to
		848	* the typedef/pointee/array member target type.
		849	*
		850	* We start by pointing at the end of pointer and array suffix
		851	* strings; as we accumulate pointers and arrays we move the pointer
		852	* or array string backwards so it will show the expected number of
		853	* '*' or '[]' for the type. BTF_SHOW_MAX_ITER of nesting of pointers
		854	* and/or arrays and typedefs are supported as a precaution.
		855	*
		856	* We also want to get typedef name while proceeding to resolve
		857	* type it points to so that we can add parentheses if it is a
		858	* "typedef struct" etc.
		859	*/
		860	for (i = 0; i < BTF_SHOW_MAX_ITER; i++) {
		861
		862	switch (BTF_INFO_KIND(t->info)) {
		863	case BTF_KIND_TYPEDEF:
		864	if (!name)
		865	name = btf_name_by_offset(show->btf,
		866	t->name_off);
		867	kinds \|= BTF_KIND_BIT(BTF_KIND_TYPEDEF);
		868	id = t->type;
		869	break;
		870	case BTF_KIND_ARRAY:
		871	kinds \|= BTF_KIND_BIT(BTF_KIND_ARRAY);
		872	parens = "[";
		873	if (!t)
		874	return "";
		875	array = btf_type_array(t);
		876	if (array_suffix > array_suffixes)
		877	array_suffix -= 2;
		878	id = array->type;
		879	break;
		880	case BTF_KIND_PTR:
		881	kinds \|= BTF_KIND_BIT(BTF_KIND_PTR);
		882	if (ptr_suffix > ptr_suffixes)
		883	ptr_suffix -= 1;
		884	id = t->type;
		885	break;
		886	default:
		887	id = 0;
		888	break;
		889	}
		890	if (!id)
		891	break;
		892	t = btf_type_skip_qualifiers(show->btf, id);
		893	}
		894	/* We may not be able to represent this type; bail to be safe */
		895	if (i == BTF_SHOW_MAX_ITER)
		896	return "";
		897
		898	if (!name)
		899	name = btf_name_by_offset(show->btf, t->name_off);
		900
		901	switch (BTF_INFO_KIND(t->info)) {
		902	case BTF_KIND_STRUCT:
		903	case BTF_KIND_UNION:
		904	prefix = BTF_INFO_KIND(t->info) == BTF_KIND_STRUCT ?
		905	"struct" : "union";
		906	/* if it's an array of struct/union, parens is already set */
		907	if (!(kinds & (BTF_KIND_BIT(BTF_KIND_ARRAY))))
		908	parens = "{";
		909	break;
		910	case BTF_KIND_ENUM:
		911	prefix = "enum";
		912	break;
		913	default:
		914	break;
		915	}
		916
		917	/* pointer does not require parens */
		918	if (kinds & BTF_KIND_BIT(BTF_KIND_PTR))
		919	parens = "";
		920	/* typedef does not require struct/union/enum prefix */
		921	if (kinds & BTF_KIND_BIT(BTF_KIND_TYPEDEF))
		922	prefix = "";
		923
		924	if (!name)
		925	name = "";
		926
		927	/* Even if we don't want type name info, we want parentheses etc */
		928	if (show->flags & BTF_SHOW_NONAME)
		929	snprintf(show->state.name, sizeof(show->state.name), "%s",
		930	parens);
		931	else
		932	snprintf(show->state.name, sizeof(show->state.name),
		933	"%s%s%s(%s%s%s%s%s%s)%s",
		934	/* first 3 strings comprise ".member = " */
		935	show_member ? "." : "",
		936	show_member ? member : "",
		937	show_member ? " = " : "",
		938	/* ...next is our prefix (struct, enum, etc) */
		939	prefix,
		940	strlen(prefix) > 0 && strlen(name) > 0 ? " " : "",
		941	/* ...this is the type name itself */
		942	name,
		943	/* ...suffixed by the appropriate '', '[]' suffixes /
		944	strlen(ptr_suffix) > 0 ? " " : "", ptr_suffix,
		945	array_suffix, parens);
		946
		947	return show->state.name;
		948	}
		949
		950	static const char __btf_show_indent(struct btf_show show)
		951	{
		952	const char *indents = " ";
		953	const char *indent = &indents[strlen(indents)];
		954
		955	if ((indent - show->state.depth) >= indents)
		956	return indent - show->state.depth;
		957	return indents;
		958	}
		959
		960	static const char btf_show_indent(struct btf_show show)
		961	{
		962	return show->flags & BTF_SHOW_COMPACT ? "" : __btf_show_indent(show);
		963	}
		964
		965	static const char btf_show_newline(struct btf_show show)
		966	{
		967	return show->flags & BTF_SHOW_COMPACT ? "" : "\n";
		968	}
		969
		970	static const char btf_show_delim(struct btf_show show)
		971	{
		972	if (show->state.depth == 0)
		973	return "";
		974
		975	if ((show->flags & BTF_SHOW_COMPACT) && show->state.type &&
		976	BTF_INFO_KIND(show->state.type->info) == BTF_KIND_UNION)
		977	return "\|";
		978
		979	return ",";
		980	}
		981
		982	__printf(2, 3) static void btf_show(struct btf_show show, const char fmt, ...)
		983	{
		984	va_list args;
		985
		986	if (!show->state.depth_check) {
		987	va_start(args, fmt);
		988	show->showfn(show, fmt, args);
		989	va_end(args);
		990	}
		991	}
		992
		993	/* Macros are used here as btf_show_type_value[s]() prepends and appends
		994	* format specifiers to the format specifier passed in; these do the work of
		995	* adding indentation, delimiters etc while the caller simply has to specify
		996	* the type value(s) in the format specifier + value(s).
		997	*/
		998	#define btf_show_type_value(show, fmt, value) \
		999	do { \
		1000	if ((value) != 0 \|\| (show->flags & BTF_SHOW_ZERO) \|\| \
		1001	show->state.depth == 0) { \
		1002	btf_show(show, "%s%s" fmt "%s%s", \
		1003	btf_show_indent(show), \
		1004	btf_show_name(show), \
		1005	value, btf_show_delim(show), \
		1006	btf_show_newline(show)); \
		1007	if (show->state.depth > show->state.depth_to_show) \
		1008	show->state.depth_to_show = show->state.depth; \
		1009	} \
		1010	} while (0)
		1011
		1012	#define btf_show_type_values(show, fmt, ...) \
		1013	do { \
		1014	btf_show(show, "%s%s" fmt "%s%s", btf_show_indent(show), \
		1015	btf_show_name(show), \
		1016	__VA_ARGS__, btf_show_delim(show), \
		1017	btf_show_newline(show)); \
		1018	if (show->state.depth > show->state.depth_to_show) \
		1019	show->state.depth_to_show = show->state.depth; \
		1020	} while (0)
		1021
		1022	/* How much is left to copy to safe buffer after @data? */
		1023	static int btf_show_obj_size_left(struct btf_show show, void data)
		1024	{
		1025	return show->obj.head + show->obj.size - data;
		1026	}
		1027
		1028	/* Is object pointed to by @data of @size already copied to our safe buffer? */
		1029	static bool btf_show_obj_is_safe(struct btf_show show, void data, int size)
		1030	{
		1031	return data >= show->obj.data &&
		1032	(data + size) < (show->obj.data + BTF_SHOW_OBJ_SAFE_SIZE);
		1033	}
		1034
		1035	/*
		1036	* If object pointed to by @data of @size falls within our safe buffer, return
		1037	* the equivalent pointer to the same safe data. Assumes
		1038	* copy_from_kernel_nofault() has already happened and our safe buffer is
		1039	* populated.
		1040	*/
		1041	static void __btf_show_obj_safe(struct btf_show show, void *data, int size)
		1042	{
		1043	if (btf_show_obj_is_safe(show, data, size))
		1044	return show->obj.safe + (data - show->obj.data);
		1045	return NULL;
		1046	}
		1047
		1048	/*
		1049	* Return a safe-to-access version of data pointed to by @data.
		1050	* We do this by copying the relevant amount of information
		1051	* to the struct btf_show obj.safe buffer using copy_from_kernel_nofault().
		1052	*
		1053	* If BTF_SHOW_UNSAFE is specified, just return data as-is; no
		1054	* safe copy is needed.
		1055	*
		1056	* Otherwise we need to determine if we have the required amount
		1057	* of data (determined by the @data pointer and the size of the
		1058	* largest base type we can encounter (represented by
		1059	* BTF_SHOW_OBJ_BASE_TYPE_SIZE). Having that much data ensures
		1060	* that we will be able to print some of the current object,
		1061	* and if more is needed a copy will be triggered.
		1062	* Some objects such as structs will not fit into the buffer;
		1063	* in such cases additional copies when we iterate over their
		1064	* members may be needed.
		1065	*
		1066	* btf_show_obj_safe() is used to return a safe buffer for
		1067	* btf_show_start_type(); this ensures that as we recurse into
		1068	* nested types we always have safe data for the given type.
		1069	* This approach is somewhat wasteful; it's possible for example
		1070	* that when iterating over a large union we'll end up copying the
		1071	* same data repeatedly, but the goal is safety not performance.
		1072	* We use stack data as opposed to per-CPU buffers because the
		1073	* iteration over a type can take some time, and preemption handling
		1074	* would greatly complicate use of the safe buffer.
		1075	*/
		1076	static void btf_show_obj_safe(struct btf_show show,
		1077	const struct btf_type *t,
		1078	void *data)
		1079	{
		1080	const struct btf_type *rt;
		1081	int size_left, size;
		1082	void *safe = NULL;
		1083
		1084	if (show->flags & BTF_SHOW_UNSAFE)
		1085	return data;
		1086
		1087	rt = btf_resolve_size(show->btf, t, &size);
		1088	if (IS_ERR(rt)) {
		1089	show->state.status = PTR_ERR(rt);
		1090	return NULL;
		1091	}
		1092
		1093	/*
		1094	* Is this toplevel object? If so, set total object size and
		1095	* initialize pointers. Otherwise check if we still fall within
		1096	* our safe object data.
		1097	*/
		1098	if (show->state.depth == 0) {
		1099	show->obj.size = size;
		1100	show->obj.head = data;
		1101	} else {
		1102	/*
		1103	* If the size of the current object is > our remaining
		1104	* safe buffer we _may_ need to do a new copy. However
		1105	* consider the case of a nested struct; it's size pushes
		1106	* us over the safe buffer limit, but showing any individual
		1107	* struct members does not. In such cases, we don't need
		1108	* to initiate a fresh copy yet; however we definitely need
		1109	* at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes left
		1110	* in our buffer, regardless of the current object size.
		1111	* The logic here is that as we resolve types we will
		1112	* hit a base type at some point, and we need to be sure
		1113	* the next chunk of data is safely available to display
		1114	* that type info safely. We cannot rely on the size of
		1115	* the current object here because it may be much larger
		1116	* than our current buffer (e.g. task_struct is 8k).
		1117	* All we want to do here is ensure that we can print the
		1118	* next basic type, which we can if either
		1119	* - the current type size is within the safe buffer; or
		1120	* - at least BTF_SHOW_OBJ_BASE_TYPE_SIZE bytes are left in
		1121	* the safe buffer.
		1122	*/
		1123	safe = __btf_show_obj_safe(show, data,
		1124	min(size,
		1125	BTF_SHOW_OBJ_BASE_TYPE_SIZE));
		1126	}
		1127
		1128	/*
		1129	* We need a new copy to our safe object, either because we haven't
		1130	* yet copied and are intializing safe data, or because the data
		1131	* we want falls outside the boundaries of the safe object.
		1132	*/
		1133	if (!safe) {
		1134	size_left = btf_show_obj_size_left(show, data);
		1135	if (size_left > BTF_SHOW_OBJ_SAFE_SIZE)
		1136	size_left = BTF_SHOW_OBJ_SAFE_SIZE;
		1137	show->state.status = copy_from_kernel_nofault(show->obj.safe,
		1138	data, size_left);
		1139	if (!show->state.status) {
		1140	show->obj.data = data;
		1141	safe = show->obj.safe;
		1142	}
		1143	}
		1144
		1145	return safe;
		1146	}
		1147
		1148	/*
		1149	* Set the type we are starting to show and return a safe data pointer
		1150	* to be used for showing the associated data.
		1151	*/
		1152	static void btf_show_start_type(struct btf_show show,
		1153	const struct btf_type *t,
		1154	u32 type_id, void *data)
		1155	{
		1156	show->state.type = t;
		1157	show->state.type_id = type_id;
		1158	show->state.name[0] = '\0';
		1159
		1160	return btf_show_obj_safe(show, t, data);
		1161	}
		1162
		1163	static void btf_show_end_type(struct btf_show *show)
		1164	{
		1165	show->state.type = NULL;
		1166	show->state.type_id = 0;
		1167	show->state.name[0] = '\0';
		1168	}
		1169
		1170	static void btf_show_start_aggr_type(struct btf_show show,
		1171	const struct btf_type *t,
		1172	u32 type_id, void *data)
		1173	{
		1174	void *safe_data = btf_show_start_type(show, t, type_id, data);
		1175
		1176	if (!safe_data)
		1177	return safe_data;
		1178
		1179	btf_show(show, "%s%s%s", btf_show_indent(show),
		1180	btf_show_name(show),
		1181	btf_show_newline(show));
		1182	show->state.depth++;
		1183	return safe_data;
		1184	}
		1185
		1186	static void btf_show_end_aggr_type(struct btf_show *show,
		1187	const char *suffix)
		1188	{
		1189	show->state.depth--;
		1190	btf_show(show, "%s%s%s%s", btf_show_indent(show), suffix,
		1191	btf_show_delim(show), btf_show_newline(show));
		1192	btf_show_end_type(show);
		1193	}
		1194
		1195	static void btf_show_start_member(struct btf_show *show,
		1196	const struct btf_member *m)
		1197	{
		1198	show->state.member = m;
		1199	}
		1200
		1201	static void btf_show_start_array_member(struct btf_show *show)
		1202	{
		1203	show->state.array_member = 1;
		1204	btf_show_start_member(show, NULL);
		1205	}
		1206
		1207	static void btf_show_end_member(struct btf_show *show)
		1208	{
		1209	show->state.member = NULL;
		1210	}
		1211
		1212	static void btf_show_end_array_member(struct btf_show *show)
		1213	{
		1214	show->state.array_member = 0;
		1215	btf_show_end_member(show);
		1216	}
		1217
		1218	static void btf_show_start_array_type(struct btf_show show,
		1219	const struct btf_type *t,
		1220	u32 type_id,
		1221	u16 array_encoding,
		1222	void *data)
		1223	{
		1224	show->state.array_encoding = array_encoding;
		1225	show->state.array_terminated = 0;
		1226	return btf_show_start_aggr_type(show, t, type_id, data);
		1227	}
		1228
		1229	static void btf_show_end_array_type(struct btf_show *show)
		1230	{
		1231	show->state.array_encoding = 0;
		1232	show->state.array_terminated = 0;
		1233	btf_show_end_aggr_type(show, "]");
		1234	}
		1235
		1236	static void btf_show_start_struct_type(struct btf_show show,
		1237	const struct btf_type *t,
		1238	u32 type_id,
		1239	void *data)
		1240	{
		1241	return btf_show_start_aggr_type(show, t, type_id, data);
		1242	}
		1243
		1244	static void btf_show_end_struct_type(struct btf_show *show)
		1245	{
		1246	btf_show_end_aggr_type(show, "}");
		1247	}
		1248
682	__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,	1249	__printf(2, 3) static void __btf_verifier_log(struct bpf_verifier_log *log,
683	const char *fmt, ...)	1250	const char *fmt, ...)
684	{	1251	{
@@ -1268,11 +1835,11 @@ static int btf_df_resolve(struct btf_verifier_env *env, · 583 unmodified lines
1268	return -EINVAL;	1835	return -EINVAL;
1269	}	1836	}
1270		1837
1271	static void btf_df_seq_show(const struct btf btf, const struct btf_type t,	1838	static void btf_df_show(const struct btf btf, const struct btf_type t,
1272	u32 type_id, void *data, u8 bits_offsets,	1839	u32 type_id, void *data, u8 bits_offsets,
1273	struct seq_file *m)	1840	struct btf_show *show)
1274	{	1841	{
1275	seq_printf(m, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));	1842	btf_show(show, "<unsupported kind:%u>", BTF_INFO_KIND(t->info));
1276	}	1843	}
1277		1844
1278	static int btf_int_check_member(struct btf_verifier_env *env,	1845	static int btf_int_check_member(struct btf_verifier_env *env,
@@ -1445,7 +2012,7 @@ static void btf_int_log(struct btf_verifier_env *env, · 166 unmodified lines
1445	btf_int_encoding_str(BTF_INT_ENCODING(int_data)));	2012	btf_int_encoding_str(BTF_INT_ENCODING(int_data)));
1446	}	2013	}
1447		2014
1448	static void btf_int128_print(struct seq_file m, void data)	2015	static void btf_int128_print(struct btf_show show, void data)
1449	{	2016	{
1450	/* data points to a __int128 number.	2017	/* data points to a __int128 number.
1451	* Suppose	2018	* Suppose
@@ -1464,9 +2031,10 @@ static void btf_int128_print(struct seq_file m, void data) · 12 unmodified lines
1464	lower_num = (u64 )data;	2031	lower_num = (u64 )data;
1465	#endif	2032	#endif
1466	if (upper_num == 0)	2033	if (upper_num == 0)
1467	seq_printf(m, "0x%llx", lower_num);	2034	btf_show_type_value(show, "0x%llx", lower_num);
1468	else	2035	else
1469	seq_printf(m, "0x%llx%016llx", upper_num, lower_num);	2036	btf_show_type_values(show, "0x%llx%016llx", upper_num,
		2037	lower_num);
1470	}	2038	}
1471		2039
1472	static void btf_int128_shift(u64 *print_num, u16 left_shift_bits,	2040	static void btf_int128_shift(u64 *print_num, u16 left_shift_bits,
@@ -1510,8 +2078,8 @@ static void btf_int128_shift(u64 *print_num, u16 left_shift_bits, · 37 unmodified lines
1510	#endif	2078	#endif
1511	}	2079	}
1512		2080
1513	static void btf_bitfield_seq_show(void *data, u8 bits_offset,	2081	static void btf_bitfield_show(void *data, u8 bits_offset,
1514	u8 nr_bits, struct seq_file *m)	2082	u8 nr_bits, struct btf_show *show)
1515	{	2083	{
1516	u16 left_shift_bits, right_shift_bits;	2084	u16 left_shift_bits, right_shift_bits;
1517	u8 nr_copy_bytes;	2085	u8 nr_copy_bytes;
@@ -1531,14 +2099,14 @@ static void btf_bitfield_seq_show(void *data, u8 bits_offset, · 13 unmodified lines
1531	right_shift_bits = BITS_PER_U128 - nr_bits;	2099	right_shift_bits = BITS_PER_U128 - nr_bits;
1532		2100
1533	btf_int128_shift(print_num, left_shift_bits, right_shift_bits);	2101	btf_int128_shift(print_num, left_shift_bits, right_shift_bits);
1534	btf_int128_print(m, print_num);	2102	btf_int128_print(show, print_num);
1535	}	2103	}
1536		2104
1537		2105
1538	static void btf_int_bits_seq_show(const struct btf *btf,	2106	static void btf_int_bits_show(const struct btf *btf,
1539	const struct btf_type *t,	2107	const struct btf_type *t,
1540	void *data, u8 bits_offset,	2108	void *data, u8 bits_offset,
1541	struct seq_file *m)	2109	struct btf_show *show)
1542	{	2110	{
1543	u32 int_data = btf_type_int(t);	2111	u32 int_data = btf_type_int(t);
1544	u8 nr_bits = BTF_INT_BITS(int_data);	2112	u8 nr_bits = BTF_INT_BITS(int_data);
@@ -1551,55 +2119,77 @@ static void btf_int_bits_seq_show(const struct btf *btf, · 6 unmodified lines
1551	total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);	2119	total_bits_offset = bits_offset + BTF_INT_OFFSET(int_data);
1552	data += BITS_ROUNDDOWN_BYTES(total_bits_offset);	2120	data += BITS_ROUNDDOWN_BYTES(total_bits_offset);
1553	bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);	2121	bits_offset = BITS_PER_BYTE_MASKED(total_bits_offset);
1554	btf_bitfield_seq_show(data, bits_offset, nr_bits, m);	2122	btf_bitfield_show(data, bits_offset, nr_bits, show);
1555	}	2123	}
1556		2124
1557	static void btf_int_seq_show(const struct btf btf, const struct btf_type t,	2125	static void btf_int_show(const struct btf btf, const struct btf_type t,
1558	u32 type_id, void *data, u8 bits_offset,	2126	u32 type_id, void *data, u8 bits_offset,
1559	struct seq_file *m)	2127	struct btf_show *show)
1560	{	2128	{
1561	u32 int_data = btf_type_int(t);	2129	u32 int_data = btf_type_int(t);
1562	u8 encoding = BTF_INT_ENCODING(int_data);	2130	u8 encoding = BTF_INT_ENCODING(int_data);
1563	bool sign = encoding & BTF_INT_SIGNED;	2131	bool sign = encoding & BTF_INT_SIGNED;
1564	u8 nr_bits = BTF_INT_BITS(int_data);	2132	u8 nr_bits = BTF_INT_BITS(int_data);
		2133	void *safe_data;
		2134
		2135	safe_data = btf_show_start_type(show, t, type_id, data);
		2136	if (!safe_data)
		2137	return;
1565		2138
1566	if (bits_offset \|\| BTF_INT_OFFSET(int_data) \|\|	2139	if (bits_offset \|\| BTF_INT_OFFSET(int_data) \|\|
1567	BITS_PER_BYTE_MASKED(nr_bits)) {	2140	BITS_PER_BYTE_MASKED(nr_bits)) {
1568	btf_int_bits_seq_show(btf, t, data, bits_offset, m);	2141	btf_int_bits_show(btf, t, safe_data, bits_offset, show);
1569	return;	2142	goto out;
1570	}	2143	}
1571		2144
1572	switch (nr_bits) {	2145	switch (nr_bits) {
1573	case 128:	2146	case 128:
1574	btf_int128_print(m, data);	2147	btf_int128_print(show, safe_data);
1575	break;	2148	break;
1576	case 64:	2149	case 64:
1577	if (sign)	2150	if (sign)
1578	seq_printf(m, "%lld", (s64 )data);	2151	btf_show_type_value(show, "%lld", (s64 )safe_data);
1579	else	2152	else
1580	seq_printf(m, "%llu", (u64 )data);	2153	btf_show_type_value(show, "%llu", (u64 )safe_data);
1581	break;	2154	break;
1582	case 32:	2155	case 32:
1583	if (sign)	2156	if (sign)
1584	seq_printf(m, "%d", (s32 )data);	2157	btf_show_type_value(show, "%d", (s32 )safe_data);
1585	else	2158	else
1586	seq_printf(m, "%u", (u32 )data);	2159	btf_show_type_value(show, "%u", (u32 )safe_data);
1587	break;	2160	break;
1588	case 16:	2161	case 16:
1589	if (sign)	2162	if (sign)
1590	seq_printf(m, "%d", (s16 )data);	2163	btf_show_type_value(show, "%d", (s16 )safe_data);
1591	else	2164	else
1592	seq_printf(m, "%u", (u16 )data);	2165	btf_show_type_value(show, "%u", (u16 )safe_data);
1593	break;	2166	break;
1594	case 8:	2167	case 8:
		2168	if (show->state.array_encoding == BTF_INT_CHAR) {
		2169	/* check for null terminator */
		2170	if (show->state.array_terminated)
		2171	break;
		2172	if ((char )data == '\0') {
		2173	show->state.array_terminated = 1;
		2174	break;
		2175	}
		2176	if (isprint((char )data)) {
		2177	btf_show_type_value(show, "'%c'",
		2178	(char )safe_data);
		2179	break;
		2180	}
		2181	}
1595	if (sign)	2182	if (sign)
1596	seq_printf(m, "%d", (s8 )data);	2183	btf_show_type_value(show, "%d", (s8 )safe_data);
1597	else	2184	else
1598	seq_printf(m, "%u", (u8 )data);	2185	btf_show_type_value(show, "%u", (u8 )safe_data);
1599	break;	2186	break;
1600	default:	2187	default:
1601	btf_int_bits_seq_show(btf, t, data, bits_offset, m);	2188	btf_int_bits_show(btf, t, safe_data, bits_offset, show);
		2189	break;
1602	}	2190	}
		2191	out:
		2192	btf_show_end_type(show);
1603	}	2193	}
1604		2194
1605	static const struct btf_kind_operations int_ops = {	2195	static const struct btf_kind_operations int_ops = {
@@ -1608,7 +2198,7 @@ static const struct btf_kind_operations int_ops = { · 2 unmodified lines
1608	.check_member = btf_int_check_member,	2198	.check_member = btf_int_check_member,
1609	.check_kflag_member = btf_int_check_kflag_member,	2199	.check_kflag_member = btf_int_check_kflag_member,
1610	.log_details = btf_int_log,	2200	.log_details = btf_int_log,
1611	.seq_show = btf_int_seq_show,	2201	.show = btf_int_show,
1612	};	2202	};
1613		2203
1614	static int btf_modifier_check_member(struct btf_verifier_env *env,	2204	static int btf_modifier_check_member(struct btf_verifier_env *env,
@@ -1872,34 +2462,44 @@ static int btf_ptr_resolve(struct btf_verifier_env *env, · 257 unmodified lines
1872	return 0;	2462	return 0;
1873	}	2463	}
1874		2464
1875	static void btf_modifier_seq_show(const struct btf *btf,	2465	static void btf_modifier_show(const struct btf *btf,
1876	const struct btf_type *t,	2466	const struct btf_type *t,
1877	u32 type_id, void *data,	2467	u32 type_id, void *data,
1878	u8 bits_offset, struct seq_file *m)	2468	u8 bits_offset, struct btf_show *show)
1879	{	2469	{
1880	if (btf->resolved_ids)	2470	if (btf->resolved_ids)
1881	t = btf_type_id_resolve(btf, &type_id);	2471	t = btf_type_id_resolve(btf, &type_id);
1882	else	2472	else
1883	t = btf_type_skip_modifiers(btf, type_id, NULL);	2473	t = btf_type_skip_modifiers(btf, type_id, NULL);
1884		2474
1885	btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);	2475	btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
1886	}	2476	}
1887		2477
1888	static void btf_var_seq_show(const struct btf btf, const struct btf_type t,	2478	static void btf_var_show(const struct btf btf, const struct btf_type t,
1889	u32 type_id, void *data, u8 bits_offset,	2479	u32 type_id, void *data, u8 bits_offset,
1890	struct seq_file *m)	2480	struct btf_show *show)
1891	{	2481	{
1892	t = btf_type_id_resolve(btf, &type_id);	2482	t = btf_type_id_resolve(btf, &type_id);
1893		2483
1894	btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);	2484	btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show);
1895	}	2485	}
1896		2486
1897	static void btf_ptr_seq_show(const struct btf btf, const struct btf_type t,	2487	static void btf_ptr_show(const struct btf btf, const struct btf_type t,
1898	u32 type_id, void *data, u8 bits_offset,	2488	u32 type_id, void *data, u8 bits_offset,
1899	struct seq_file *m)	2489	struct btf_show *show)
1900	{	2490	{
1901	/* It is a hashed value */	2491	void *safe_data;
1902	seq_printf(m, "%p", (void *)data);	2492
		2493	safe_data = btf_show_start_type(show, t, type_id, data);
		2494	if (!safe_data)
		2495	return;
		2496
		2497	/* It is a hashed value unless BTF_SHOW_PTR_RAW is specified */
		2498	if (show->flags & BTF_SHOW_PTR_RAW)
		2499	btf_show_type_value(show, "0x%px", (void *)safe_data);
		2500	else
		2501	btf_show_type_value(show, "0x%p", (void *)safe_data);
		2502	btf_show_end_type(show);
1903	}	2503	}
1904		2504
1905	static void btf_ref_type_log(struct btf_verifier_env *env,	2505	static void btf_ref_type_log(struct btf_verifier_env *env,
@@ -1914,7 +2514,7 @@ static struct btf_kind_operations modifier_ops = { · 8 unmodified lines
1914	.check_member = btf_modifier_check_member,	2514	.check_member = btf_modifier_check_member,
1915	.check_kflag_member = btf_modifier_check_kflag_member,	2515	.check_kflag_member = btf_modifier_check_kflag_member,
1916	.log_details = btf_ref_type_log,	2516	.log_details = btf_ref_type_log,
1917	.seq_show = btf_modifier_seq_show,	2517	.show = btf_modifier_show,
1918	};	2518	};
1919		2519
1920	static struct btf_kind_operations ptr_ops = {	2520	static struct btf_kind_operations ptr_ops = {
@@ -1923,7 +2523,7 @@ static struct btf_kind_operations ptr_ops = { · 2 unmodified lines
1923	.check_member = btf_ptr_check_member,	2523	.check_member = btf_ptr_check_member,
1924	.check_kflag_member = btf_generic_check_kflag_member,	2524	.check_kflag_member = btf_generic_check_kflag_member,
1925	.log_details = btf_ref_type_log,	2525	.log_details = btf_ref_type_log,
1926	.seq_show = btf_ptr_seq_show,	2526	.show = btf_ptr_show,
1927	};	2527	};
1928		2528
1929	static s32 btf_fwd_check_meta(struct btf_verifier_env *env,	2529	static s32 btf_fwd_check_meta(struct btf_verifier_env *env,
@@ -1964,7 +2564,7 @@ static struct btf_kind_operations fwd_ops = { · 34 unmodified lines
1964	.check_member = btf_df_check_member,	2564	.check_member = btf_df_check_member,
1965	.check_kflag_member = btf_df_check_kflag_member,	2565	.check_kflag_member = btf_df_check_kflag_member,
1966	.log_details = btf_fwd_type_log,	2566	.log_details = btf_fwd_type_log,
1967	.seq_show = btf_df_seq_show,	2567	.show = btf_df_show,
1968	};	2568	};
1969		2569
1970	static int btf_array_check_member(struct btf_verifier_env *env,	2570	static int btf_array_check_member(struct btf_verifier_env *env,
@@ -2123,28 +2723,90 @@ static void btf_array_log(struct btf_verifier_env *env, · 152 unmodified lines
2123	array->type, array->index_type, array->nelems);	2723	array->type, array->index_type, array->nelems);
2124	}	2724	}
2125		2725
2126	static void btf_array_seq_show(const struct btf btf, const struct btf_type t,	2726	static void __btf_array_show(const struct btf btf, const struct btf_type t,
2127	u32 type_id, void *data, u8 bits_offset,	2727	u32 type_id, void *data, u8 bits_offset,
2128	struct seq_file *m)	2728	struct btf_show *show)
2129	{	2729	{
2130	const struct btf_array *array = btf_type_array(t);	2730	const struct btf_array *array = btf_type_array(t);
2131	const struct btf_kind_operations *elem_ops;	2731	const struct btf_kind_operations *elem_ops;
2132	const struct btf_type *elem_type;	2732	const struct btf_type *elem_type;
2133	u32 i, elem_size, elem_type_id;	2733	u32 i, elem_size = 0, elem_type_id;
		2734	u16 encoding = 0;
2134		2735
2135	elem_type_id = array->type;	2736	elem_type_id = array->type;
2136	elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);	2737	elem_type = btf_type_skip_modifiers(btf, elem_type_id, NULL);
		2738	if (elem_type && btf_type_has_size(elem_type))
		2739	elem_size = elem_type->size;
		2740
		2741	if (elem_type && btf_type_is_int(elem_type)) {
		2742	u32 int_type = btf_type_int(elem_type);
		2743
		2744	encoding = BTF_INT_ENCODING(int_type);
		2745
		2746	/*
		2747	* BTF_INT_CHAR encoding never seems to be set for
		2748	* char arrays, so if size is 1 and element is
		2749	* printable as a char, we'll do that.
		2750	*/
		2751	if (elem_size == 1)
		2752	encoding = BTF_INT_CHAR;
		2753	}
		2754
		2755	if (!btf_show_start_array_type(show, t, type_id, encoding, data))
		2756	return;
		2757
		2758	if (!elem_type)
		2759	goto out;
2137	elem_ops = btf_type_ops(elem_type);	2760	elem_ops = btf_type_ops(elem_type);
2138	seq_puts(m, "[");	2761
2139	for (i = 0; i < array->nelems; i++) {	2762	for (i = 0; i < array->nelems; i++) {
2140	if (i)
2141	seq_puts(m, ",");
2142		2763
2143	elem_ops->seq_show(btf, elem_type, elem_type_id, data,	2764	btf_show_start_array_member(show);
2144	bits_offset, m);	2765
		2766	elem_ops->show(btf, elem_type, elem_type_id, data,
		2767	bits_offset, show);
2145	data += elem_size;	2768	data += elem_size;
		2769
		2770	btf_show_end_array_member(show);
		2771
		2772	if (show->state.array_terminated)
		2773	break;
		2774	}
		2775	out:
		2776	btf_show_end_array_type(show);
		2777	}
		2778
		2779	static void btf_array_show(const struct btf btf, const struct btf_type t,
		2780	u32 type_id, void *data, u8 bits_offset,
		2781	struct btf_show *show)
		2782	{
		2783	const struct btf_member *m = show->state.member;
		2784
		2785	/*
		2786	* First check if any members would be shown (are non-zero).
		2787	* See comments above "struct btf_show" definition for more
		2788	* details on how this works at a high-level.
		2789	*/
		2790	if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
		2791	if (!show->state.depth_check) {
		2792	show->state.depth_check = show->state.depth + 1;
		2793	show->state.depth_to_show = 0;
		2794	}
		2795	__btf_array_show(btf, t, type_id, data, bits_offset, show);
		2796	show->state.member = m;
		2797
		2798	if (show->state.depth_check != show->state.depth + 1)
		2799	return;
		2800	show->state.depth_check = 0;
		2801
		2802	if (show->state.depth_to_show <= show->state.depth)
		2803	return;
		2804	/*
		2805	* Reaching here indicates we have recursed and found
		2806	* non-zero array member(s).
		2807	*/
2146	}	2808	}
2147	seq_puts(m, "]");	2809	__btf_array_show(btf, t, type_id, data, bits_offset, show);
2148	}	2810	}
2149		2811
2150	static struct btf_kind_operations array_ops = {	2812	static struct btf_kind_operations array_ops = {
@@ -2153,7 +2815,7 @@ static struct btf_kind_operations array_ops = { · 2 unmodified lines
2153	.check_member = btf_array_check_member,	2815	.check_member = btf_array_check_member,
2154	.check_kflag_member = btf_generic_check_kflag_member,	2816	.check_kflag_member = btf_generic_check_kflag_member,
2155	.log_details = btf_array_log,	2817	.log_details = btf_array_log,
2156	.seq_show = btf_array_seq_show,	2818	.show = btf_array_show,
2157	};	2819	};
2158		2820
2159	static int btf_struct_check_member(struct btf_verifier_env *env,	2821	static int btf_struct_check_member(struct btf_verifier_env *env,
@@ -2376,15 +3038,18 @@ int btf_find_spin_lock(const struct btf btf, const struct btf_type t) · 216 unmodified lines
2376	return off;	3038	return off;
2377	}	3039	}
2378		3040
2379	static void btf_struct_seq_show(const struct btf btf, const struct btf_type t,	3041	static void __btf_struct_show(const struct btf btf, const struct btf_type t,
2380	u32 type_id, void *data, u8 bits_offset,	3042	u32 type_id, void *data, u8 bits_offset,
2381	struct seq_file *m)	3043	struct btf_show *show)
2382	{	3044	{
2383	const char *seq = BTF_INFO_KIND(t->info) == BTF_KIND_UNION ? "\|" : ",";
2384	const struct btf_member *member;	3045	const struct btf_member *member;
		3046	void *safe_data;
2385	u32 i;	3047	u32 i;
2386		3048
2387	seq_puts(m, "{");	3049	safe_data = btf_show_start_struct_type(show, t, type_id, data);
		3050	if (!safe_data)
		3051	return;
		3052
2388	for_each_member(i, t, member) {	3053	for_each_member(i, t, member) {
2389	const struct btf_type *member_type = btf_type_by_id(btf,	3054	const struct btf_type *member_type = btf_type_by_id(btf,
2390	member->type);	3055	member->type);
@@ -2393,23 +3058,65 @@ static void btf_struct_seq_show(const struct btf btf, const struct btf_type t, · 2 unmodified lines
2393	u32 bytes_offset;	3058	u32 bytes_offset;
2394	u8 bits8_offset;	3059	u8 bits8_offset;
2395		3060
2396	if (i)	3061	btf_show_start_member(show, member);
2397	seq_puts(m, seq);
2398		3062
2399	member_offset = btf_member_bit_offset(t, member);	3063	member_offset = btf_member_bit_offset(t, member);
2400	bitfield_size = btf_member_bitfield_size(t, member);	3064	bitfield_size = btf_member_bitfield_size(t, member);
2401	bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);	3065	bytes_offset = BITS_ROUNDDOWN_BYTES(member_offset);
2402	bits8_offset = BITS_PER_BYTE_MASKED(member_offset);	3066	bits8_offset = BITS_PER_BYTE_MASKED(member_offset);
2403	if (bitfield_size) {	3067	if (bitfield_size) {
2404	btf_bitfield_seq_show(data + bytes_offset, bits8_offset,	3068	safe_data = btf_show_start_type(show, member_type,
2405	bitfield_size, m);	3069	member->type,
		3070	data + bytes_offset);
		3071	if (safe_data)
		3072	btf_bitfield_show(safe_data,
		3073	bits8_offset,
		3074	bitfield_size, show);
		3075	btf_show_end_type(show);
2406	} else {	3076	} else {
2407	ops = btf_type_ops(member_type);	3077	ops = btf_type_ops(member_type);
2408	ops->seq_show(btf, member_type, member->type,	3078	ops->show(btf, member_type, member->type,
2409	data + bytes_offset, bits8_offset, m);	3079	data + bytes_offset, bits8_offset, show);
2410	}	3080	}
		3081
		3082	btf_show_end_member(show);
2411	}	3083	}
2412	seq_puts(m, "}");	3084
		3085	btf_show_end_struct_type(show);
		3086	}
		3087
		3088	static void btf_struct_show(const struct btf btf, const struct btf_type t,
		3089	u32 type_id, void *data, u8 bits_offset,
		3090	struct btf_show *show)
		3091	{
		3092	const struct btf_member *m = show->state.member;
		3093
		3094	/*
		3095	* First check if any members would be shown (are non-zero).
		3096	* See comments above "struct btf_show" definition for more
		3097	* details on how this works at a high-level.
		3098	*/
		3099	if (show->state.depth > 0 && !(show->flags & BTF_SHOW_ZERO)) {
		3100	if (!show->state.depth_check) {
		3101	show->state.depth_check = show->state.depth + 1;
		3102	show->state.depth_to_show = 0;
		3103	}
		3104	__btf_struct_show(btf, t, type_id, data, bits_offset, show);
		3105	/* Restore saved member data here */
		3106	show->state.member = m;
		3107	if (show->state.depth_check != show->state.depth + 1)
		3108	return;
		3109	show->state.depth_check = 0;
		3110
		3111	if (show->state.depth_to_show <= show->state.depth)
		3112	return;
		3113	/*
		3114	* Reaching here indicates we have recursed and found
		3115	* non-zero child values.
		3116	*/
		3117	}
		3118
		3119	__btf_struct_show(btf, t, type_id, data, bits_offset, show);
2413	}	3120	}
2414		3121
2415	static struct btf_kind_operations struct_ops = {	3122	static struct btf_kind_operations struct_ops = {
@@ -2418,7 +3125,7 @@ static struct btf_kind_operations struct_ops = { · 2 unmodified lines
2418	.check_member = btf_struct_check_member,	3125	.check_member = btf_struct_check_member,
2419	.check_kflag_member = btf_generic_check_kflag_member,	3126	.check_kflag_member = btf_generic_check_kflag_member,
2420	.log_details = btf_struct_log,	3127	.log_details = btf_struct_log,
2421	.seq_show = btf_struct_seq_show,	3128	.show = btf_struct_show,
2422	};	3129	};
2423		3130
2424	static int btf_enum_check_member(struct btf_verifier_env *env,	3131	static int btf_enum_check_member(struct btf_verifier_env *env,
@@ -2549,24 +3256,35 @@ static void btf_enum_log(struct btf_verifier_env *env, · 124 unmodified lines
2549	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));	3256	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
2550	}	3257	}
2551		3258
2552	static void btf_enum_seq_show(const struct btf btf, const struct btf_type t,	3259	static void btf_enum_show(const struct btf btf, const struct btf_type t,
2553	u32 type_id, void *data, u8 bits_offset,	3260	u32 type_id, void *data, u8 bits_offset,
2554	struct seq_file *m)	3261	struct btf_show *show)
2555	{	3262	{
2556	const struct btf_enum *enums = btf_type_enum(t);	3263	const struct btf_enum *enums = btf_type_enum(t);
2557	u32 i, nr_enums = btf_type_vlen(t);	3264	u32 i, nr_enums = btf_type_vlen(t);
2558	int v = (int )data;	3265	void *safe_data;
		3266	int v;
		3267
		3268	safe_data = btf_show_start_type(show, t, type_id, data);
		3269	if (!safe_data)
		3270	return;
		3271
		3272	v = (int )safe_data;
2559		3273
2560	for (i = 0; i < nr_enums; i++) {	3274	for (i = 0; i < nr_enums; i++) {
2561	if (v == enums[i].val) {	3275	if (v != enums[i].val)
2562	seq_printf(m, "%s",	3276	continue;
2563	__btf_name_by_offset(btf,	3277
2564	enums[i].name_off));	3278	btf_show_type_value(show, "%s",
2565	return;	3279	__btf_name_by_offset(btf,
2566	}	3280	enums[i].name_off));
		3281
		3282	btf_show_end_type(show);
		3283	return;
2567	}	3284	}
2568		3285
2569	seq_printf(m, "%d", v);	3286	btf_show_type_value(show, "%d", v);
		3287	btf_show_end_type(show);
2570	}	3288	}
2571		3289
2572	static struct btf_kind_operations enum_ops = {	3290	static struct btf_kind_operations enum_ops = {
@@ -2575,7 +3293,7 @@ static struct btf_kind_operations enum_ops = { · 2 unmodified lines
2575	.check_member = btf_enum_check_member,	3293	.check_member = btf_enum_check_member,
2576	.check_kflag_member = btf_enum_check_kflag_member,	3294	.check_kflag_member = btf_enum_check_kflag_member,
2577	.log_details = btf_enum_log,	3295	.log_details = btf_enum_log,
2578	.seq_show = btf_enum_seq_show,	3296	.show = btf_enum_show,
2579	};	3297	};
2580		3298
2581	static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,	3299	static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
@@ -2662,7 +3380,7 @@ static struct btf_kind_operations func_proto_ops = { · 80 unmodified lines
2662	.check_member = btf_df_check_member,	3380	.check_member = btf_df_check_member,
2663	.check_kflag_member = btf_df_check_kflag_member,	3381	.check_kflag_member = btf_df_check_kflag_member,
2664	.log_details = btf_func_proto_log,	3382	.log_details = btf_func_proto_log,
2665	.seq_show = btf_df_seq_show,	3383	.show = btf_df_show,
2666	};	3384	};
2667		3385
2668	static s32 btf_func_check_meta(struct btf_verifier_env *env,	3386	static s32 btf_func_check_meta(struct btf_verifier_env *env,
@@ -2696,7 +3414,7 @@ static struct btf_kind_operations func_ops = { · 27 unmodified lines
2696	.check_member = btf_df_check_member,	3414	.check_member = btf_df_check_member,
2697	.check_kflag_member = btf_df_check_kflag_member,	3415	.check_kflag_member = btf_df_check_kflag_member,
2698	.log_details = btf_ref_type_log,	3416	.log_details = btf_ref_type_log,
2699	.seq_show = btf_df_seq_show,	3417	.show = btf_df_show,
2700	};	3418	};
2701		3419
2702	static s32 btf_var_check_meta(struct btf_verifier_env *env,	3420	static s32 btf_var_check_meta(struct btf_verifier_env *env,
@@ -2760,7 +3478,7 @@ static const struct btf_kind_operations var_ops = { · 57 unmodified lines
2760	.check_member = btf_df_check_member,	3478	.check_member = btf_df_check_member,
2761	.check_kflag_member = btf_df_check_kflag_member,	3479	.check_kflag_member = btf_df_check_kflag_member,
2762	.log_details = btf_var_log,	3480	.log_details = btf_var_log,
2763	.seq_show = btf_var_seq_show,	3481	.show = btf_var_show,
2764	};	3482	};
2765		3483
2766	static s32 btf_datasec_check_meta(struct btf_verifier_env *env,	3484	static s32 btf_datasec_check_meta(struct btf_verifier_env *env,
@@ -2886,24 +3604,28 @@ static void btf_datasec_log(struct btf_verifier_env *env, · 119 unmodified lines
2886	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));	3604	btf_verifier_log(env, "size=%u vlen=%u", t->size, btf_type_vlen(t));
2887	}	3605	}
2888		3606
2889	static void btf_datasec_seq_show(const struct btf *btf,	3607	static void btf_datasec_show(const struct btf *btf,
2890	const struct btf_type *t, u32 type_id,	3608	const struct btf_type *t, u32 type_id,
2891	void *data, u8 bits_offset,	3609	void *data, u8 bits_offset,
2892	struct seq_file *m)	3610	struct btf_show *show)
2893	{	3611	{
2894	const struct btf_var_secinfo *vsi;	3612	const struct btf_var_secinfo *vsi;
2895	const struct btf_type *var;	3613	const struct btf_type *var;
2896	u32 i;	3614	u32 i;
2897		3615
2898	seq_printf(m, "section (\"%s\") = {", __btf_name_by_offset(btf, t->name_off));	3616	if (!btf_show_start_type(show, t, type_id, data))
		3617	return;
		3618
		3619	btf_show_type_value(show, "section (\"%s\") = {",
		3620	__btf_name_by_offset(btf, t->name_off));
2899	for_each_vsi(i, t, vsi) {	3621	for_each_vsi(i, t, vsi) {
2900	var = btf_type_by_id(btf, vsi->type);	3622	var = btf_type_by_id(btf, vsi->type);
2901	if (i)	3623	if (i)
2902	seq_puts(m, ",");	3624	btf_show(show, ",");
2903	btf_type_ops(var)->seq_show(btf, var, vsi->type,	3625	btf_type_ops(var)->show(btf, var, vsi->type,
2904	data + vsi->offset, bits_offset, m);	3626	data + vsi->offset, bits_offset, show);
2905	}	3627	}
2906	seq_puts(m, "}");	3628	btf_show_end_type(show);
2907	}	3629	}
2908		3630
2909	static const struct btf_kind_operations datasec_ops = {	3631	static const struct btf_kind_operations datasec_ops = {
@@ -2912,7 +3634,7 @@ static const struct btf_kind_operations datasec_ops = { · 2 unmodified lines
2912	.check_member = btf_df_check_member,	3634	.check_member = btf_df_check_member,
2913	.check_kflag_member = btf_df_check_kflag_member,	3635	.check_kflag_member = btf_df_check_kflag_member,
2914	.log_details = btf_datasec_log,	3636	.log_details = btf_datasec_log,
2915	.seq_show = btf_datasec_seq_show,	3637	.show = btf_datasec_show,
2916	};	3638	};
2917		3639
2918	static int btf_func_proto_check(struct btf_verifier_env *env,	3640	static int btf_func_proto_check(struct btf_verifier_env *env,
@@ -3706,7 +4428,7 @@ errout: · 787 unmodified lines
3706		4428
3707	struct btf bpf_prog_get_target_btf(const struct bpf_prog prog)	4429	struct btf bpf_prog_get_target_btf(const struct bpf_prog prog)
3708	{	4430	{
3709	struct bpf_prog *tgt_prog = prog->aux->linked_prog;	4431	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
3710		4432
3711	if (tgt_prog) {	4433	if (tgt_prog) {
3712	return tgt_prog->aux->btf;	4434	return tgt_prog->aux->btf;
@@ -3733,7 +4455,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 20 unmodified lines
3733	struct bpf_insn_access_aux *info)	4455	struct bpf_insn_access_aux *info)
3734	{	4456	{
3735	const struct btf_type *t = prog->aux->attach_func_proto;	4457	const struct btf_type *t = prog->aux->attach_func_proto;
3736	struct bpf_prog *tgt_prog = prog->aux->linked_prog;	4458	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
3737	struct btf *btf = bpf_prog_get_target_btf(prog);	4459	struct btf *btf = bpf_prog_get_target_btf(prog);
3738	const char *tname = prog->aux->attach_func_name;	4460	const char *tname = prog->aux->attach_func_name;
3739	struct bpf_verifier_log *log = info->log;	4461	struct bpf_verifier_log *log = info->log;
@@ -3860,7 +4582,14 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 120 unmodified lines
3860		4582
3861	info->reg_type = PTR_TO_BTF_ID;	4583	info->reg_type = PTR_TO_BTF_ID;
3862	if (tgt_prog) {	4584	if (tgt_prog) {
3863	ret = btf_translate_to_vmlinux(log, btf, t, tgt_prog->type, arg);	4585	enum bpf_prog_type tgt_type;
		4586
		4587	if (tgt_prog->type == BPF_PROG_TYPE_EXT)
		4588	tgt_type = tgt_prog->aux->saved_dst_prog_type;
		4589	else
		4590	tgt_type = tgt_prog->type;
		4591
		4592	ret = btf_translate_to_vmlinux(log, btf, t, tgt_type, arg);
3864	if (ret > 0) {	4593	if (ret > 0) {
3865	info->btf_id = ret;	4594	info->btf_id = ret;
3866	return true;	4595	return true;
@@ -4388,7 +5117,7 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, · 521 unmodified lines
4388	}	5117	}
4389		5118
4390	/* Compare BTFs of given program with BTF of target program */	5119	/* Compare BTFs of given program with BTF of target program */
4391	int btf_check_type_match(struct bpf_verifier_env env, struct bpf_prog prog,	5120	int btf_check_type_match(struct bpf_verifier_log log, const struct bpf_prog prog,
4392	struct btf btf2, const struct btf_type t2)	5121	struct btf btf2, const struct btf_type t2)
4393	{	5122	{
4394	struct btf *btf1 = prog->aux->btf;	5123	struct btf *btf1 = prog->aux->btf;
@@ -4396,7 +5125,7 @@ int btf_check_type_match(struct bpf_verifier_env env, struct bpf_prog prog, · 1 unmodified line
4396	u32 btf_id = 0;	5125	u32 btf_id = 0;
4397		5126
4398	if (!prog->aux->func_info) {	5127	if (!prog->aux->func_info) {
4399	bpf_log(&env->log, "Program extension requires BTF\n");	5128	bpf_log(log, "Program extension requires BTF\n");
4400	return -EINVAL;	5129	return -EINVAL;
4401	}	5130	}
4402		5131
@@ -4408,7 +5137,7 @@ int btf_check_type_match(struct bpf_verifier_env env, struct bpf_prog prog, · 5 unmodified lines
4408	if (!t1 \|\| !btf_type_is_func(t1))	5137	if (!t1 \|\| !btf_type_is_func(t1))
4409	return -EFAULT;	5138	return -EFAULT;
4410		5139
4411	return btf_check_func_type_match(&env->log, btf1, t1, btf2, t2);	5140	return btf_check_func_type_match(log, btf1, t1, btf2, t2);
4412	}	5141	}
4413		5142
4414	/* Compare BTF of a function with given bpf_reg_state.	5143	/* Compare BTF of a function with given bpf_reg_state.
@@ -4559,7 +5288,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, · 144 unmodified lines
4559	return -EFAULT;	5288	return -EFAULT;
4560	}	5289	}
4561	if (prog_type == BPF_PROG_TYPE_EXT)	5290	if (prog_type == BPF_PROG_TYPE_EXT)
4562	prog_type = prog->aux->linked_prog->type;	5291	prog_type = prog->aux->dst_prog->type;
4563		5292
4564	t = btf_type_by_id(btf, t->type);	5293	t = btf_type_by_id(btf, t->type);
4565	if (!t \|\| !btf_type_is_func_proto(t)) {	5294	if (!t \|\| !btf_type_is_func_proto(t)) {
@@ -4606,12 +5335,93 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog, · 40 unmodified lines
4606	return 0;	5335	return 0;
4607	}	5336	}
4608		5337
		5338	static void btf_type_show(const struct btf btf, u32 type_id, void obj,
		5339	struct btf_show *show)
		5340	{
		5341	const struct btf_type *t = btf_type_by_id(btf, type_id);
		5342
		5343	show->btf = btf;
		5344	memset(&show->state, 0, sizeof(show->state));
		5345	memset(&show->obj, 0, sizeof(show->obj));
		5346
		5347	btf_type_ops(t)->show(btf, t, type_id, obj, 0, show);
		5348	}
		5349
		5350	static void btf_seq_show(struct btf_show show, const char fmt,
		5351	va_list args)
		5352	{
		5353	seq_vprintf((struct seq_file *)show->target, fmt, args);
		5354	}
		5355
		5356	int btf_type_seq_show_flags(const struct btf *btf, u32 type_id,
		5357	void obj, struct seq_file m, u64 flags)
		5358	{
		5359	struct btf_show sseq;
		5360
		5361	sseq.target = m;
		5362	sseq.showfn = btf_seq_show;
		5363	sseq.flags = flags;
		5364
		5365	btf_type_show(btf, type_id, obj, &sseq);
		5366
		5367	return sseq.state.status;
		5368	}
		5369
4609	void btf_type_seq_show(const struct btf btf, u32 type_id, void obj,	5370	void btf_type_seq_show(const struct btf btf, u32 type_id, void obj,
4610	struct seq_file *m)	5371	struct seq_file *m)
4611	{	5372	{
4612	const struct btf_type *t = btf_type_by_id(btf, type_id);	5373	(void) btf_type_seq_show_flags(btf, type_id, obj, m,
		5374	BTF_SHOW_NONAME \| BTF_SHOW_COMPACT \|
		5375	BTF_SHOW_ZERO \| BTF_SHOW_UNSAFE);
		5376	}
		5377
		5378	struct btf_show_snprintf {
		5379	struct btf_show show;
		5380	int len_left; /* space left in string */
		5381	int len; /* length we would have written */
		5382	};
		5383
		5384	static void btf_snprintf_show(struct btf_show show, const char fmt,
		5385	va_list args)
		5386	{
		5387	struct btf_show_snprintf ssnprintf = (struct btf_show_snprintf )show;
		5388	int len;
		5389
		5390	len = vsnprintf(show->target, ssnprintf->len_left, fmt, args);
		5391
		5392	if (len < 0) {
		5393	ssnprintf->len_left = 0;
		5394	ssnprintf->len = len;
		5395	} else if (len > ssnprintf->len_left) {
		5396	/* no space, drive on to get length we would have written */
		5397	ssnprintf->len_left = 0;
		5398	ssnprintf->len += len;
		5399	} else {
		5400	ssnprintf->len_left -= len;
		5401	ssnprintf->len += len;
		5402	show->target += len;
		5403	}
		5404	}
		5405
		5406	int btf_type_snprintf_show(const struct btf btf, u32 type_id, void obj,
		5407	char *buf, int len, u64 flags)
		5408	{
		5409	struct btf_show_snprintf ssnprintf;
		5410
		5411	ssnprintf.show.target = buf;
		5412	ssnprintf.show.flags = flags;
		5413	ssnprintf.show.showfn = btf_snprintf_show;
		5414	ssnprintf.len_left = len;
		5415	ssnprintf.len = 0;
		5416
		5417	btf_type_show(btf, type_id, obj, (struct btf_show *)&ssnprintf);
		5418
		5419	/* If we encontered an error, return it. */
		5420	if (ssnprintf.show.state.status)
		5421	return ssnprintf.show.state.status;
4613		5422
4614	btf_type_ops(t)->seq_show(btf, t, type_id, obj, 0, m);	5423	/* Otherwise return length we would have written */
		5424	return ssnprintf.len;
4615	}	5425	}
4616		5426
4617	#ifdef CONFIG_PROC_FS	5427	#ifdef CONFIG_PROC_FS

kernel/bpf/core.c+8−3

 	INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode);
 	mutex_init(&fp->aux->used_maps_mutex);
+	mutex_init(&fp->aux->dst_mutex);
 	return fp;
 }
 {
 	if (fp->aux) {
 		mutex_destroy(&fp->aux->used_maps_mutex);
+		mutex_destroy(&fp->aux->dst_mutex);
 		free_percpu(fp->aux->stats);
 		kfree(fp->aux->poke_tab);
 		kfree(fp->aux);
 	if (aux->prog->has_callchain_buf)
 		put_callchain_buffers();
 #endif
-	bpf_trampoline_put(aux->trampoline);
+	if (aux->dst_trampoline)
+		bpf_trampoline_put(aux->dst_trampoline);
 	for (i = 0; i < aux->func_cnt; i++)
 		bpf_jit_free(aux->func[i]);
 	if (aux->func_cnt) {
 {
 	struct bpf_prog_aux *aux = fp->aux;
-	if (aux->linked_prog)
+	if (aux->dst_prog)
-		bpf_prog_put(aux->linked_prog);
+		bpf_prog_put(aux->dst_prog);
 	INIT_WORK(&aux->work, bpf_prog_free_deferred);
 	schedule_work(&aux->work);
 }
 const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto __weak;
 const struct bpf_func_proto bpf_get_local_storage_proto __weak;
 const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto __weak;
+const struct bpf_func_proto bpf_snprintf_btf_proto __weak;
+const struct bpf_func_proto bpf_seq_printf_btf_proto __weak;
 const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void)
 {

kernel/bpf/cpumap.c+2−3

 	kthread_stop(rcpu->kthread);
 }
-static struct sk_buff *cpu_map_build_skb(struct bpf_cpu_map_entry *rcpu,
+static struct sk_buff *cpu_map_build_skb(struct xdp_frame *xdpf,
-					 struct xdp_frame *xdpf,
 					 struct sk_buff *skb)
 {
 	unsigned int hard_start_headroom;
 			struct sk_buff *skb = skbs[i];
 			int ret;
-			skb = cpu_map_build_skb(rcpu, xdpf, skb);
+			skb = cpu_map_build_skb(xdpf, skb);
 			if (!skb) {
 				xdp_return_frame(xdpf);
 				continue;

kernel/bpf/helpers.c+4−0

 		if (!perfmon_capable())
 			return NULL;
 		return bpf_get_trace_printk_proto();
+	case BPF_FUNC_snprintf_btf:
+		if (!perfmon_capable())
+			return NULL;
+		return &bpf_snprintf_btf_proto;
 	case BPF_FUNC_jiffies64:
 		return &bpf_jiffies64_proto;
 	default:

kernel/bpf/preload/.gitignore+4−0

@@ -0,0 +1,4 @@
	1	/FEATURE-DUMP.libbpf
	2	/bpf_helper_defs.h
	3	/feature
	4	/bpf_preload_umd

kernel/bpf/preload/Makefile+2−0

@@ -12,6 +12,8 @@ userccflags += -I $(srctree)/tools/include/ -I $(srctree)/tools/include/uapi \ · 11 unmodified lines
12		12
13	userprogs := bpf_preload_umd	13	userprogs := bpf_preload_umd
14		14
		15	clean-files := $(userprogs) bpf_helper_defs.h FEATURE-DUMP.libbpf staticobjs/ feature/
		16
15	bpf_preload_umd-objs := iterators/iterators.o	17	bpf_preload_umd-objs := iterators/iterators.o
16	bpf_preload_umd-userldlibs := $(LIBBPF_A) -lelf -lz	18	bpf_preload_umd-userldlibs := $(LIBBPF_A) -lelf -lz
17		19

kernel/bpf/preload/iterators/iterators.bpf.c+2−2

@@ -42,7 +42,7 @@ struct bpf_prog_aux { · 41 unmodified lines
42	__u32 id;	42	__u32 id;
43	char name[16];	43	char name[16];
44	const char *attach_func_name;	44	const char *attach_func_name;
45	struct bpf_prog *linked_prog;	45	struct bpf_prog *dst_prog;
46	struct bpf_func_info *func_info;	46	struct bpf_func_info *func_info;
47	struct btf *btf;	47	struct btf *btf;
48	};	48	};
@@ -108,7 +108,7 @@ int dump_bpf_prog(struct bpf_iter__bpf_prog *ctx) · 59 unmodified lines
108		108
109	BPF_SEQ_PRINTF(seq, "%4u %-16s %s %s\n", aux->id,	109	BPF_SEQ_PRINTF(seq, "%4u %-16s %s %s\n", aux->id,
110	get_name(aux->btf, aux->func_info[0].type_id, aux->name),	110	get_name(aux->btf, aux->func_info[0].type_id, aux->name),
111	aux->attach_func_name, aux->linked_prog->aux->name);	111	aux->attach_func_name, aux->dst_prog->aux->name);
112	return 0;	112	return 0;
113	}	113	}
114	char LICENSE[] SEC("license") = "GPL";	114	char LICENSE[] SEC("license") = "GPL";

kernel/bpf/preload/iterators/iterators.skel.h+223−221

@@ -47,7 +47,7 @@ iterators_bpf__open_opts(const struct bpf_object_open_opts *opts) · 46 unmodified lines
47	{	47	{
48	struct iterators_bpf *obj;	48	struct iterators_bpf *obj;
49		49
50	obj = (typeof(obj))calloc(1, sizeof(*obj));	50	obj = (struct iterators_bpf )calloc(1, sizeof(obj));
51	if (!obj)	51	if (!obj)
52	return NULL;	52	return NULL;
53	if (iterators_bpf__create_skeleton(obj))	53	if (iterators_bpf__create_skeleton(obj))
@@ -105,7 +105,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) · 51 unmodified lines
105	{	105	{
106	struct bpf_object_skeleton *s;	106	struct bpf_object_skeleton *s;
107		107
108	s = (typeof(s))calloc(1, sizeof(*s));	108	s = (struct bpf_object_skeleton )calloc(1, sizeof(s));
109	if (!s)	109	if (!s)
110	return -1;	110	return -1;
111	obj->skeleton = s;	111	obj->skeleton = s;
@@ -117,7 +117,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) · 5 unmodified lines
117	/* maps */	117	/* maps */
118	s->map_cnt = 1;	118	s->map_cnt = 1;
119	s->map_skel_sz = sizeof(*s->maps);	119	s->map_skel_sz = sizeof(*s->maps);
120	s->maps = (typeof(s->maps))calloc(s->map_cnt, s->map_skel_sz);	120	s->maps = (struct bpf_map_skeleton *)calloc(s->map_cnt, s->map_skel_sz);
121	if (!s->maps)	121	if (!s->maps)
122	goto err;	122	goto err;
123		123
@@ -128,7 +128,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) · 4 unmodified lines
128	/* programs */	128	/* programs */
129	s->prog_cnt = 2;	129	s->prog_cnt = 2;
130	s->prog_skel_sz = sizeof(*s->progs);	130	s->prog_skel_sz = sizeof(*s->progs);
131	s->progs = (typeof(s->progs))calloc(s->prog_cnt, s->prog_skel_sz);	131	s->progs = (struct bpf_prog_skeleton *)calloc(s->prog_cnt, s->prog_skel_sz);
132	if (!s->progs)	132	if (!s->progs)
133	goto err;	133	goto err;
134		134
@@ -140,10 +140,10 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) · 5 unmodified lines
140	s->progs[1].prog = &obj->progs.dump_bpf_prog;	140	s->progs[1].prog = &obj->progs.dump_bpf_prog;
141	s->progs[1].link = &obj->links.dump_bpf_prog;	141	s->progs[1].link = &obj->links.dump_bpf_prog;
142		142
143	s->data_sz = 7128;	143	s->data_sz = 7176;
144	s->data = (void *)"\	144	s->data = (void *)"\
145	\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\	145	\x7f\x45\x4c\x46\x02\x01\x01\0\0\0\0\0\0\0\0\0\x01\0\xf7\0\x01\0\0\0\0\0\0\0\0\
146	\0\0\0\0\0\0\0\0\0\0\0\x18\x18\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0f\0\	146	\0\0\0\0\0\0\0\0\0\0\0\x48\x18\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\x40\0\x0f\0\
147	\x0e\0\x79\x12\0\0\0\0\0\0\x79\x26\0\0\0\0\0\0\x79\x17\x08\0\0\0\0\0\x15\x07\	147	\x0e\0\x79\x12\0\0\0\0\0\0\x79\x26\0\0\0\0\0\0\x79\x17\x08\0\0\0\0\0\x15\x07\
148	\x1a\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\	148	\x1a\0\0\0\0\0\x79\x21\x10\0\0\0\0\0\x55\x01\x08\0\0\0\0\0\xbf\xa4\0\0\0\0\0\0\
149	\x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\0\	149	\x07\x04\0\0\xe8\xff\xff\xff\xbf\x61\0\0\0\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\0\
@@ -164,7 +164,7 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) · 14 unmodified lines
164	\x79\x86\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf8\xff\xff\xff\xb7\x02\0\	164	\x79\x86\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf8\xff\xff\xff\xb7\x02\0\
165	\0\x08\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x01\0\0\0\0\0\0\x79\xa3\xf8\xff\0\0\0\0\	165	\0\x08\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x01\0\0\0\0\0\0\x79\xa3\xf8\xff\0\0\0\0\
166	\x0f\x13\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf4\xff\xff\xff\xb7\x02\0\	166	\x0f\x13\0\0\0\0\0\0\xbf\xa1\0\0\0\0\0\0\x07\x01\0\0\xf4\xff\xff\xff\xb7\x02\0\
167	\0\x04\0\0\0\x85\0\0\0\x04\0\0\0\xb7\x03\0\0\x04\0\0\0\x61\xa1\xf4\xff\0\0\0\0\	167	\0\x04\0\0\0\x85\0\0\0\x71\0\0\0\xb7\x03\0\0\x04\0\0\0\x61\xa1\xf4\xff\0\0\0\0\
168	\x61\x82\x10\0\0\0\0\0\x3d\x21\x02\0\0\0\0\0\x0f\x16\0\0\0\0\0\0\xbf\x69\0\0\0\	168	\x61\x82\x10\0\0\0\0\0\x3d\x21\x02\0\0\0\0\0\x0f\x16\0\0\0\0\0\0\xbf\x69\0\0\0\
169	\0\0\0\x7b\x9a\xd8\xff\0\0\0\0\x79\x71\x18\0\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\	169	\0\0\0\x7b\x9a\xd8\xff\0\0\0\0\x79\x71\x18\0\0\0\0\0\x7b\x1a\xe0\xff\0\0\0\0\
170	\x79\x71\x20\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\x31\0\0\0\0\0\0\x7b\x1a\xe8\xff\	170	\x79\x71\x20\0\0\0\0\0\x79\x11\0\0\0\0\0\0\x0f\x31\0\0\0\0\0\0\x7b\x1a\xe8\xff\
@@ -176,230 +176,232 @@ iterators_bpf__create_skeleton(struct iterators_bpf *obj) · 5 unmodified lines
176	\x73\x25\x36\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\	176	\x73\x25\x36\x64\x0a\0\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\
177	\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x0a\0\x25\x34\	177	\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x0a\0\x25\x34\
178	\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x0a\0\x47\x50\x4c\0\x9f\	178	\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x0a\0\x47\x50\x4c\0\x9f\
179	\xeb\x01\0\x18\0\0\0\0\0\0\0\x1c\x04\0\0\x1c\x04\0\0\0\x05\0\0\0\0\0\0\0\0\0\	179	\xeb\x01\0\x18\0\0\0\0\0\0\0\x1c\x04\0\0\x1c\x04\0\0\x09\x05\0\0\0\0\0\0\0\0\0\
180	\x02\x02\0\0\0\x01\0\0\0\x02\0\0\x04\x10\0\0\0\x13\0\0\0\x03\0\0\0\0\0\0\0\x18\	180	\x02\x02\0\0\0\x01\0\0\0\x02\0\0\x04\x10\0\0\0\x13\0\0\0\x03\0\0\0\0\0\0\0\x18\
181	\0\0\0\x04\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x02\x0d\0\	181	\0\0\0\x04\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\x08\0\0\0\0\0\0\0\0\0\0\x02\x0d\0\
182	\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x01\0\0\0\x20\0\0\0\0\0\0\x01\x04\	182	\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x01\0\0\0\x20\0\0\0\0\0\0\x01\x04\
183	\0\0\0\x20\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xa3\0\0\0\x03\0\0\x04\x18\0\	183	\0\0\0\x20\0\0\x01\x24\0\0\0\x01\0\0\x0c\x05\0\0\0\xaf\0\0\0\x03\0\0\x04\x18\0\
184	\0\0\xb1\0\0\0\x09\0\0\0\0\0\0\0\xb5\0\0\0\x0b\0\0\0\x40\0\0\0\xc0\0\0\0\x0b\0\	184	\0\0\xbd\0\0\0\x09\0\0\0\0\0\0\0\xc1\0\0\0\x0b\0\0\0\x40\0\0\0\xcc\0\0\0\x0b\0\
185	\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xc8\0\0\0\0\0\0\x07\0\0\0\0\xd1\0\0\	185	\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x0a\0\0\0\xd4\0\0\0\0\0\0\x07\0\0\0\0\xdd\0\0\
186	\0\0\0\0\x08\x0c\0\0\0\xd7\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\x98\x01\0\0\x03\	186	\0\0\0\0\x08\x0c\0\0\0\xe3\0\0\0\0\0\0\x01\x08\0\0\0\x40\0\0\0\xa4\x01\0\0\x03\
187	\0\0\x04\x18\0\0\0\xa0\x01\0\0\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\	187	\0\0\x04\x18\0\0\0\xac\x01\0\0\x0e\0\0\0\0\0\0\0\xaf\x01\0\0\x11\0\0\0\x20\0\0\
188	\0\xa8\x01\0\0\x0e\0\0\0\xa0\0\0\0\xb4\x01\0\0\0\0\0\x08\x0f\0\0\0\xba\x01\0\0\	188	\0\xb4\x01\0\0\x0e\0\0\0\xa0\0\0\0\xc0\x01\0\0\0\0\0\x08\x0f\0\0\0\xc6\x01\0\0\
189	\0\0\0\x01\x04\0\0\0\x20\0\0\0\xc7\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\	189	\0\0\0\x01\x04\0\0\0\x20\0\0\0\xd3\x01\0\0\0\0\0\x01\x01\0\0\0\x08\0\0\x01\0\0\
190	\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xcc\x01\0\0\0\0\0\x01\x04\	190	\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x10\0\0\0\xd8\x01\0\0\0\0\0\x01\x04\
191	\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\x30\x02\0\0\x02\0\0\x04\x10\0\0\0\	191	\0\0\0\x20\0\0\0\0\0\0\0\0\0\0\x02\x14\0\0\0\x3c\x02\0\0\x02\0\0\x04\x10\0\0\0\
192	\x13\0\0\0\x03\0\0\0\0\0\0\0\x43\x02\0\0\x15\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\	192	\x13\0\0\0\x03\0\0\0\0\0\0\0\x4f\x02\0\0\x15\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\x02\
193	\x18\0\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x13\0\0\0\x48\x02\0\0\x01\0\	193	\x18\0\0\0\0\0\0\0\x01\0\0\x0d\x06\0\0\0\x1c\0\0\0\x13\0\0\0\x54\x02\0\0\x01\0\
194	\0\x0c\x16\0\0\0\x94\x02\0\0\x01\0\0\x04\x08\0\0\0\x9d\x02\0\0\x19\0\0\0\0\0\0\	194	\0\x0c\x16\0\0\0\xa0\x02\0\0\x01\0\0\x04\x08\0\0\0\xa9\x02\0\0\x19\0\0\0\0\0\0\
195	\0\0\0\0\0\0\0\0\x02\x1a\0\0\0\xee\x02\0\0\x06\0\0\x04\x38\0\0\0\xa0\x01\0\0\	195	\0\0\0\0\0\0\0\0\x02\x1a\0\0\0\xfa\x02\0\0\x06\0\0\x04\x38\0\0\0\xac\x01\0\0\
196	\x0e\0\0\0\0\0\0\0\xa3\x01\0\0\x11\0\0\0\x20\0\0\0\xfb\x02\0\0\x1b\0\0\0\xc0\0\	196	\x0e\0\0\0\0\0\0\0\xaf\x01\0\0\x11\0\0\0\x20\0\0\0\x07\x03\0\0\x1b\0\0\0\xc0\0\
197	\0\0\x0c\x03\0\0\x15\0\0\0\0\x01\0\0\x18\x03\0\0\x1d\0\0\0\x40\x01\0\0\x22\x03\	197	\0\0\x18\x03\0\0\x15\0\0\0\0\x01\0\0\x21\x03\0\0\x1d\0\0\0\x40\x01\0\0\x2b\x03\
198	\0\0\x1e\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x02\x1c\0\0\0\0\0\0\0\0\0\0\x0a\x10\0\	198	\0\0\x1e\0\0\0\x80\x01\0\0\0\0\0\0\0\0\0\x02\x1c\0\0\0\0\0\0\0\0\0\0\x0a\x10\0\
199	\0\0\0\0\0\0\0\0\0\x02\x1f\0\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\x6c\x03\0\0\x02\0\	199	\0\0\0\0\0\0\0\0\0\x02\x1f\0\0\0\0\0\0\0\0\0\0\x02\x20\0\0\0\x75\x03\0\0\x02\0\
200	\0\x04\x08\0\0\0\x7a\x03\0\0\x0e\0\0\0\0\0\0\0\x83\x03\0\0\x0e\0\0\0\x20\0\0\0\	200	\0\x04\x08\0\0\0\x83\x03\0\0\x0e\0\0\0\0\0\0\0\x8c\x03\0\0\x0e\0\0\0\x20\0\0\0\
201	\x22\x03\0\0\x03\0\0\x04\x18\0\0\0\x8d\x03\0\0\x1b\0\0\0\0\0\0\0\x95\x03\0\0\	201	\x2b\x03\0\0\x03\0\0\x04\x18\0\0\0\x96\x03\0\0\x1b\0\0\0\0\0\0\0\x9e\x03\0\0\
202	\x21\0\0\0\x40\0\0\0\x9b\x03\0\0\x23\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x22\0\0\	202	\x21\0\0\0\x40\0\0\0\xa4\x03\0\0\x23\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\x02\x22\0\0\
203	\0\0\0\0\0\0\0\0\x02\x24\0\0\0\x9f\x03\0\0\x01\0\0\x04\x04\0\0\0\xaa\x03\0\0\	203	\0\0\0\0\0\0\0\0\x02\x24\0\0\0\xa8\x03\0\0\x01\0\0\x04\x04\0\0\0\xb3\x03\0\0\
204	\x0e\0\0\0\0\0\0\0\x13\x04\0\0\x01\0\0\x04\x04\0\0\0\x1c\x04\0\0\x0e\0\0\0\0\0\	204	\x0e\0\0\0\0\0\0\0\x1c\x04\0\0\x01\0\0\x04\x04\0\0\0\x25\x04\0\0\x0e\0\0\0\0\0\
205	\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\0\x92\x04\0\0\0\0\0\	205	\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x23\0\0\0\x9b\x04\0\0\0\0\0\
206	\x0e\x25\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x0e\0\0\0\	206	\x0e\x25\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\x12\0\0\0\x0e\0\0\0\
207	\xa6\x04\0\0\0\0\0\x0e\x27\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\	207	\xaf\x04\0\0\0\0\0\x0e\x27\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\0\0\0\x1c\0\0\0\
208	\x12\0\0\0\x20\0\0\0\xbc\x04\0\0\0\0\0\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\	208	\x12\0\0\0\x20\0\0\0\xc5\x04\0\0\0\0\0\x0e\x29\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\
209	\0\0\0\0\x1c\0\0\0\x12\0\0\0\x11\0\0\0\xd1\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\	209	\0\0\0\0\x1c\0\0\0\x12\0\0\0\x11\0\0\0\xda\x04\0\0\0\0\0\x0e\x2b\0\0\0\0\0\0\0\
210	\0\0\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\xe8\x04\0\0\0\0\0\x0e\	210	\0\0\0\0\0\0\0\x03\0\0\0\0\x10\0\0\0\x12\0\0\0\x04\0\0\0\xf1\x04\0\0\0\0\0\x0e\
211	\x2d\0\0\0\x01\0\0\0\xf0\x04\0\0\x04\0\0\x0f\0\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\	211	\x2d\0\0\0\x01\0\0\0\xf9\x04\0\0\x04\0\0\x0f\0\0\0\0\x26\0\0\0\0\0\0\0\x23\0\0\
212	\0\x28\0\0\0\x23\0\0\0\x0e\0\0\0\x2a\0\0\0\x31\0\0\0\x20\0\0\0\x2c\0\0\0\x51\0\	212	\0\x28\0\0\0\x23\0\0\0\x0e\0\0\0\x2a\0\0\0\x31\0\0\0\x20\0\0\0\x2c\0\0\0\x51\0\
213	\0\0\x11\0\0\0\xf8\x04\0\0\x01\0\0\x0f\0\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\0\0\0\	213	\0\0\x11\0\0\0\x01\x05\0\0\x01\0\0\x0f\0\0\0\0\x2e\0\0\0\0\0\0\0\x04\0\0\0\0\
214	\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x6d\x65\	214	\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x6d\x65\
215	\x74\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\x64\x75\x6d\x70\x5f\x62\	215	\x74\x61\0\x6d\x61\x70\0\x63\x74\x78\0\x69\x6e\x74\0\x64\x75\x6d\x70\x5f\x62\
216	\x70\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\	216	\x70\x66\x5f\x6d\x61\x70\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\
217	\x30\x3a\x30\0\x2f\x77\x2f\x6e\x65\x74\x2d\x6e\x65\x78\x74\x2f\x6b\x65\x72\x6e\	217	\x30\x3a\x30\0\x2f\x68\x6f\x6d\x65\x2f\x61\x6c\x72\x75\x61\x2f\x62\x75\x69\x6c\
218	\x65\x6c\x2f\x62\x70\x66\x2f\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\	218	\x64\x2f\x6c\x69\x6e\x75\x78\x2f\x6b\x65\x72\x6e\x65\x6c\x2f\x62\x70\x66\x2f\
219	\x61\x74\x6f\x72\x73\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\	219	\x70\x72\x65\x6c\x6f\x61\x64\x2f\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2f\x69\
220	\x2e\x63\0\x09\x73\x74\x72\x75\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\	220	\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\x2e\x63\0\x09\x73\x74\x72\x75\
221	\x2a\x73\x65\x71\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\	221	\x63\x74\x20\x73\x65\x71\x5f\x66\x69\x6c\x65\x20\x2a\x73\x65\x71\x20\x3d\x20\
222	\x65\x71\x3b\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\	222	\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\x71\x3b\0\x62\x70\x66\x5f\
223	\x71\0\x73\x65\x73\x73\x69\x6f\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\	223	\x69\x74\x65\x72\x5f\x6d\x65\x74\x61\0\x73\x65\x71\0\x73\x65\x73\x73\x69\x6f\
224	\x73\x65\x71\x5f\x66\x69\x6c\x65\0\x5f\x5f\x75\x36\x34\0\x6c\x6f\x6e\x67\x20\	224	\x6e\x5f\x69\x64\0\x73\x65\x71\x5f\x6e\x75\x6d\0\x73\x65\x71\x5f\x66\x69\x6c\
225	\x6c\x6f\x6e\x67\x20\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x30\x3a\	225	\x65\0\x5f\x5f\x75\x36\x34\0\x6c\x6f\x6e\x67\x20\x6c\x6f\x6e\x67\x20\x75\x6e\
226	\x31\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\	226	\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x30\x3a\x31\0\x09\x73\x74\x72\x75\
227	\x61\x70\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\	227	\x63\x74\x20\x62\x70\x66\x5f\x6d\x61\x70\x20\x2a\x6d\x61\x70\x20\x3d\x20\x63\
228	\x21\x6d\x61\x70\x29\0\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\	228	\x74\x78\x2d\x3e\x6d\x61\x70\x3b\0\x09\x69\x66\x20\x28\x21\x6d\x61\x70\x29\0\
229	\x5f\x6e\x75\x6d\x20\x3d\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\	229	\x30\x3a\x32\0\x09\x5f\x5f\x75\x36\x34\x20\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\
230	\x65\x71\x5f\x6e\x75\x6d\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\	230	\x20\x63\x74\x78\x2d\x3e\x6d\x65\x74\x61\x2d\x3e\x73\x65\x71\x5f\x6e\x75\x6d\
231	\x20\x3d\x3d\x20\x30\x29\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\	231	\x3b\0\x09\x69\x66\x20\x28\x73\x65\x71\x5f\x6e\x75\x6d\x20\x3d\x3d\x20\x30\x29\
232	\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\	232	\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\
233	\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\	233	\x71\x2c\x20\x22\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\
234	\x74\x72\x69\x65\x73\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\	234	\x20\x20\x20\x20\x20\x20\x20\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x5c\
235	\x64\0\x6e\x61\x6d\x65\0\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\	235	\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x6d\x61\x70\0\x69\x64\0\x6e\x61\x6d\x65\0\
236	\x75\x33\x32\0\x75\x6e\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\	236	\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\0\x5f\x5f\x75\x33\x32\0\x75\x6e\
237	\x72\0\x5f\x5f\x41\x52\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\	237	\x73\x69\x67\x6e\x65\x64\x20\x69\x6e\x74\0\x63\x68\x61\x72\0\x5f\x5f\x41\x52\
238	\x5f\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\	238	\x52\x41\x59\x5f\x53\x49\x5a\x45\x5f\x54\x59\x50\x45\x5f\x5f\0\x09\x42\x50\x46\
239	\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x5c\x6e\x22\	239	\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\
240	\x2c\x20\x6d\x61\x70\x2d\x3e\x69\x64\x2c\x20\x6d\x61\x70\x2d\x3e\x6e\x61\x6d\	240	\x34\x75\x20\x25\x2d\x31\x36\x73\x25\x36\x64\x5c\x6e\x22\x2c\x20\x6d\x61\x70\
241	\x65\x2c\x20\x6d\x61\x70\x2d\x3e\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\	241	\x2d\x3e\x69\x64\x2c\x20\x6d\x61\x70\x2d\x3e\x6e\x61\x6d\x65\x2c\x20\x6d\x61\
242	\x29\x3b\0\x7d\0\x62\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x70\	242	\x70\x2d\x3e\x6d\x61\x78\x5f\x65\x6e\x74\x72\x69\x65\x73\x29\x3b\0\x7d\0\x62\
243	\x72\x6f\x67\0\x70\x72\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\	243	\x70\x66\x5f\x69\x74\x65\x72\x5f\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x70\x72\
244	\x6f\x67\0\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\x74\	244	\x6f\x67\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x69\x74\x65\
245	\x72\x75\x63\x74\x20\x62\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\x67\	245	\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x09\x73\x74\x72\x75\x63\x74\x20\x62\
246	\x20\x3d\x20\x63\x74\x78\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\x21\	246	\x70\x66\x5f\x70\x72\x6f\x67\x20\x2a\x70\x72\x6f\x67\x20\x3d\x20\x63\x74\x78\
247	\x70\x72\x6f\x67\x29\0\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\x61\	247	\x2d\x3e\x70\x72\x6f\x67\x3b\0\x09\x69\x66\x20\x28\x21\x70\x72\x6f\x67\x29\0\
248	\x75\x78\x20\x3d\x20\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\x50\	248	\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x61\x75\x78\0\x09\x61\x75\x78\x20\x3d\x20\
249	\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\	249	\x70\x72\x6f\x67\x2d\x3e\x61\x75\x78\x3b\0\x09\x09\x42\x50\x46\x5f\x53\x45\x51\
250	\x20\x20\x69\x64\x20\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\	250	\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x20\x20\x69\x64\x20\
251	\x20\x20\x20\x61\x74\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\	251	\x6e\x61\x6d\x65\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20\x61\x74\
252	\x5f\x70\x72\x6f\x67\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\x6e\	252	\x74\x61\x63\x68\x65\x64\x5c\x6e\x22\x29\x3b\0\x62\x70\x66\x5f\x70\x72\x6f\x67\
253	\x63\x5f\x6e\x61\x6d\x65\0\x6c\x69\x6e\x6b\x65\x64\x5f\x70\x72\x6f\x67\0\x66\	253	\x5f\x61\x75\x78\0\x61\x74\x74\x61\x63\x68\x5f\x66\x75\x6e\x63\x5f\x6e\x61\x6d\
254	\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\	254	\x65\0\x64\x73\x74\x5f\x70\x72\x6f\x67\0\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\
255	\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\	255	\x62\x74\x66\0\x09\x42\x50\x46\x5f\x53\x45\x51\x5f\x50\x52\x49\x4e\x54\x46\x28\
256	\x25\x2d\x31\x36\x73\x20\x25\x73\x20\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\	256	\x73\x65\x71\x2c\x20\x22\x25\x34\x75\x20\x25\x2d\x31\x36\x73\x20\x25\x73\x20\
257	\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\	257	\x25\x73\x5c\x6e\x22\x2c\x20\x61\x75\x78\x2d\x3e\x69\x64\x2c\0\x30\x3a\x34\0\
258	\x74\x66\x29\0\x62\x70\x66\x5f\x66\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\	258	\x30\x3a\x35\0\x09\x69\x66\x20\x28\x21\x62\x74\x66\x29\0\x62\x70\x66\x5f\x66\
259	\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\	259	\x75\x6e\x63\x5f\x69\x6e\x66\x6f\0\x69\x6e\x73\x6e\x5f\x6f\x66\x66\0\x74\x79\
260	\x6e\x67\x73\0\x74\x79\x70\x65\x73\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\	260	\x70\x65\x5f\x69\x64\0\x30\0\x73\x74\x72\x69\x6e\x67\x73\0\x74\x79\x70\x65\x73\
261	\x64\x65\x72\0\x73\x74\x72\x5f\x6c\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\	261	\0\x68\x64\x72\0\x62\x74\x66\x5f\x68\x65\x61\x64\x65\x72\0\x73\x74\x72\x5f\x6c\
262	\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\	262	\x65\x6e\0\x09\x74\x79\x70\x65\x73\x20\x3d\x20\x62\x74\x66\x2d\x3e\x74\x79\x70\
263	\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\	263	\x65\x73\x3b\0\x09\x62\x70\x66\x5f\x70\x72\x6f\x62\x65\x5f\x72\x65\x61\x64\x5f\
264	\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\	264	\x6b\x65\x72\x6e\x65\x6c\x28\x26\x74\x2c\x20\x73\x69\x7a\x65\x6f\x66\x28\x74\
265	\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\	265	\x29\x2c\x20\x74\x79\x70\x65\x73\x20\x2b\x20\x62\x74\x66\x5f\x69\x64\x29\x3b\0\
266	\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\	266	\x09\x73\x74\x72\x20\x3d\x20\x62\x74\x66\x2d\x3e\x73\x74\x72\x69\x6e\x67\x73\
267	\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\	267	\x3b\0\x62\x74\x66\x5f\x74\x79\x70\x65\0\x6e\x61\x6d\x65\x5f\x6f\x66\x66\0\x09\
268	\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\	268	\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3d\x20\x42\x50\x46\x5f\x43\x4f\x52\x45\
269	\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\	269	\x5f\x52\x45\x41\x44\x28\x74\x2c\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x29\x3b\0\
270	\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\	270	\x30\x3a\x32\x3a\x30\0\x09\x69\x66\x20\x28\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x20\
271	\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\	271	\x3e\x3d\x20\x62\x74\x66\x2d\x3e\x68\x64\x72\x2e\x73\x74\x72\x5f\x6c\x65\x6e\
272	\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\	272	\x29\0\x09\x72\x65\x74\x75\x72\x6e\x20\x73\x74\x72\x20\x2b\x20\x6e\x61\x6d\x65\
273	\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\	273	\x5f\x6f\x66\x66\x3b\0\x30\x3a\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\
274	\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\	274	\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\
275	\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\	275	\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\x64\x75\x6d\x70\x5f\x62\x70\x66\
276	\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\	276	\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\
277	\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\x4e\x53\x45\0\x2e\x72\x6f\x64\x61\	277	\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x4c\x49\x43\x45\
278	\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x9f\xeb\x01\0\x20\0\0\0\0\0\0\0\x24\0\	278	\x4e\x53\x45\0\x2e\x72\x6f\x64\x61\x74\x61\0\x6c\x69\x63\x65\x6e\x73\x65\0\x9f\
279	\0\0\x24\0\0\0\x44\x02\0\0\x68\x02\0\0\xa4\x01\0\0\x08\0\0\0\x31\0\0\0\x01\0\0\	279	\xeb\x01\0\x20\0\0\0\0\0\0\0\x24\0\0\0\x24\0\0\0\x44\x02\0\0\x68\x02\0\0\xa4\
280	\0\0\0\0\0\x07\0\0\0\x56\x02\0\0\x01\0\0\0\0\0\0\0\x17\0\0\0\x10\0\0\0\x31\0\0\	280	\x01\0\0\x08\0\0\0\x31\0\0\0\x01\0\0\0\0\0\0\0\x07\0\0\0\x62\x02\0\0\x01\0\0\0\
281	\0\x09\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\x1e\x40\x01\0\x08\0\0\0\x42\0\0\0\x7b\	281	\0\0\0\0\x17\0\0\0\x10\0\0\0\x31\0\0\0\x09\0\0\0\0\0\0\0\x42\0\0\0\x87\0\0\0\
282	\0\0\0\x24\x40\x01\0\x10\0\0\0\x42\0\0\0\xf2\0\0\0\x1d\x48\x01\0\x18\0\0\0\x42\	282	\x1e\x40\x01\0\x08\0\0\0\x42\0\0\0\x87\0\0\0\x24\x40\x01\0\x10\0\0\0\x42\0\0\0\
283	\0\0\0\x13\x01\0\0\x06\x50\x01\0\x20\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\x44\x01\0\	283	\xfe\0\0\0\x1d\x48\x01\0\x18\0\0\0\x42\0\0\0\x1f\x01\0\0\x06\x50\x01\0\x20\0\0\
284	\x28\0\0\0\x42\0\0\0\x47\x01\0\0\x06\x5c\x01\0\x38\0\0\0\x42\0\0\0\x5a\x01\0\0\	284	\0\x42\0\0\0\x2e\x01\0\0\x1d\x44\x01\0\x28\0\0\0\x42\0\0\0\x53\x01\0\0\x06\x5c\
285	\x03\x60\x01\0\x70\0\0\0\x42\0\0\0\xe0\x01\0\0\x02\x68\x01\0\xf0\0\0\0\x42\0\0\	285	\x01\0\x38\0\0\0\x42\0\0\0\x66\x01\0\0\x03\x60\x01\0\x70\0\0\0\x42\0\0\0\xec\
286	\0\x2e\x02\0\0\x01\x70\x01\0\x56\x02\0\0\x1a\0\0\0\0\0\0\0\x42\0\0\0\x7b\0\0\0\	286	\x01\0\0\x02\x68\x01\0\xf0\0\0\0\x42\0\0\0\x3a\x02\0\0\x01\x70\x01\0\x62\x02\0\
287	\x1e\x84\x01\0\x08\0\0\0\x42\0\0\0\x7b\0\0\0\x24\x84\x01\0\x10\0\0\0\x42\0\0\0\	287	\0\x1a\0\0\0\0\0\0\0\x42\0\0\0\x87\0\0\0\x1e\x84\x01\0\x08\0\0\0\x42\0\0\0\x87\
288	\x64\x02\0\0\x1f\x8c\x01\0\x18\0\0\0\x42\0\0\0\x88\x02\0\0\x06\x98\x01\0\x20\0\	288	\0\0\0\x24\x84\x01\0\x10\0\0\0\x42\0\0\0\x70\x02\0\0\x1f\x8c\x01\0\x18\0\0\0\
289	\0\0\x42\0\0\0\xa1\x02\0\0\x0e\xa4\x01\0\x28\0\0\0\x42\0\0\0\x22\x01\0\0\x1d\	289	\x42\0\0\0\x94\x02\0\0\x06\x98\x01\0\x20\0\0\0\x42\0\0\0\xad\x02\0\0\x0e\xa4\
290	\x88\x01\0\x30\0\0\0\x42\0\0\0\x47\x01\0\0\x06\xa8\x01\0\x40\0\0\0\x42\0\0\0\	290	\x01\0\x28\0\0\0\x42\0\0\0\x2e\x01\0\0\x1d\x88\x01\0\x30\0\0\0\x42\0\0\0\x53\
291	\xb3\x02\0\0\x03\xac\x01\0\x80\0\0\0\x42\0\0\0\x26\x03\0\0\x02\xb4\x01\0\xb8\0\	291	\x01\0\0\x06\xa8\x01\0\x40\0\0\0\x42\0\0\0\xbf\x02\0\0\x03\xac\x01\0\x80\0\0\0\
292	\0\0\x42\0\0\0\x61\x03\0\0\x06\x08\x01\0\xd0\0\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\	292	\x42\0\0\0\x2f\x03\0\0\x02\xb4\x01\0\xb8\0\0\0\x42\0\0\0\x6a\x03\0\0\x06\x08\
293	\xd8\0\0\0\x42\0\0\0\xb2\x03\0\0\x0f\x14\x01\0\xe0\0\0\0\x42\0\0\0\xc7\x03\0\0\	293	\x01\0\xd0\0\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\xd8\0\0\0\x42\0\0\0\xbb\x03\0\0\x0f\
294	\x2d\x18\x01\0\xf0\0\0\0\x42\0\0\0\xfe\x03\0\0\x0d\x10\x01\0\0\x01\0\0\x42\0\0\	294	\x14\x01\0\xe0\0\0\0\x42\0\0\0\xd0\x03\0\0\x2d\x18\x01\0\xf0\0\0\0\x42\0\0\0\
295	\0\0\0\0\0\0\0\0\0\x08\x01\0\0\x42\0\0\0\xc7\x03\0\0\x02\x18\x01\0\x20\x01\0\0\	295	\x07\x04\0\0\x0d\x10\x01\0\0\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x08\x01\0\0\x42\
296	\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x38\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x40\	296	\0\0\0\xd0\x03\0\0\x02\x18\x01\0\x20\x01\0\0\x42\0\0\0\x2e\x04\0\0\x0d\x1c\x01\
297	\x01\0\0\x42\0\0\0\x25\x04\0\0\x0d\x1c\x01\0\x58\x01\0\0\x42\0\0\0\x25\x04\0\0\	297	\0\x38\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\x42\0\0\0\x2e\x04\0\0\x0d\
298	\x0d\x1c\x01\0\x60\x01\0\0\x42\0\0\0\x53\x04\0\0\x1b\x20\x01\0\x68\x01\0\0\x42\	298	\x1c\x01\0\x58\x01\0\0\x42\0\0\0\x2e\x04\0\0\x0d\x1c\x01\0\x60\x01\0\0\x42\0\0\
299	\0\0\0\x53\x04\0\0\x06\x20\x01\0\x70\x01\0\0\x42\0\0\0\x76\x04\0\0\x0d\x28\x01\	299	\0\x5c\x04\0\0\x1b\x20\x01\0\x68\x01\0\0\x42\0\0\0\x5c\x04\0\0\x06\x20\x01\0\
300	\0\x78\x01\0\0\x42\0\0\0\0\0\0\0\0\0\0\0\x80\x01\0\0\x42\0\0\0\x26\x03\0\0\x02\	300	\x70\x01\0\0\x42\0\0\0\x7f\x04\0\0\x0d\x28\x01\0\x78\x01\0\0\x42\0\0\0\0\0\0\0\
301	\xb4\x01\0\xf8\x01\0\0\x42\0\0\0\x2e\x02\0\0\x01\xc4\x01\0\x10\0\0\0\x31\0\0\0\	301	\0\0\0\0\x80\x01\0\0\x42\0\0\0\x2f\x03\0\0\x02\xb4\x01\0\xf8\x01\0\0\x42\0\0\0\
302	\x07\0\0\0\0\0\0\0\x02\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\	302	\x3a\x02\0\0\x01\xc4\x01\0\x10\0\0\0\x31\0\0\0\x07\0\0\0\0\0\0\0\x02\0\0\0\x3e\
303	\0\0\0\x10\0\0\0\x02\0\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x08\0\0\0\x1e\x01\0\0\0\	303	\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x02\0\0\0\xfa\0\
304	\0\0\0\x70\0\0\0\x0d\0\0\0\x3e\0\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\0\xee\0\0\0\0\0\	304	\0\0\0\0\0\0\x20\0\0\0\x08\0\0\0\x2a\x01\0\0\0\0\0\0\x70\0\0\0\x0d\0\0\0\x3e\0\
305	\0\0\xa0\0\0\0\x0d\0\0\0\x1e\x01\0\0\0\0\0\0\x56\x02\0\0\x12\0\0\0\0\0\0\0\x14\	305	\0\0\0\0\0\0\x80\0\0\0\x0d\0\0\0\xfa\0\0\0\0\0\0\0\xa0\0\0\0\x0d\0\0\0\x2a\x01\
306	\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x14\0\	306	\0\0\0\0\0\0\x62\x02\0\0\x12\0\0\0\0\0\0\0\x14\0\0\0\x3e\0\0\0\0\0\0\0\x08\0\0\
307	\0\0\xee\0\0\0\0\0\0\0\x20\0\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\	307	\0\x08\0\0\0\x3e\0\0\0\0\0\0\0\x10\0\0\0\x14\0\0\0\xfa\0\0\0\0\0\0\0\x20\0\0\0\
308	\0\x1e\x01\0\0\0\0\0\0\x80\0\0\0\x1a\0\0\0\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1a\0\0\	308	\x18\0\0\0\x3e\0\0\0\0\0\0\0\x28\0\0\0\x08\0\0\0\x2a\x01\0\0\0\0\0\0\x80\0\0\0\
309	\0\xee\0\0\0\0\0\0\0\xa8\0\0\0\x1a\0\0\0\x59\x03\0\0\0\0\0\0\xb0\0\0\0\x1a\0\0\	309	\x1a\0\0\0\x3e\0\0\0\0\0\0\0\x90\0\0\0\x1a\0\0\0\xfa\0\0\0\0\0\0\0\xa8\0\0\0\
310	\0\x5d\x03\0\0\0\0\0\0\xc0\0\0\0\x1f\0\0\0\x8b\x03\0\0\0\0\0\0\xd8\0\0\0\x20\0\	310	\x1a\0\0\0\x62\x03\0\0\0\0\0\0\xb0\0\0\0\x1a\0\0\0\x66\x03\0\0\0\0\0\0\xc0\0\0\
311	\0\0\xee\0\0\0\0\0\0\0\xf0\0\0\0\x20\0\0\0\x3e\0\0\0\0\0\0\0\x18\x01\0\0\x24\0\	311	\0\x1f\0\0\0\x94\x03\0\0\0\0\0\0\xd8\0\0\0\x20\0\0\0\xfa\0\0\0\0\0\0\0\xf0\0\0\
312	\0\0\x3e\0\0\0\0\0\0\0\x50\x01\0\0\x1a\0\0\0\xee\0\0\0\0\0\0\0\x60\x01\0\0\x20\	312	\0\x20\0\0\0\x3e\0\0\0\0\0\0\0\x18\x01\0\0\x24\0\0\0\x3e\0\0\0\0\0\0\0\x50\x01\
313	\0\0\0\x4d\x04\0\0\0\0\0\0\x88\x01\0\0\x1a\0\0\0\x1e\x01\0\0\0\0\0\0\x98\x01\0\	313	\0\0\x1a\0\0\0\xfa\0\0\0\0\0\0\0\x60\x01\0\0\x20\0\0\0\x56\x04\0\0\0\0\0\0\x88\
314	\0\x1a\0\0\0\x8e\x04\0\0\0\0\0\0\xa0\x01\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\0\0\0\	314	\x01\0\0\x1a\0\0\0\x2a\x01\0\0\0\0\0\0\x98\x01\0\0\x1a\0\0\0\x97\x04\0\0\0\0\0\
315	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd6\0\0\0\0\0\x02\0\x70\0\0\0\0\	315	\0\xa0\x01\0\0\x18\0\0\0\x3e\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
316	\0\0\0\0\0\0\0\0\0\0\0\xc8\0\0\0\0\0\x02\0\xf0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\	316	\0\0\0\0\0\0\0\x91\0\0\0\x04\0\xf1\xff\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xe6\0\0\
317	\xcf\0\0\0\0\0\x03\0\x78\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc1\0\0\0\0\0\x03\0\x80\	317	\0\0\0\x02\0\x70\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd8\0\0\0\0\0\x02\0\xf0\0\0\0\0\
318	\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xba\0\0\0\0\0\x03\0\xf8\x01\0\0\0\0\0\0\0\0\0\	318	\0\0\0\0\0\0\0\0\0\0\0\xdf\0\0\0\0\0\x03\0\x78\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
319	\0\0\0\0\0\x14\0\0\0\x01\0\x04\0\0\0\0\0\0\0\0\0\x23\0\0\0\0\0\0\0\xf4\0\0\0\	319	\xd1\0\0\0\0\0\x03\0\x80\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xca\0\0\0\0\0\x03\0\
320	\x01\0\x04\0\x23\0\0\0\0\0\0\0\x0e\0\0\0\0\0\0\0\x28\0\0\0\x01\0\x04\0\x31\0\0\	320	\xf8\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x14\0\0\0\x01\0\x04\0\0\0\0\0\0\0\0\0\x23\
321	\0\0\0\0\0\x20\0\0\0\0\0\0\0\xdd\0\0\0\x01\0\x04\0\x51\0\0\0\0\0\0\0\x11\0\0\0\	321	\0\0\0\0\0\0\0\x04\x01\0\0\x01\0\x04\0\x23\0\0\0\0\0\0\0\x0e\0\0\0\0\0\0\0\x28\
322	\0\0\0\0\0\0\0\0\x03\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x03\	322	\0\0\0\x01\0\x04\0\x31\0\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\xed\0\0\0\x01\0\x04\0\
323	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\	323	\x51\0\0\0\0\0\0\0\x11\0\0\0\0\0\0\0\0\0\0\0\x03\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\
324	\0\0\0\0\xb2\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\x3d\0\0\0\x12\	324	\0\0\0\0\0\0\0\0\0\x03\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x03\0\
325	\0\x02\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\x5b\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\	325	\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xc2\0\0\0\x11\0\x05\0\0\0\0\0\0\0\0\0\
326	\0\x08\x02\0\0\0\0\0\0\x48\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xc8\0\0\0\0\0\0\0\	326	\x04\0\0\0\0\0\0\0\x3d\0\0\0\x12\0\x02\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\x5b\
327	\x01\0\0\0\x0c\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\0\x0c\0\0\0\xd0\x01\0\0\0\0\0\0\	327	\0\0\0\x12\0\x03\0\0\0\0\0\0\0\0\0\x08\x02\0\0\0\0\0\0\x48\0\0\0\0\0\0\0\x01\0\
328	\x01\0\0\0\x0c\0\0\0\xf0\x03\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\xfc\x03\0\0\0\0\0\	328	\0\0\x0d\0\0\0\xc8\0\0\0\0\0\0\0\x01\0\0\0\x0d\0\0\0\x50\0\0\0\0\0\0\0\x01\0\0\
329	\0\x0a\0\0\0\x0c\0\0\0\x08\x04\0\0\0\0\0\0\x0a\0\0\0\x0c\0\0\0\x14\x04\0\0\0\0\	329	\0\x0d\0\0\0\xd0\x01\0\0\0\0\0\0\x01\0\0\0\x0d\0\0\0\xf0\x03\0\0\0\0\0\0\x0a\0\
330	\0\0\x0a\0\0\0\x0c\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\x0d\0\0\0\x2c\0\0\0\0\0\0\	330	\0\0\x0d\0\0\0\xfc\x03\0\0\0\0\0\0\x0a\0\0\0\x0d\0\0\0\x08\x04\0\0\0\0\0\0\x0a\
331	\0\0\0\0\0\x0a\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x50\0\0\0\0\0\0\0\0\0\	331	\0\0\0\x0d\0\0\0\x14\x04\0\0\0\0\0\0\x0a\0\0\0\x0d\0\0\0\x2c\x04\0\0\0\0\0\0\0\
332	\0\0\x0a\0\0\0\x60\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x70\0\0\0\0\0\0\0\0\0\0\0\	332	\0\0\0\x0e\0\0\0\x2c\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x3c\0\0\0\0\0\0\0\0\0\0\0\
333	\x0a\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\x90\0\0\0\0\0\0\0\0\0\0\0\x0a\0\	333	\x0c\0\0\0\x50\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x60\0\0\0\0\0\0\0\0\0\0\0\x0b\0\
334	\0\0\xa0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\	334	\0\0\x70\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\
335	\xc0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd0\0\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xe8\0\	335	\x90\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xb0\0\
336	\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xf8\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x01\0\0\	336	\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xd0\0\0\0\0\
337	\0\0\0\0\0\0\0\0\x0b\0\0\0\x18\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x01\0\0\0\	337	\0\0\0\0\0\0\0\x0b\0\0\0\xe8\0\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xf8\0\0\0\0\0\0\0\
338	\0\0\0\0\0\0\0\x0b\0\0\0\x38\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x01\0\0\0\0\	338	\0\0\0\0\x0c\0\0\0\x08\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x18\x01\0\0\0\0\0\0\0\
339	\0\0\0\0\0\0\x0b\0\0\0\x58\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x01\0\0\0\0\0\	339	\0\0\0\x0c\0\0\0\x28\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x38\x01\0\0\0\0\0\0\0\0\
340	\0\0\0\0\0\x0b\0\0\0\x78\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x88\x01\0\0\0\0\0\0\	340	\0\0\x0c\0\0\0\x48\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x58\x01\0\0\0\0\0\0\0\0\0\
341	\0\0\0\0\x0b\0\0\0\x98\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa8\x01\0\0\0\0\0\0\0\	341	\0\x0c\0\0\0\x68\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x78\x01\0\0\0\0\0\0\0\0\0\0\
342	\0\0\0\x0b\0\0\0\xb8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc8\x01\0\0\0\0\0\0\0\0\	342	\x0c\0\0\0\x88\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x98\x01\0\0\0\0\0\0\0\0\0\0\
343	\0\0\x0b\0\0\0\xd8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xe8\x01\0\0\0\0\0\0\0\0\0\	343	\x0c\0\0\0\xa8\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xb8\x01\0\0\0\0\0\0\0\0\0\0\
344	\0\x0b\0\0\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\	344	\x0c\0\0\0\xc8\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xd8\x01\0\0\0\0\0\0\0\0\0\0\
345	\x0b\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x28\x02\0\0\0\0\0\0\0\0\0\0\	345	\x0c\0\0\0\xe8\x01\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xf8\x01\0\0\0\0\0\0\0\0\0\0\
346	\x0b\0\0\0\x38\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x48\x02\0\0\0\0\0\0\0\0\0\0\	346	\x0c\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x18\x02\0\0\0\0\0\0\0\0\0\0\
347	\x0b\0\0\0\x58\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x68\x02\0\0\0\0\0\0\0\0\0\0\	347	\x0c\0\0\0\x28\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x38\x02\0\0\0\0\0\0\0\0\0\0\
348	\x0b\0\0\0\x78\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x94\x02\0\0\0\0\0\0\0\0\0\0\	348	\x0c\0\0\0\x48\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x58\x02\0\0\0\0\0\0\0\0\0\0\
349	\x0a\0\0\0\xa4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xb4\x02\0\0\0\0\0\0\0\0\0\0\	349	\x0c\0\0\0\x68\x02\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x78\x02\0\0\0\0\0\0\0\0\0\0\
350	\x0a\0\0\0\xc4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xd4\x02\0\0\0\0\0\0\0\0\0\0\	350	\x0c\0\0\0\x94\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xa4\x02\0\0\0\0\0\0\0\0\0\0\
351	\x0a\0\0\0\xe4\x02\0\0\0\0\0\0\0\0\0\0\x0a\0\0\0\xf4\x02\0\0\0\0\0\0\0\0\0\0\	351	\x0b\0\0\0\xb4\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xc4\x02\0\0\0\0\0\0\0\0\0\0\
352	\x0a\0\0\0\x0c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x03\0\0\0\0\0\0\0\0\0\0\	352	\x0b\0\0\0\xd4\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xe4\x02\0\0\0\0\0\0\0\0\0\0\
353	\x0b\0\0\0\x2c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x3c\x03\0\0\0\0\0\0\0\0\0\0\	353	\x0b\0\0\0\xf4\x02\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x0c\x03\0\0\0\0\0\0\0\0\0\0\
354	\x0b\0\0\0\x4c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x5c\x03\0\0\0\0\0\0\0\0\0\0\	354	\x0c\0\0\0\x1c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x2c\x03\0\0\0\0\0\0\0\0\0\0\
355	\x0b\0\0\0\x6c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x7c\x03\0\0\0\0\0\0\0\0\0\0\	355	\x0c\0\0\0\x3c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x4c\x03\0\0\0\0\0\0\0\0\0\0\
356	\x0b\0\0\0\x8c\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x9c\x03\0\0\0\0\0\0\0\0\0\0\	356	\x0c\0\0\0\x5c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x6c\x03\0\0\0\0\0\0\0\0\0\0\
357	\x0b\0\0\0\xac\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xbc\x03\0\0\0\0\0\0\0\0\0\0\	357	\x0c\0\0\0\x7c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x8c\x03\0\0\0\0\0\0\0\0\0\0\
358	\x0b\0\0\0\xcc\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xdc\x03\0\0\0\0\0\0\0\0\0\0\	358	\x0c\0\0\0\x9c\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xac\x03\0\0\0\0\0\0\0\0\0\0\
359	\x0b\0\0\0\xec\x03\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\xfc\x03\0\0\0\0\0\0\0\0\0\0\	359	\x0c\0\0\0\xbc\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xcc\x03\0\0\0\0\0\0\0\0\0\0\
360	\x0b\0\0\0\x0c\x04\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x1c\x04\0\0\0\0\0\0\0\0\0\0\	360	\x0c\0\0\0\xdc\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\xec\x03\0\0\0\0\0\0\0\0\0\0\
361	\x0b\0\0\0\x4e\x4f\x41\x42\x43\x44\x4d\0\x2e\x74\x65\x78\x74\0\x2e\x72\x65\x6c\	361	\x0c\0\0\0\xfc\x03\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x0c\x04\0\0\0\0\0\0\0\0\0\0\
362	\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\	362	\x0c\0\0\0\x1c\x04\0\0\0\0\0\0\0\0\0\0\x0c\0\0\0\x4d\x4e\x40\x41\x42\x43\x4c\0\
363	\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\	363	\x2e\x74\x65\x78\x74\0\x2e\x72\x65\x6c\x2e\x42\x54\x46\x2e\x65\x78\x74\0\x64\
364	\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\	364	\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\0\x64\
365	\x61\x70\0\x2e\x72\x65\x6c\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\	365	\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\0\
366	\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x72\x65\x6c\x69\x74\	366	\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\0\x2e\x72\x65\x6c\x69\x74\x65\
367	\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\	367	\x72\x2f\x62\x70\x66\x5f\x6d\x61\x70\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\
368	\x64\x72\x73\x69\x67\0\x6c\x69\x63\x65\x6e\x73\x65\0\x2e\x73\x74\x72\x74\x61\	368	\x72\x6f\x67\0\x2e\x72\x65\x6c\x69\x74\x65\x72\x2f\x62\x70\x66\x5f\x70\x72\x6f\
369	\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x6f\x64\x61\x74\x61\0\x2e\x72\x65\	369	\x67\0\x2e\x6c\x6c\x76\x6d\x5f\x61\x64\x64\x72\x73\x69\x67\0\x6c\x69\x63\x65\
370	\x6c\x2e\x42\x54\x46\0\x4c\x49\x43\x45\x4e\x53\x45\0\x4c\x42\x42\x31\x5f\x37\0\	370	\x6e\x73\x65\0\x69\x74\x65\x72\x61\x74\x6f\x72\x73\x2e\x62\x70\x66\x2e\x63\0\
371	\x4c\x42\x42\x31\x5f\x36\0\x4c\x42\x42\x30\x5f\x34\0\x4c\x42\x42\x31\x5f\x33\0\	371	\x2e\x73\x74\x72\x74\x61\x62\0\x2e\x73\x79\x6d\x74\x61\x62\0\x2e\x72\x6f\x64\
372	\x4c\x42\x42\x30\x5f\x33\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x70\x72\x6f\x67\	372	\x61\x74\x61\0\x2e\x72\x65\x6c\x2e\x42\x54\x46\0\x4c\x49\x43\x45\x4e\x53\x45\0\
373	\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x64\x75\x6d\x70\x5f\x62\x70\x66\x5f\x6d\	373	\x4c\x42\x42\x31\x5f\x37\0\x4c\x42\x42\x31\x5f\x36\0\x4c\x42\x42\x30\x5f\x34\0\
374	\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\	374	\x4c\x42\x42\x31\x5f\x33\0\x4c\x42\x42\x30\x5f\x33\0\x64\x75\x6d\x70\x5f\x62\
		375	\x70\x66\x5f\x70\x72\x6f\x67\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x32\0\x64\x75\x6d\
		376	\x70\x5f\x62\x70\x66\x5f\x6d\x61\x70\x2e\x5f\x5f\x5f\x66\x6d\x74\x2e\x31\0\0\0\
375	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\	377	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
376	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\	378	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\x01\0\0\
377	\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\	379	\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
378	\0\0\0\0\0\0\0\x4e\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\0\0\0\	380	\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x4e\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\
379	\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\	381	\0\0\0\0\0\0\0\0\0\x40\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\
380	\x6d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x01\0\0\0\0\0\0\x08\	382	\0\0\0\0\0\0\0\0\0\0\0\0\0\x6d\0\0\0\x01\0\0\0\x06\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
381	\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\0\0\0\	383	\0\x40\x01\0\0\0\0\0\0\x08\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x08\0\0\0\0\0\0\0\0\
382	\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x03\0\0\0\0\0\0\x62\0\0\0\0\0\	384	\0\0\0\0\0\0\0\xb1\0\0\0\x01\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x48\x03\0\
383	\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x89\0\0\0\x01\0\0\0\x03\	385	\0\0\0\0\0\x62\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
384	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\x03\0\0\0\0\0\0\x04\0\0\0\0\0\0\0\0\0\0\0\0\	386	\x89\0\0\0\x01\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xaa\x03\0\0\0\0\0\0\x04\
385	\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xad\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\	387	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xbd\0\0\0\x01\
386	\0\0\0\0\0\0\0\xae\x03\0\0\0\0\0\0\x34\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\	388	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xae\x03\0\0\0\0\0\0\x3d\x09\0\0\0\0\0\0\
387	\0\0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\	389	\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x0b\0\0\0\x01\0\0\0\0\0\0\0\
388	\xe2\x0c\0\0\0\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\	390	\0\0\0\0\0\0\0\0\0\0\0\0\xeb\x0c\0\0\0\0\0\0\x2c\x04\0\0\0\0\0\0\0\0\0\0\0\0\0\
389	\0\0\0\0\0\0\x99\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x10\x11\0\0\0\	391	\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa9\0\0\0\x02\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
390	\0\0\0\x80\x01\0\0\0\0\0\0\x0e\0\0\0\x0d\0\0\0\x08\0\0\0\0\0\0\0\x18\0\0\0\0\0\	392	\0\0\0\0\0\x18\x11\0\0\0\0\0\0\x98\x01\0\0\0\0\0\0\x0e\0\0\0\x0e\0\0\0\x08\0\0\
391	\0\0\x4a\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x90\x12\0\0\0\0\0\0\	393	\0\0\0\0\0\x18\0\0\0\0\0\0\0\x4a\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\
392	\x20\0\0\0\0\0\0\0\x08\0\0\0\x02\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x69\	394	\0\xb0\x12\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x08\0\0\0\x02\0\0\0\x08\0\0\0\0\0\0\0\
393	\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xb0\x12\0\0\0\0\0\0\x20\0\0\0\	395	\x10\0\0\0\0\0\0\0\x69\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x12\
394	\0\0\0\0\x08\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\xa9\0\0\0\x09\	396	\0\0\0\0\0\0\x20\0\0\0\0\0\0\0\x08\0\0\0\x03\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\
395	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xd0\x12\0\0\0\0\0\0\x50\0\0\0\0\0\0\0\	397	\0\0\0\0\xb9\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xf0\x12\0\0\0\0\0\
396	\x08\0\0\0\x06\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x07\0\0\0\x09\0\0\0\0\	398	\0\x50\0\0\0\0\0\0\0\x08\0\0\0\x06\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\
397	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x20\x13\0\0\0\0\0\0\xe0\x03\0\0\0\0\0\0\x08\0\0\	399	\x07\0\0\0\x09\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x40\x13\0\0\0\0\0\0\xe0\
398	\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x7b\0\0\0\x03\x4c\xff\x6f\0\0\	400	\x03\0\0\0\0\0\0\x08\0\0\0\x07\0\0\0\x08\0\0\0\0\0\0\0\x10\0\0\0\0\0\0\0\x7b\0\
399	\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\0\x17\0\0\0\0\0\0\x07\0\0\0\0\0\0\0\0\0\0\0\0\0\	401	\0\0\x03\x4c\xff\x6f\0\0\0\x80\0\0\0\0\0\0\0\0\0\0\0\0\x20\x17\0\0\0\0\0\0\x07\
400	\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x91\0\0\0\x03\0\0\0\0\0\0\0\0\0\0\0\0\0\	402	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\xa1\0\0\0\x03\
401	\0\0\0\0\0\0\x07\x17\0\0\0\0\0\0\x0a\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x01\0\0\0\	403	\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\x27\x17\0\0\0\0\0\0\x1a\x01\0\0\0\0\0\0\
402	\0\0\0\0\0\0\0\0\0\0\0\0";	404	\0\0\0\0\0\0\0\0\x01\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
403		405
404	return 0;	406	return 0;
405	err:	407	err:

kernel/bpf/reuseport_array.c+1−1

@@ -191,7 +191,7 @@ int bpf_fd_reuseport_array_lookup_elem(struct bpf_map map, void key, · 190 unmodified lines
191	rcu_read_lock();	191	rcu_read_lock();
192	sk = reuseport_array_lookup_elem(map, key);	192	sk = reuseport_array_lookup_elem(map, key);
193	if (sk) {	193	if (sk) {
194	(u64 )value = sock_gen_cookie(sk);	194	(u64 )value = __sock_gen_cookie(sk);
195	err = 0;	195	err = 0;
196	} else {	196	} else {
197	err = -ENOENT;	197	err = -ENOENT;

kernel/bpf/syscall.c+163−27

@@ -4,6 +4,7 @@ · 3 unmodified lines
4	#include <linux/bpf.h>	4	#include <linux/bpf.h>
5	#include <linux/bpf_trace.h>	5	#include <linux/bpf_trace.h>
6	#include <linux/bpf_lirc.h>	6	#include <linux/bpf_lirc.h>
		7	#include <linux/bpf_verifier.h>
7	#include <linux/btf.h>	8	#include <linux/btf.h>
8	#include <linux/syscalls.h>	9	#include <linux/syscalls.h>
9	#include <linux/slab.h>	10	#include <linux/slab.h>
@@ -2154,14 +2155,14 @@ static int bpf_prog_load(union bpf_attr attr, union bpf_attr __user uattr) · 2144 unmodified lines
2154	prog->expected_attach_type = attr->expected_attach_type;	2155	prog->expected_attach_type = attr->expected_attach_type;
2155	prog->aux->attach_btf_id = attr->attach_btf_id;	2156	prog->aux->attach_btf_id = attr->attach_btf_id;
2156	if (attr->attach_prog_fd) {	2157	if (attr->attach_prog_fd) {
2157	struct bpf_prog *tgt_prog;	2158	struct bpf_prog *dst_prog;
2158		2159
2159	tgt_prog = bpf_prog_get(attr->attach_prog_fd);	2160	dst_prog = bpf_prog_get(attr->attach_prog_fd);
2160	if (IS_ERR(tgt_prog)) {	2161	if (IS_ERR(dst_prog)) {
2161	err = PTR_ERR(tgt_prog);	2162	err = PTR_ERR(dst_prog);
2162	goto free_prog_nouncharge;	2163	goto free_prog_nouncharge;
2163	}	2164	}
2164	prog->aux->linked_prog = tgt_prog;	2165	prog->aux->dst_prog = dst_prog;
2165	}	2166	}
2166		2167
2167	prog->aux->offload_requested = !!attr->prog_ifindex;	2168	prog->aux->offload_requested = !!attr->prog_ifindex;
@@ -2345,8 +2346,12 @@ void bpf_link_put(struct bpf_link *link) · 177 unmodified lines
2345	if (!atomic64_dec_and_test(&link->refcnt))	2346	if (!atomic64_dec_and_test(&link->refcnt))
2346	return;	2347	return;
2347		2348
2348	INIT_WORK(&link->work, bpf_link_put_deferred);	2349	if (in_atomic()) {
2349	schedule_work(&link->work);	2350	INIT_WORK(&link->work, bpf_link_put_deferred);
		2351	schedule_work(&link->work);
		2352	} else {
		2353	bpf_link_free(link);
		2354	}
2350	}	2355	}
2351		2356
2352	static int bpf_link_release(struct inode inode, struct file filp)	2357	static int bpf_link_release(struct inode inode, struct file filp)
@@ -2494,11 +2499,23 @@ struct bpf_link *bpf_link_get_from_fd(u32 ufd) · 141 unmodified lines
2494	struct bpf_tracing_link {	2499	struct bpf_tracing_link {
2495	struct bpf_link link;	2500	struct bpf_link link;
2496	enum bpf_attach_type attach_type;	2501	enum bpf_attach_type attach_type;
		2502	struct bpf_trampoline *trampoline;
		2503	struct bpf_prog *tgt_prog;
2497	};	2504	};
2498		2505
2499	static void bpf_tracing_link_release(struct bpf_link *link)	2506	static void bpf_tracing_link_release(struct bpf_link *link)
2500	{	2507	{
2501	WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog));	2508	struct bpf_tracing_link *tr_link =
		2509	container_of(link, struct bpf_tracing_link, link);
		2510
		2511	WARN_ON_ONCE(bpf_trampoline_unlink_prog(link->prog,
		2512	tr_link->trampoline));
		2513
		2514	bpf_trampoline_put(tr_link->trampoline);
		2515
		2516	/* tgt_prog is NULL if target is a kernel function */
		2517	if (tr_link->tgt_prog)
		2518	bpf_prog_put(tr_link->tgt_prog);
2502	}	2519	}
2503		2520
2504	static void bpf_tracing_link_dealloc(struct bpf_link *link)	2521	static void bpf_tracing_link_dealloc(struct bpf_link *link)
@@ -2538,10 +2555,15 @@ static const struct bpf_link_ops bpf_tracing_link_lops = { · 33 unmodified lines
2538	.fill_link_info = bpf_tracing_link_fill_link_info,	2555	.fill_link_info = bpf_tracing_link_fill_link_info,
2539	};	2556	};
2540		2557
2541	static int bpf_tracing_prog_attach(struct bpf_prog *prog)	2558	static int bpf_tracing_prog_attach(struct bpf_prog *prog,
		2559	int tgt_prog_fd,
		2560	u32 btf_id)
2542	{	2561	{
2543	struct bpf_link_primer link_primer;	2562	struct bpf_link_primer link_primer;
		2563	struct bpf_prog *tgt_prog = NULL;
		2564	struct bpf_trampoline *tr = NULL;
2544	struct bpf_tracing_link *link;	2565	struct bpf_tracing_link *link;
		2566	u64 key = 0;
2545	int err;	2567	int err;
2546		2568
2547	switch (prog->type) {	2569	switch (prog->type) {
@@ -2570,6 +2592,28 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog) · 22 unmodified lines
2570	goto out_put_prog;	2592	goto out_put_prog;
2571	}	2593	}
2572		2594
		2595	if (!!tgt_prog_fd != !!btf_id) {
		2596	err = -EINVAL;
		2597	goto out_put_prog;
		2598	}
		2599
		2600	if (tgt_prog_fd) {
		2601	/* For now we only allow new targets for BPF_PROG_TYPE_EXT */
		2602	if (prog->type != BPF_PROG_TYPE_EXT) {
		2603	err = -EINVAL;
		2604	goto out_put_prog;
		2605	}
		2606
		2607	tgt_prog = bpf_prog_get(tgt_prog_fd);
		2608	if (IS_ERR(tgt_prog)) {
		2609	err = PTR_ERR(tgt_prog);
		2610	tgt_prog = NULL;
		2611	goto out_put_prog;
		2612	}
		2613
		2614	key = bpf_trampoline_compute_key(tgt_prog, btf_id);
		2615	}
		2616
2573	link = kzalloc(sizeof(*link), GFP_USER);	2617	link = kzalloc(sizeof(*link), GFP_USER);
2574	if (!link) {	2618	if (!link) {
2575	err = -ENOMEM;	2619	err = -ENOMEM;
@@ -2579,20 +2623,100 @@ static int bpf_tracing_prog_attach(struct bpf_prog *prog) · 3 unmodified lines
2579	&bpf_tracing_link_lops, prog);	2623	&bpf_tracing_link_lops, prog);
2580	link->attach_type = prog->expected_attach_type;	2624	link->attach_type = prog->expected_attach_type;
2581		2625
2582	err = bpf_link_prime(&link->link, &link_primer);	2626	mutex_lock(&prog->aux->dst_mutex);
2583	if (err) {	2627
2584	kfree(link);	2628	/* There are a few possible cases here:
2585	goto out_put_prog;	2629	*
		2630	* - if prog->aux->dst_trampoline is set, the program was just loaded
		2631	* and not yet attached to anything, so we can use the values stored
		2632	* in prog->aux
		2633	*
		2634	* - if prog->aux->dst_trampoline is NULL, the program has already been
		2635	* attached to a target and its initial target was cleared (below)
		2636	*
		2637	* - if tgt_prog != NULL, the caller specified tgt_prog_fd +
		2638	* target_btf_id using the link_create API.
		2639	*
		2640	* - if tgt_prog == NULL when this function was called using the old
		2641	* raw_tracepoint_open API, and we need a target from prog->aux
		2642	*
		2643	* The combination of no saved target in prog->aux, and no target
		2644	* specified on load is illegal, and we reject that here.
		2645	*/
		2646	if (!prog->aux->dst_trampoline && !tgt_prog) {
		2647	err = -ENOENT;
		2648	goto out_unlock;
2586	}	2649	}
2587		2650
2588	err = bpf_trampoline_link_prog(prog);	2651	if (!prog->aux->dst_trampoline \|\|
		2652	(key && key != prog->aux->dst_trampoline->key)) {
		2653	/* If there is no saved target, or the specified target is
		2654	* different from the destination specified at load time, we
		2655	* need a new trampoline and a check for compatibility
		2656	*/
		2657	struct bpf_attach_target_info tgt_info = {};
		2658
		2659	err = bpf_check_attach_target(NULL, prog, tgt_prog, btf_id,
		2660	&tgt_info);
		2661	if (err)
		2662	goto out_unlock;
		2663
		2664	tr = bpf_trampoline_get(key, &tgt_info);
		2665	if (!tr) {
		2666	err = -ENOMEM;
		2667	goto out_unlock;
		2668	}
		2669	} else {
		2670	/* The caller didn't specify a target, or the target was the
		2671	* same as the destination supplied during program load. This
		2672	* means we can reuse the trampoline and reference from program
		2673	* load time, and there is no need to allocate a new one. This
		2674	* can only happen once for any program, as the saved values in
		2675	* prog->aux are cleared below.
		2676	*/
		2677	tr = prog->aux->dst_trampoline;
		2678	tgt_prog = prog->aux->dst_prog;
		2679	}
		2680
		2681	err = bpf_link_prime(&link->link, &link_primer);
		2682	if (err)
		2683	goto out_unlock;
		2684
		2685	err = bpf_trampoline_link_prog(prog, tr);
2589	if (err) {	2686	if (err) {
2590	bpf_link_cleanup(&link_primer);	2687	bpf_link_cleanup(&link_primer);
2591	goto out_put_prog;	2688	link = NULL;
		2689	goto out_unlock;
2592	}	2690	}
2593		2691
		2692	link->tgt_prog = tgt_prog;
		2693	link->trampoline = tr;
		2694
		2695	/* Always clear the trampoline and target prog from prog->aux to make
		2696	* sure the original attach destination is not kept alive after a
		2697	* program is (re-)attached to another target.
		2698	*/
		2699	if (prog->aux->dst_prog &&
		2700	(tgt_prog_fd \|\| tr != prog->aux->dst_trampoline))
		2701	/* got extra prog ref from syscall, or attaching to different prog */
		2702	bpf_prog_put(prog->aux->dst_prog);
		2703	if (prog->aux->dst_trampoline && tr != prog->aux->dst_trampoline)
		2704	/* we allocated a new trampoline, so free the old one */
		2705	bpf_trampoline_put(prog->aux->dst_trampoline);
		2706
		2707	prog->aux->dst_prog = NULL;
		2708	prog->aux->dst_trampoline = NULL;
		2709	mutex_unlock(&prog->aux->dst_mutex);
		2710
2594	return bpf_link_settle(&link_primer);	2711	return bpf_link_settle(&link_primer);
		2712	out_unlock:
		2713	if (tr && tr != prog->aux->dst_trampoline)
		2714	bpf_trampoline_put(tr);
		2715	mutex_unlock(&prog->aux->dst_mutex);
		2716	kfree(link);
2595	out_put_prog:	2717	out_put_prog:
		2718	if (tgt_prog_fd && tgt_prog)
		2719	bpf_prog_put(tgt_prog);
2596	bpf_prog_put(prog);	2720	bpf_prog_put(prog);
2597	return err;	2721	return err;
2598	}	2722	}
@@ -2706,7 +2830,7 @@ static int bpf_raw_tracepoint_open(const union bpf_attr *attr) · 107 unmodified lines
2706	tp_name = prog->aux->attach_func_name;	2830	tp_name = prog->aux->attach_func_name;
2707	break;	2831	break;
2708	}	2832	}
2709	return bpf_tracing_prog_attach(prog);	2833	return bpf_tracing_prog_attach(prog, 0, 0);
2710	case BPF_PROG_TYPE_RAW_TRACEPOINT:	2834	case BPF_PROG_TYPE_RAW_TRACEPOINT:
2711	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:	2835	case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
2712	if (strncpy_from_user(buf,	2836	if (strncpy_from_user(buf,
@@ -2975,7 +3099,7 @@ static int bpf_prog_query(const union bpf_attr *attr, · 262 unmodified lines
2975	}	3099	}
2976	}	3100	}
2977		3101
2978	#define BPF_PROG_TEST_RUN_LAST_FIELD test.ctx_out	3102	#define BPF_PROG_TEST_RUN_LAST_FIELD test.cpu
2979		3103
2980	static int bpf_prog_test_run(const union bpf_attr *attr,	3104	static int bpf_prog_test_run(const union bpf_attr *attr,
2981	union bpf_attr __user *uattr)	3105	union bpf_attr __user *uattr)
@@ -3890,10 +4014,15 @@ err_put: · 908 unmodified lines
3890		4014
3891	static int tracing_bpf_link_attach(const union bpf_attr attr, struct bpf_prog prog)	4015	static int tracing_bpf_link_attach(const union bpf_attr attr, struct bpf_prog prog)
3892	{	4016	{
3893	if (attr->link_create.attach_type == BPF_TRACE_ITER &&	4017	if (attr->link_create.attach_type != prog->expected_attach_type)
3894	prog->expected_attach_type == BPF_TRACE_ITER)	4018	return -EINVAL;
3895	return bpf_iter_link_attach(attr, prog);
3896		4019
		4020	if (prog->expected_attach_type == BPF_TRACE_ITER)
		4021	return bpf_iter_link_attach(attr, prog);
		4022	else if (prog->type == BPF_PROG_TYPE_EXT)
		4023	return bpf_tracing_prog_attach(prog,
		4024	attr->link_create.target_fd,
		4025	attr->link_create.target_btf_id);
3897	return -EINVAL;	4026	return -EINVAL;
3898	}	4027	}
3899		4028
@@ -3907,18 +4036,25 @@ static int link_create(union bpf_attr *attr) · 7 unmodified lines
3907	if (CHECK_ATTR(BPF_LINK_CREATE))	4036	if (CHECK_ATTR(BPF_LINK_CREATE))
3908	return -EINVAL;	4037	return -EINVAL;
3909		4038
3910	ptype = attach_type_to_prog_type(attr->link_create.attach_type);	4039	prog = bpf_prog_get(attr->link_create.prog_fd);
3911	if (ptype == BPF_PROG_TYPE_UNSPEC)
3912	return -EINVAL;
3913
3914	prog = bpf_prog_get_type(attr->link_create.prog_fd, ptype);
3915	if (IS_ERR(prog))	4040	if (IS_ERR(prog))
3916	return PTR_ERR(prog);	4041	return PTR_ERR(prog);
3917		4042
3918	ret = bpf_prog_attach_check_attach_type(prog,	4043	ret = bpf_prog_attach_check_attach_type(prog,
3919	attr->link_create.attach_type);	4044	attr->link_create.attach_type);
3920	if (ret)	4045	if (ret)
3921	goto err_out;	4046	goto out;
		4047
		4048	if (prog->type == BPF_PROG_TYPE_EXT) {
		4049	ret = tracing_bpf_link_attach(attr, prog);
		4050	goto out;
		4051	}
		4052
		4053	ptype = attach_type_to_prog_type(attr->link_create.attach_type);
		4054	if (ptype == BPF_PROG_TYPE_UNSPEC \|\| ptype != prog->type) {
		4055	ret = -EINVAL;
		4056	goto out;
		4057	}
3922		4058
3923	switch (ptype) {	4059	switch (ptype) {
3924	case BPF_PROG_TYPE_CGROUP_SKB:	4060	case BPF_PROG_TYPE_CGROUP_SKB:
@@ -3946,7 +4082,7 @@ static int link_create(union bpf_attr *attr) · 21 unmodified lines
3946	ret = -EINVAL;	4082	ret = -EINVAL;
3947	}	4083	}
3948		4084
3949	err_out:	4085	out:
3950	if (ret < 0)	4086	if (ret < 0)
3951	bpf_prog_put(prog);	4087	bpf_prog_put(prog);
3952	return ret;	4088	return ret;

kernel/bpf/trampoline.c+25−9

 	bpf_image_ksym_add(tr->image, ksym);
 }
-struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
+static struct bpf_trampoline *bpf_trampoline_lookup(u64 key)
 {
 	struct bpf_trampoline *tr;
 	struct hlist_head *head;
 	}
 }
-int bpf_trampoline_link_prog(struct bpf_prog *prog)
+int bpf_trampoline_link_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
 {
 	enum bpf_tramp_prog_type kind;
-	struct bpf_trampoline *tr;
 	int err = 0;
 	int cnt;
-	tr = prog->aux->trampoline;
 	kind = bpf_attach_type_to_tramp(prog);
 	mutex_lock(&tr->mutex);
 	if (tr->extension_prog) {
 	}
 	hlist_add_head(&prog->aux->tramp_hlist, &tr->progs_hlist[kind]);
 	tr->progs_cnt[kind]++;
-	err = bpf_trampoline_update(prog->aux->trampoline);
+	err = bpf_trampoline_update(tr);
 	if (err) {
 		hlist_del(&prog->aux->tramp_hlist);
 		tr->progs_cnt[kind]--;
 }
 /* bpf_trampoline_unlink_prog() should never fail. */
-int bpf_trampoline_unlink_prog(struct bpf_prog *prog)
+int bpf_trampoline_unlink_prog(struct bpf_prog *prog, struct bpf_trampoline *tr)
 {
 	enum bpf_tramp_prog_type kind;
-	struct bpf_trampoline *tr;
 	int err;
-	tr = prog->aux->trampoline;
 	kind = bpf_attach_type_to_tramp(prog);
 	mutex_lock(&tr->mutex);
 	if (kind == BPF_TRAMP_REPLACE) {
 	}
 	hlist_del(&prog->aux->tramp_hlist);
 	tr->progs_cnt[kind]--;
-	err = bpf_trampoline_update(prog->aux->trampoline);
+	err = bpf_trampoline_update(tr);
 out:
 	mutex_unlock(&tr->mutex);
 	return err;
 }
+struct bpf_trampoline *bpf_trampoline_get(u64 key,
+					  struct bpf_attach_target_info *tgt_info)
+{
+	struct bpf_trampoline *tr;
+	tr = bpf_trampoline_lookup(key);
+	if (!tr)
+		return NULL;
+	mutex_lock(&tr->mutex);
+	if (tr->func.addr)
+		goto out;
+	memcpy(&tr->func.model, &tgt_info->fmodel, sizeof(tgt_info->fmodel));
+	tr->func.addr = (void *)tgt_info->tgt_addr;
+out:
+	mutex_unlock(&tr->mutex);
+	return tr;
+}
 void bpf_trampoline_put(struct bpf_trampoline *tr)
 {
 	if (!tr)

kernel/bpf/verifier.c+211−146

@@ -486,7 +486,12 @@ static bool is_acquire_function(enum bpf_func_id func_id, · 485 unmodified lines
486	static bool is_ptr_cast_function(enum bpf_func_id func_id)	486	static bool is_ptr_cast_function(enum bpf_func_id func_id)
487	{	487	{
488	return func_id == BPF_FUNC_tcp_sock \|\|	488	return func_id == BPF_FUNC_tcp_sock \|\|
489	func_id == BPF_FUNC_sk_fullsock;	489	func_id == BPF_FUNC_sk_fullsock \|\|
		490	func_id == BPF_FUNC_skc_to_tcp_sock \|\|
		491	func_id == BPF_FUNC_skc_to_tcp6_sock \|\|
		492	func_id == BPF_FUNC_skc_to_udp6_sock \|\|
		493	func_id == BPF_FUNC_skc_to_tcp_timewait_sock \|\|
		494	func_id == BPF_FUNC_skc_to_tcp_request_sock;
490	}	495	}
491		496
492	/* string representation of 'enum bpf_reg_type' */	497	/* string representation of 'enum bpf_reg_type' */
@@ -2643,8 +2648,7 @@ static int check_map_access(struct bpf_verifier_env *env, u32 regno, · 2150 unmodified lines
2643		2648
2644	static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)	2649	static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)
2645	{	2650	{
2646	return prog->aux->linked_prog ? prog->aux->linked_prog->type	2651	return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type;
2647	: prog->type;
2648	}	2652	}
2649		2653
2650	static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,	2654	static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
@@ -3938,7 +3942,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, · 1287 unmodified lines
3938	case BPF_MAP_TYPE_SOCKMAP:	3942	case BPF_MAP_TYPE_SOCKMAP:
3939	case BPF_MAP_TYPE_SOCKHASH:	3943	case BPF_MAP_TYPE_SOCKHASH:
3940	if (*arg_type == ARG_PTR_TO_MAP_VALUE) {	3944	if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
3941	*arg_type = ARG_PTR_TO_SOCKET;	3945	*arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
3942	} else {	3946	} else {
3943	verbose(env, "invalid arg_type for sockmap/sockhash\n");	3947	verbose(env, "invalid arg_type for sockmap/sockhash\n");
3944	return -EINVAL;	3948	return -EINVAL;
@@ -3953,6 +3957,7 @@ static int resolve_map_arg_type(struct bpf_verifier_env *env, · 8 unmodified lines
3953		3957
3954	struct bpf_reg_types {	3958	struct bpf_reg_types {
3955	const enum bpf_reg_type types[10];	3959	const enum bpf_reg_type types[10];
		3960	u32 *btf_id;
3956	};	3961	};
3957		3962
3958	static const struct bpf_reg_types map_key_value_types = {	3963	static const struct bpf_reg_types map_key_value_types = {
@@ -3973,6 +3978,17 @@ static const struct bpf_reg_types sock_types = { · 14 unmodified lines
3973	},	3978	},
3974	};	3979	};
3975		3980
		3981	static const struct bpf_reg_types btf_id_sock_common_types = {
		3982	.types = {
		3983	PTR_TO_SOCK_COMMON,
		3984	PTR_TO_SOCKET,
		3985	PTR_TO_TCP_SOCK,
		3986	PTR_TO_XDP_SOCK,
		3987	PTR_TO_BTF_ID,
		3988	},
		3989	.btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
		3990	};
		3991
3976	static const struct bpf_reg_types mem_types = {	3992	static const struct bpf_reg_types mem_types = {
3977	.types = {	3993	.types = {
3978	PTR_TO_STACK,	3994	PTR_TO_STACK,
@@ -4014,6 +4030,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 35 unmodified lines
4014	[ARG_PTR_TO_CTX] = &context_types,	4030	[ARG_PTR_TO_CTX] = &context_types,
4015	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,	4031	[ARG_PTR_TO_CTX_OR_NULL] = &context_types,
4016	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,	4032	[ARG_PTR_TO_SOCK_COMMON] = &sock_types,
		4033	[ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
4017	[ARG_PTR_TO_SOCKET] = &fullsock_types,	4034	[ARG_PTR_TO_SOCKET] = &fullsock_types,
4018	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,	4035	[ARG_PTR_TO_SOCKET_OR_NULL] = &fullsock_types,
4019	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,	4036	[ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
@@ -4028,19 +4045,27 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { · 8 unmodified lines
4028	};	4045	};
4029		4046
4030	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,	4047	static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
4031	const struct bpf_reg_types *compatible)	4048	enum bpf_arg_type arg_type,
		4049	const u32 *arg_btf_id)
4032	{	4050	{
4033	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];	4051	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];
4034	enum bpf_reg_type expected, type = reg->type;	4052	enum bpf_reg_type expected, type = reg->type;
		4053	const struct bpf_reg_types *compatible;
4035	int i, j;	4054	int i, j;
4036		4055
		4056	compatible = compatible_reg_types[arg_type];
		4057	if (!compatible) {
		4058	verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
		4059	return -EFAULT;
		4060	}
		4061
4037	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {	4062	for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
4038	expected = compatible->types[i];	4063	expected = compatible->types[i];
4039	if (expected == NOT_INIT)	4064	if (expected == NOT_INIT)
4040	break;	4065	break;
4041		4066
4042	if (type == expected)	4067	if (type == expected)
4043	return 0;	4068	goto found;
4044	}	4069	}
4045		4070
4046	verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);	4071	verbose(env, "R%d type=%s expected=", regno, reg_type_str[type]);
@@ -4048,6 +4073,33 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno, · 1 unmodified line
4048	verbose(env, "%s, ", reg_type_str[compatible->types[j]]);	4073	verbose(env, "%s, ", reg_type_str[compatible->types[j]]);
4049	verbose(env, "%s\n", reg_type_str[compatible->types[j]]);	4074	verbose(env, "%s\n", reg_type_str[compatible->types[j]]);
4050	return -EACCES;	4075	return -EACCES;
		4076
		4077	found:
		4078	if (type == PTR_TO_BTF_ID) {
		4079	if (!arg_btf_id) {
		4080	if (!compatible->btf_id) {
		4081	verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
		4082	return -EFAULT;
		4083	}
		4084	arg_btf_id = compatible->btf_id;
		4085	}
		4086
		4087	if (!btf_struct_ids_match(&env->log, reg->off, reg->btf_id,
		4088	*arg_btf_id)) {
		4089	verbose(env, "R%d is of type %s but %s is expected\n",
		4090	regno, kernel_type_name(reg->btf_id),
		4091	kernel_type_name(*arg_btf_id));
		4092	return -EACCES;
		4093	}
		4094
		4095	if (!tnum_is_const(reg->var_off) \|\| reg->var_off.value) {
		4096	verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",
		4097	regno);
		4098	return -EACCES;
		4099	}
		4100	}
		4101
		4102	return 0;
4051	}	4103	}
4052		4104
4053	static int check_func_arg(struct bpf_verifier_env *env, u32 arg,	4105	static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
@@ -4057,7 +4109,6 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 3 unmodified lines
4057	u32 regno = BPF_REG_1 + arg;	4109	u32 regno = BPF_REG_1 + arg;
4058	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];	4110	struct bpf_reg_state regs = cur_regs(env), reg = &regs[regno];
4059	enum bpf_arg_type arg_type = fn->arg_type[arg];	4111	enum bpf_arg_type arg_type = fn->arg_type[arg];
4060	const struct bpf_reg_types *compatible;
4061	enum bpf_reg_type type = reg->type;	4112	enum bpf_reg_type type = reg->type;
4062	int err = 0;	4113	int err = 0;
4063		4114
@@ -4097,35 +4148,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg, · 33 unmodified lines
4097	*/	4148	*/
4098	goto skip_type_check;	4149	goto skip_type_check;
4099		4150
4100	compatible = compatible_reg_types[arg_type];	4151	err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]);
4101	if (!compatible) {
4102	verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
4103	return -EFAULT;
4104	}
4105
4106	err = check_reg_type(env, regno, compatible);
4107	if (err)	4152	if (err)
4108	return err;	4153	return err;
4109		4154
4110	if (type == PTR_TO_BTF_ID) {	4155	if (type == PTR_TO_CTX) {
4111	const u32 *btf_id = fn->arg_btf_id[arg];
4112
4113	if (!btf_id) {
4114	verbose(env, "verifier internal error: missing BTF ID\n");
4115	return -EFAULT;
4116	}
4117
4118	if (!btf_struct_ids_match(&env->log, reg->off, reg->btf_id, *btf_id)) {
4119	verbose(env, "R%d is of type %s but %s is expected\n",
4120	regno, kernel_type_name(reg->btf_id), kernel_type_name(*btf_id));
4121	return -EACCES;
4122	}
4123	if (!tnum_is_const(reg->var_off) \|\| reg->var_off.value) {
4124	verbose(env, "R%d is a pointer to in-kernel struct with non-zero offset\n",
4125	regno);
4126	return -EACCES;
4127	}
4128	} else if (type == PTR_TO_CTX) {
4129	err = check_ctx_reg(env, reg, regno);	4156	err = check_ctx_reg(env, reg, regno);
4130	if (err < 0)	4157	if (err < 0)
4131	return err;	4158	return err;
@@ -4573,10 +4600,14 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn) · 441 unmodified lines
4573	{	4600	{
4574	int i;	4601	int i;
4575		4602
4576	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++)	4603	for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
4577	if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])	4604	if (fn->arg_type[i] == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])
4578	return false;	4605	return false;
4579		4606
		4607	if (fn->arg_type[i] != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i])
		4608	return false;
		4609	}
		4610
4580	return true;	4611	return true;
4581	}	4612	}
4582		4613
@@ -5817,8 +5848,7 @@ static void scalar_min_max_and(struct bpf_reg_state *dst_reg, · 1234 unmodified lines
5817	u64 umax_val = src_reg->umax_value;	5848	u64 umax_val = src_reg->umax_value;
5818		5849
5819	if (src_known && dst_known) {	5850	if (src_known && dst_known) {
5820	__mark_reg_known(dst_reg, dst_reg->var_off.value &	5851	__mark_reg_known(dst_reg, dst_reg->var_off.value);
5821	src_reg->var_off.value);
5822	return;	5852	return;
5823	}	5853	}
5824		5854
@@ -5888,8 +5918,7 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg, · 63 unmodified lines
5888	u64 umin_val = src_reg->umin_value;	5918	u64 umin_val = src_reg->umin_value;
5889		5919
5890	if (src_known && dst_known) {	5920	if (src_known && dst_known) {
5891	__mark_reg_known(dst_reg, dst_reg->var_off.value \|	5921	__mark_reg_known(dst_reg, dst_reg->var_off.value);
5892	src_reg->var_off.value);
5893	return;	5922	return;
5894	}	5923	}
5895		5924
@@ -11173,11 +11202,10 @@ static int check_struct_ops_btf_id(struct bpf_verifier_env *env) · 5277 unmodified lines
11173	}	11202	}
11174	#define SECURITY_PREFIX "security_"	11203	#define SECURITY_PREFIX "security_"
11175		11204
11176	static int check_attach_modify_return(struct bpf_prog *prog, unsigned long addr)	11205	static int check_attach_modify_return(unsigned long addr, const char *func_name)
11177	{	11206	{
11178	if (within_error_injection_list(addr) \|\|	11207	if (within_error_injection_list(addr) \|\|
11179	!strncmp(SECURITY_PREFIX, prog->aux->attach_func_name,	11208	!strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
11180	sizeof(SECURITY_PREFIX) - 1))
11181	return 0;	11209	return 0;
11182		11210
11183	return -EINVAL;	11211	return -EINVAL;
@@ -11214,55 +11242,39 @@ static int check_non_sleepable_error_inject(u32 btf_id) · 30 unmodified lines
11214	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);	11242	return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
11215	}	11243	}
11216		11244
11217	static int check_attach_btf_id(struct bpf_verifier_env *env)	11245	int bpf_check_attach_target(struct bpf_verifier_log *log,
		11246	const struct bpf_prog *prog,
		11247	const struct bpf_prog *tgt_prog,
		11248	u32 btf_id,
		11249	struct bpf_attach_target_info *tgt_info)
11218	{	11250	{
11219	struct bpf_prog *prog = env->prog;
11220	bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;	11251	bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
11221	struct bpf_prog *tgt_prog = prog->aux->linked_prog;
11222	u32 btf_id = prog->aux->attach_btf_id;
11223	const char prefix[] = "btf_trace_";	11252	const char prefix[] = "btf_trace_";
11224	struct btf_func_model fmodel;
11225	int ret = 0, subprog = -1, i;	11253	int ret = 0, subprog = -1, i;
11226	struct bpf_trampoline *tr;
11227	const struct btf_type *t;	11254	const struct btf_type *t;
11228	bool conservative = true;	11255	bool conservative = true;
11229	const char *tname;	11256	const char *tname;
11230	struct btf *btf;	11257	struct btf *btf;
11231	long addr;	11258	long addr = 0;
11232	u64 key;
11233
11234	if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
11235	prog->type != BPF_PROG_TYPE_LSM) {
11236	verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
11237	return -EINVAL;
11238	}
11239
11240	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
11241	return check_struct_ops_btf_id(env);
11242
11243	if (prog->type != BPF_PROG_TYPE_TRACING &&
11244	prog->type != BPF_PROG_TYPE_LSM &&
11245	!prog_extension)
11246	return 0;
11247		11259
11248	if (!btf_id) {	11260	if (!btf_id) {
11249	verbose(env, "Tracing programs must provide btf_id\n");	11261	bpf_log(log, "Tracing programs must provide btf_id\n");
11250	return -EINVAL;	11262	return -EINVAL;
11251	}	11263	}
11252	btf = bpf_prog_get_target_btf(prog);	11264	btf = tgt_prog ? tgt_prog->aux->btf : btf_vmlinux;
11253	if (!btf) {	11265	if (!btf) {
11254	verbose(env,	11266	bpf_log(log,
11255	"FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");	11267	"FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
11256	return -EINVAL;	11268	return -EINVAL;
11257	}	11269	}
11258	t = btf_type_by_id(btf, btf_id);	11270	t = btf_type_by_id(btf, btf_id);
11259	if (!t) {	11271	if (!t) {
11260	verbose(env, "attach_btf_id %u is invalid\n", btf_id);	11272	bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
11261	return -EINVAL;	11273	return -EINVAL;
11262	}	11274	}
11263	tname = btf_name_by_offset(btf, t->name_off);	11275	tname = btf_name_by_offset(btf, t->name_off);
11264	if (!tname) {	11276	if (!tname) {
11265	verbose(env, "attach_btf_id %u doesn't have a name\n", btf_id);	11277	bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
11266	return -EINVAL;	11278	return -EINVAL;
11267	}	11279	}
11268	if (tgt_prog) {	11280	if (tgt_prog) {
@@ -11274,26 +11286,24 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 5 unmodified lines
11274	break;	11286	break;
11275	}	11287	}
11276	if (subprog == -1) {	11288	if (subprog == -1) {
11277	verbose(env, "Subprog %s doesn't exist\n", tname);	11289	bpf_log(log, "Subprog %s doesn't exist\n", tname);
11278	return -EINVAL;	11290	return -EINVAL;
11279	}	11291	}
11280	conservative = aux->func_info_aux[subprog].unreliable;	11292	conservative = aux->func_info_aux[subprog].unreliable;
11281	if (prog_extension) {	11293	if (prog_extension) {
11282	if (conservative) {	11294	if (conservative) {
11283	verbose(env,	11295	bpf_log(log,
11284	"Cannot replace static functions\n");	11296	"Cannot replace static functions\n");
11285	return -EINVAL;	11297	return -EINVAL;
11286	}	11298	}
11287	if (!prog->jit_requested) {	11299	if (!prog->jit_requested) {
11288	verbose(env,	11300	bpf_log(log,
11289	"Extension programs should be JITed\n");	11301	"Extension programs should be JITed\n");
11290	return -EINVAL;	11302	return -EINVAL;
11291	}	11303	}
11292	env->ops = bpf_verifier_ops[tgt_prog->type];
11293	prog->expected_attach_type = tgt_prog->expected_attach_type;
11294	}	11304	}
11295	if (!tgt_prog->jited) {	11305	if (!tgt_prog->jited) {
11296	verbose(env, "Can attach to only JITed progs\n");	11306	bpf_log(log, "Can attach to only JITed progs\n");
11297	return -EINVAL;	11307	return -EINVAL;
11298	}	11308	}
11299	if (tgt_prog->type == prog->type) {	11309	if (tgt_prog->type == prog->type) {
@@ -11301,7 +11311,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 1 unmodified line
11301	* Cannot attach program extension to another extension.	11311	* Cannot attach program extension to another extension.
11302	* It's ok to attach fentry/fexit to extension program.	11312	* It's ok to attach fentry/fexit to extension program.
11303	*/	11313	*/
11304	verbose(env, "Cannot recursively attach\n");	11314	bpf_log(log, "Cannot recursively attach\n");
11305	return -EINVAL;	11315	return -EINVAL;
11306	}	11316	}
11307	if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&	11317	if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
@@ -11323,32 +11333,30 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 15 unmodified lines
11323	* reasonable stack size. Hence extending fentry is not	11333	* reasonable stack size. Hence extending fentry is not
11324	* allowed.	11334	* allowed.
11325	*/	11335	*/
11326	verbose(env, "Cannot extend fentry/fexit\n");	11336	bpf_log(log, "Cannot extend fentry/fexit\n");
11327	return -EINVAL;	11337	return -EINVAL;
11328	}	11338	}
11329	key = ((u64)aux->id) << 32 \| btf_id;
11330	} else {	11339	} else {
11331	if (prog_extension) {	11340	if (prog_extension) {
11332	verbose(env, "Cannot replace kernel functions\n");	11341	bpf_log(log, "Cannot replace kernel functions\n");
11333	return -EINVAL;	11342	return -EINVAL;
11334	}	11343	}
11335	key = btf_id;
11336	}	11344	}
11337		11345
11338	switch (prog->expected_attach_type) {	11346	switch (prog->expected_attach_type) {
11339	case BPF_TRACE_RAW_TP:	11347	case BPF_TRACE_RAW_TP:
11340	if (tgt_prog) {	11348	if (tgt_prog) {
11341	verbose(env,	11349	bpf_log(log,
11342	"Only FENTRY/FEXIT progs are attachable to another BPF prog\n");	11350	"Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
11343	return -EINVAL;	11351	return -EINVAL;
11344	}	11352	}
11345	if (!btf_type_is_typedef(t)) {	11353	if (!btf_type_is_typedef(t)) {
11346	verbose(env, "attach_btf_id %u is not a typedef\n",	11354	bpf_log(log, "attach_btf_id %u is not a typedef\n",
11347	btf_id);	11355	btf_id);
11348	return -EINVAL;	11356	return -EINVAL;
11349	}	11357	}
11350	if (strncmp(prefix, tname, sizeof(prefix) - 1)) {	11358	if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
11351	verbose(env, "attach_btf_id %u points to wrong type name %s\n",	11359	bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
11352	btf_id, tname);	11360	btf_id, tname);
11353	return -EINVAL;	11361	return -EINVAL;
11354	}	11362	}
@@ -11362,29 +11370,20 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 7 unmodified lines
11362	/* should never happen in valid vmlinux build */	11370	/* should never happen in valid vmlinux build */
11363	return -EINVAL;	11371	return -EINVAL;
11364		11372
11365	/* remember two read only pointers that are valid for	11373	break;
11366	* the life time of the kernel
11367	*/
11368	prog->aux->attach_func_name = tname;
11369	prog->aux->attach_func_proto = t;
11370	prog->aux->attach_btf_trace = true;
11371	return 0;
11372	case BPF_TRACE_ITER:	11374	case BPF_TRACE_ITER:
11373	if (!btf_type_is_func(t)) {	11375	if (!btf_type_is_func(t)) {
11374	verbose(env, "attach_btf_id %u is not a function\n",	11376	bpf_log(log, "attach_btf_id %u is not a function\n",
11375	btf_id);	11377	btf_id);
11376	return -EINVAL;	11378	return -EINVAL;
11377	}	11379	}
11378	t = btf_type_by_id(btf, t->type);	11380	t = btf_type_by_id(btf, t->type);
11379	if (!btf_type_is_func_proto(t))	11381	if (!btf_type_is_func_proto(t))
11380	return -EINVAL;	11382	return -EINVAL;
11381	prog->aux->attach_func_name = tname;	11383	ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
11382	prog->aux->attach_func_proto = t;	11384	if (ret)
11383	if (!bpf_iter_prog_supported(prog))	11385	return ret;
11384	return -EINVAL;	11386	break;
11385	ret = btf_distill_func_proto(&env->log, btf, t,
11386	tname, &fmodel);
11387	return ret;
11388	default:	11387	default:
11389	if (!prog_extension)	11388	if (!prog_extension)
11390	return -EINVAL;	11389	return -EINVAL;
@@ -11393,42 +11392,30 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 2 unmodified lines
11393	case BPF_LSM_MAC:	11392	case BPF_LSM_MAC:
11394	case BPF_TRACE_FENTRY:	11393	case BPF_TRACE_FENTRY:
11395	case BPF_TRACE_FEXIT:	11394	case BPF_TRACE_FEXIT:
11396	prog->aux->attach_func_name = tname;
11397	if (prog->type == BPF_PROG_TYPE_LSM) {
11398	ret = bpf_lsm_verify_prog(&env->log, prog);
11399	if (ret < 0)
11400	return ret;
11401	}
11402
11403	if (!btf_type_is_func(t)) {	11395	if (!btf_type_is_func(t)) {
11404	verbose(env, "attach_btf_id %u is not a function\n",	11396	bpf_log(log, "attach_btf_id %u is not a function\n",
11405	btf_id);	11397	btf_id);
11406	return -EINVAL;	11398	return -EINVAL;
11407	}	11399	}
11408	if (prog_extension &&	11400	if (prog_extension &&
11409	btf_check_type_match(env, prog, btf, t))	11401	btf_check_type_match(log, prog, btf, t))
11410	return -EINVAL;	11402	return -EINVAL;
11411	t = btf_type_by_id(btf, t->type);	11403	t = btf_type_by_id(btf, t->type);
11412	if (!btf_type_is_func_proto(t))	11404	if (!btf_type_is_func_proto(t))
11413	return -EINVAL;	11405	return -EINVAL;
11414	tr = bpf_trampoline_lookup(key);	11406
11415	if (!tr)	11407	if ((prog->aux->saved_dst_prog_type \|\| prog->aux->saved_dst_attach_type) &&
11416	return -ENOMEM;	11408	(!tgt_prog \|\| prog->aux->saved_dst_prog_type != tgt_prog->type \|\|
11417	/* t is either vmlinux type or another program's type */	11409	prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
11418	prog->aux->attach_func_proto = t;	11410	return -EINVAL;
11419	mutex_lock(&tr->mutex);	11411
11420	if (tr->func.addr) {	11412	if (tgt_prog && conservative)
11421	prog->aux->trampoline = tr;
11422	goto out;
11423	}
11424	if (tgt_prog && conservative) {
11425	prog->aux->attach_func_proto = NULL;
11426	t = NULL;	11413	t = NULL;
11427	}	11414
11428	ret = btf_distill_func_proto(&env->log, btf, t,	11415	ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
11429	tname, &tr->func.model);
11430	if (ret < 0)	11416	if (ret < 0)
11431	goto out;	11417	return ret;
		11418
11432	if (tgt_prog) {	11419	if (tgt_prog) {
11433	if (subprog == 0)	11420	if (subprog == 0)
11434	addr = (long) tgt_prog->bpf_func;	11421	addr = (long) tgt_prog->bpf_func;
@@ -11437,11 +11424,10 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 2 unmodified lines
11437	} else {	11424	} else {
11438	addr = kallsyms_lookup_name(tname);	11425	addr = kallsyms_lookup_name(tname);
11439	if (!addr) {	11426	if (!addr) {
11440	verbose(env,	11427	bpf_log(log,
11441	"The address of function %s cannot be found\n",	11428	"The address of function %s cannot be found\n",
11442	tname);	11429	tname);
11443	ret = -ENOENT;	11430	return -ENOENT;
11444	goto out;
11445	}	11431	}
11446	}	11432	}
11447		11433
@@ -11466,25 +11452,109 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 18 unmodified lines
11466	default:	11452	default:
11467	break;	11453	break;
11468	}	11454	}
11469	if (ret)	11455	if (ret) {
11470	verbose(env, "%s is not sleepable\n",	11456	bpf_log(log, "%s is not sleepable\n", tname);
11471	prog->aux->attach_func_name);	11457	return ret;
		11458	}
11472	} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {	11459	} else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
11473	ret = check_attach_modify_return(prog, addr);	11460	if (tgt_prog) {
11474	if (ret)	11461	bpf_log(log, "can't modify return codes of BPF programs\n");
11475	verbose(env, "%s() is not modifiable\n",	11462	return -EINVAL;
11476	prog->aux->attach_func_name);	11463	}
		11464	ret = check_attach_modify_return(addr, tname);
		11465	if (ret) {
		11466	bpf_log(log, "%s() is not modifiable\n", tname);
		11467	return ret;
		11468	}
11477	}	11469	}
11478	if (ret)	11470
11479	goto out;	11471	break;
11480	tr->func.addr = (void *)addr;	11472	}
11481	prog->aux->trampoline = tr;	11473	tgt_info->tgt_addr = addr;
11482	out:	11474	tgt_info->tgt_name = tname;
11483	mutex_unlock(&tr->mutex);	11475	tgt_info->tgt_type = t;
11484	if (ret)	11476	return 0;
11485	bpf_trampoline_put(tr);	11477	}
		11478
		11479	static int check_attach_btf_id(struct bpf_verifier_env *env)
		11480	{
		11481	struct bpf_prog *prog = env->prog;
		11482	struct bpf_prog *tgt_prog = prog->aux->dst_prog;
		11483	struct bpf_attach_target_info tgt_info = {};
		11484	u32 btf_id = prog->aux->attach_btf_id;
		11485	struct bpf_trampoline *tr;
		11486	int ret;
		11487	u64 key;
		11488
		11489	if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
		11490	prog->type != BPF_PROG_TYPE_LSM) {
		11491	verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
		11492	return -EINVAL;
		11493	}
		11494
		11495	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
		11496	return check_struct_ops_btf_id(env);
		11497
		11498	if (prog->type != BPF_PROG_TYPE_TRACING &&
		11499	prog->type != BPF_PROG_TYPE_LSM &&
		11500	prog->type != BPF_PROG_TYPE_EXT)
		11501	return 0;
		11502
		11503	ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
		11504	if (ret)
11486	return ret;	11505	return ret;
		11506
		11507	if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
		11508	/* to make freplace equivalent to their targets, they need to
		11509	* inherit env->ops and expected_attach_type for the rest of the
		11510	* verification
		11511	*/
		11512	env->ops = bpf_verifier_ops[tgt_prog->type];
		11513	prog->expected_attach_type = tgt_prog->expected_attach_type;
11487	}	11514	}
		11515
		11516	/* store info about the attachment target that will be used later */
		11517	prog->aux->attach_func_proto = tgt_info.tgt_type;
		11518	prog->aux->attach_func_name = tgt_info.tgt_name;
		11519
		11520	if (tgt_prog) {
		11521	prog->aux->saved_dst_prog_type = tgt_prog->type;
		11522	prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
		11523	}
		11524
		11525	if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
		11526	prog->aux->attach_btf_trace = true;
		11527	return 0;
		11528	} else if (prog->expected_attach_type == BPF_TRACE_ITER) {
		11529	if (!bpf_iter_prog_supported(prog))
		11530	return -EINVAL;
		11531	return 0;
		11532	}
		11533
		11534	if (prog->type == BPF_PROG_TYPE_LSM) {
		11535	ret = bpf_lsm_verify_prog(&env->log, prog);
		11536	if (ret < 0)
		11537	return ret;
		11538	}
		11539
		11540	key = bpf_trampoline_compute_key(tgt_prog, btf_id);
		11541	tr = bpf_trampoline_get(key, &tgt_info);
		11542	if (!tr)
		11543	return -ENOMEM;
		11544
		11545	prog->aux->dst_trampoline = tr;
		11546	return 0;
		11547	}
		11548
		11549	struct btf *bpf_get_btf_vmlinux(void)
		11550	{
		11551	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
		11552	mutex_lock(&bpf_verifier_lock);
		11553	if (!btf_vmlinux)
		11554	btf_vmlinux = btf_parse_vmlinux();
		11555	mutex_unlock(&bpf_verifier_lock);
		11556	}
		11557	return btf_vmlinux;
11488	}	11558	}
11489		11559
11490	int bpf_check(struct bpf_prog *prog, union bpf_attr attr,	11560	int bpf_check(struct bpf_prog *prog, union bpf_attr attr,
@@ -11520,12 +11590,7 @@ int bpf_check(struct bpf_prog *prog, union bpf_attr attr, · 29 unmodified lines
11520	env->ops = bpf_verifier_ops[env->prog->type];	11590	env->ops = bpf_verifier_ops[env->prog->type];
11521	is_priv = bpf_capable();	11591	is_priv = bpf_capable();
11522		11592
11523	if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {	11593	bpf_get_btf_vmlinux();
11524	mutex_lock(&bpf_verifier_lock);
11525	if (!btf_vmlinux)
11526	btf_vmlinux = btf_parse_vmlinux();
11527	mutex_unlock(&bpf_verifier_lock);
11528	}
11529		11594
11530	/* grab the mutex to protect few globals used by verifier */	11595	/* grab the mutex to protect few globals used by verifier */
11531	if (!is_priv)	11596	if (!is_priv)

kernel/rcu/tasks.h+41−12

  * @kthread_ptr: This flavor's grace-period/callback-invocation kthread.
  * @gp_func: This flavor's grace-period-wait function.
  * @gp_state: Grace period's most recent state transition (debugging).
+ * @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping.
+ * @init_fract: Initial backoff sleep interval.
  * @gp_jiffies: Time of last @gp_state transition.
  * @gp_start: Most recent grace-period start in jiffies.
  * @n_gps: Number of grace periods completed since boot.
 	struct wait_queue_head cbs_wq;
 	raw_spinlock_t cbs_lock;
 	int gp_state;
+	int gp_sleep;
+	int init_fract;
 	unsigned long gp_jiffies;
 	unsigned long gp_start;
 	unsigned long n_gps;
 DEFINE_STATIC_SRCU(tasks_rcu_exit_srcu);
 /* Avoid IPIing CPUs early in the grace period. */
-#define RCU_TASK_IPI_DELAY (HZ / 2)
+#define RCU_TASK_IPI_DELAY (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) ? HZ / 2 : 0)
 static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY;
 module_param(rcu_task_ipi_delay, int, 0644);
 			cond_resched();
 		}
 		/* Paranoid sleep to keep this from entering a tight loop */
-		schedule_timeout_idle(HZ/10);
+		schedule_timeout_idle(rtp->gp_sleep);
 		set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
 	}
 	 */
 	lastreport = jiffies;
-	/* Start off with HZ/10 wait and slowly back off to 1 HZ wait. */
+	// Start off with initial wait and slowly back off to 1 HZ wait.
-	fract = 10;
+	fract = rtp->init_fract;
+	if (fract > HZ)
+		fract = HZ;
 	for (;;) {
 		bool firstreport;
 static int __init rcu_spawn_tasks_kthread(void)
 {
+	rcu_tasks.gp_sleep = HZ / 10;
+	rcu_tasks.init_fract = 10;
 	rcu_tasks.pregp_func = rcu_tasks_pregp_step;
 	rcu_tasks.pertask_func = rcu_tasks_pertask;
 	rcu_tasks.postscan_func = rcu_tasks_postscan;
 static int __init rcu_spawn_tasks_rude_kthread(void)
 {
+	rcu_tasks_rude.gp_sleep = HZ / 10;
 	rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude);
 	return 0;
 }
 // The number of detections of task quiescent state relying on
 // heavyweight readers executing explicit memory barriers.
-unsigned long n_heavy_reader_attempts;
+static unsigned long n_heavy_reader_attempts;
-unsigned long n_heavy_reader_updates;
+static unsigned long n_heavy_reader_updates;
-unsigned long n_heavy_reader_ofl_updates;
+static unsigned long n_heavy_reader_ofl_updates;
 void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func);
 DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace,
 		WRITE_ONCE(t->trc_reader_checked, true);
 		goto reset_ipi;
 	}
+	// If we are racing with an rcu_read_unlock_trace(), try again later.
+	if (unlikely(t->trc_reader_nesting < 0)) {
+		if (WARN_ON_ONCE(atomic_dec_and_test(&trc_n_readers_need_end)))
+			wake_up(&trc_wait);
+		goto reset_ipi;
+	}
 	WRITE_ONCE(t->trc_reader_checked, true);
 	// Get here if the task is in a read-side critical section.  Set
 	// If currently running, send an IPI, either way, add to list.
 	trc_add_holdout(t, bhp);
-	if (task_curr(t) && time_after(jiffies, rcu_tasks_trace.gp_start + rcu_task_ipi_delay)) {
+	if (task_curr(t) &&
+	    time_after(jiffies + 1, rcu_tasks_trace.gp_start + rcu_task_ipi_delay)) {
 		// The task is currently running, so try IPIing it.
 		cpu = task_cpu(t);
 		if (ret)
 			break;  // Count reached zero.
 		// Stall warning time, so make a list of the offenders.
+		rcu_read_lock();
 		for_each_process_thread(g, t)
 			if (READ_ONCE(t->trc_reader_special.b.need_qs))
 				trc_add_holdout(t, &holdouts);
+		rcu_read_unlock();
 		firstreport = true;
-		list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list)
+		list_for_each_entry_safe(t, g, &holdouts, trc_holdout_list) {
-			if (READ_ONCE(t->trc_reader_special.b.need_qs)) {
+			if (READ_ONCE(t->trc_reader_special.b.need_qs))
 				show_stalled_task_trace(t, &firstreport);
-				trc_del_holdout(t);
+			trc_del_holdout(t); // Release task_struct reference.
-			}
+		}
 		if (firstreport)
 			pr_err("INFO: rcu_tasks_trace detected stalls? (Counter/taskslist mismatch?)\n");
 		show_stalled_ipi_trace();
 static int __init rcu_spawn_tasks_trace_kthread(void)
 {
+	if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) {
+		rcu_tasks_trace.gp_sleep = HZ / 10;
+		rcu_tasks_trace.init_fract = 10;
+	} else {
+		rcu_tasks_trace.gp_sleep = HZ / 200;
+		if (rcu_tasks_trace.gp_sleep <= 0)
+			rcu_tasks_trace.gp_sleep = 1;
+		rcu_tasks_trace.init_fract = HZ / 5;
+		if (rcu_tasks_trace.init_fract <= 0)
+			rcu_tasks_trace.init_fract = 1;
+	}
 	rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step;
 	rcu_tasks_trace.pertask_func = rcu_tasks_trace_pertask;
 	rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan;

kernel/trace/bpf_trace.c+99−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include <linux/slab.h>	7	#include <linux/slab.h>
8	#include <linux/bpf.h>	8	#include <linux/bpf.h>
9	#include <linux/bpf_perf_event.h>	9	#include <linux/bpf_perf_event.h>
		10	#include <linux/btf.h>
10	#include <linux/filter.h>	11	#include <linux/filter.h>
11	#include <linux/uaccess.h>	12	#include <linux/uaccess.h>
12	#include <linux/ctype.h>	13	#include <linux/ctype.h>
@@ -16,6 +17,9 @@ · 3 unmodified lines
16	#include <linux/error-injection.h>	17	#include <linux/error-injection.h>
17	#include <linux/btf_ids.h>	18	#include <linux/btf_ids.h>
18		19
		20	#include <uapi/linux/bpf.h>
		21	#include <uapi/linux/btf.h>
		22
19	#include <asm/tlb.h>	23	#include <asm/tlb.h>
20		24
21	#include "trace_probe.h"	25	#include "trace_probe.h"
@@ -67,6 +71,10 @@ static struct bpf_raw_event_map bpf_get_raw_tracepoint_module(const char name) · 45 unmodified lines
67	u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);	71	u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
68	u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);	72	u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
69		73
		74	static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
		75	u64 flags, const struct btf **btf,
		76	s32 *btf_id);
		77
70	/**	78	/**
71	* trace_call_bpf - invoke BPF program	79	* trace_call_bpf - invoke BPF program
72	* @call: tracepoint event	80	* @call: tracepoint event
@@ -772,6 +780,31 @@ static const struct bpf_func_proto bpf_seq_write_proto = { · 699 unmodified lines
772	.arg3_type = ARG_CONST_SIZE_OR_ZERO,	780	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
773	};	781	};
774		782
		783	BPF_CALL_4(bpf_seq_printf_btf, struct seq_file , m, struct btf_ptr , ptr,
		784	u32, btf_ptr_size, u64, flags)
		785	{
		786	const struct btf *btf;
		787	s32 btf_id;
		788	int ret;
		789
		790	ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id);
		791	if (ret)
		792	return ret;
		793
		794	return btf_type_seq_show_flags(btf, btf_id, ptr->ptr, m, flags);
		795	}
		796
		797	static const struct bpf_func_proto bpf_seq_printf_btf_proto = {
		798	.func = bpf_seq_printf_btf,
		799	.gpl_only = true,
		800	.ret_type = RET_INTEGER,
		801	.arg1_type = ARG_PTR_TO_BTF_ID,
		802	.arg1_btf_id = &btf_seq_file_ids[0],
		803	.arg2_type = ARG_PTR_TO_MEM,
		804	.arg3_type = ARG_CONST_SIZE_OR_ZERO,
		805	.arg4_type = ARG_ANYTHING,
		806	};
		807
775	static __always_inline int	808	static __always_inline int
776	get_map_perf_counter(struct bpf_map *map, u64 flags,	809	get_map_perf_counter(struct bpf_map *map, u64 flags,
777	u64 value, u64 enabled, u64 *running)	810	u64 value, u64 enabled, u64 *running)
@@ -1147,6 +1180,65 @@ static const struct bpf_func_proto bpf_d_path_proto = { · 369 unmodified lines
1147	.allowed = bpf_d_path_allowed,	1180	.allowed = bpf_d_path_allowed,
1148	};	1181	};
1149		1182
		1183	#define BTF_F_ALL (BTF_F_COMPACT \| BTF_F_NONAME \| \
		1184	BTF_F_PTR_RAW \| BTF_F_ZERO)
		1185
		1186	static int bpf_btf_printf_prepare(struct btf_ptr *ptr, u32 btf_ptr_size,
		1187	u64 flags, const struct btf **btf,
		1188	s32 *btf_id)
		1189	{
		1190	const struct btf_type *t;
		1191
		1192	if (unlikely(flags & ~(BTF_F_ALL)))
		1193	return -EINVAL;
		1194
		1195	if (btf_ptr_size != sizeof(struct btf_ptr))
		1196	return -EINVAL;
		1197
		1198	*btf = bpf_get_btf_vmlinux();
		1199
		1200	if (IS_ERR_OR_NULL(*btf))
		1201	return PTR_ERR(*btf);
		1202
		1203	if (ptr->type_id > 0)
		1204	*btf_id = ptr->type_id;
		1205	else
		1206	return -EINVAL;
		1207
		1208	if (*btf_id > 0)
		1209	t = btf_type_by_id(btf, btf_id);
		1210	if (*btf_id <= 0 \|\| !t)
		1211	return -ENOENT;
		1212
		1213	return 0;
		1214	}
		1215
		1216	BPF_CALL_5(bpf_snprintf_btf, char , str, u32, str_size, struct btf_ptr , ptr,
		1217	u32, btf_ptr_size, u64, flags)
		1218	{
		1219	const struct btf *btf;
		1220	s32 btf_id;
		1221	int ret;
		1222
		1223	ret = bpf_btf_printf_prepare(ptr, btf_ptr_size, flags, &btf, &btf_id);
		1224	if (ret)
		1225	return ret;
		1226
		1227	return btf_type_snprintf_show(btf, btf_id, ptr->ptr, str, str_size,
		1228	flags);
		1229	}
		1230
		1231	const struct bpf_func_proto bpf_snprintf_btf_proto = {
		1232	.func = bpf_snprintf_btf,
		1233	.gpl_only = false,
		1234	.ret_type = RET_INTEGER,
		1235	.arg1_type = ARG_PTR_TO_MEM,
		1236	.arg2_type = ARG_CONST_SIZE,
		1237	.arg3_type = ARG_PTR_TO_MEM,
		1238	.arg4_type = ARG_CONST_SIZE,
		1239	.arg5_type = ARG_ANYTHING,
		1240	};
		1241
1150	const struct bpf_func_proto *	1242	const struct bpf_func_proto *
1151	bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)	1243	bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
1152	{	1244	{
@@ -1233,6 +1325,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 80 unmodified lines
1233	return &bpf_get_task_stack_proto;	1325	return &bpf_get_task_stack_proto;
1234	case BPF_FUNC_copy_from_user:	1326	case BPF_FUNC_copy_from_user:
1235	return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;	1327	return prog->aux->sleepable ? &bpf_copy_from_user_proto : NULL;
		1328	case BPF_FUNC_snprintf_btf:
		1329	return &bpf_snprintf_btf_proto;
1236	default:	1330	default:
1237	return NULL;	1331	return NULL;
1238	}	1332	}
@@ -1630,6 +1724,10 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 391 unmodified lines
1630	return prog->expected_attach_type == BPF_TRACE_ITER ?	1724	return prog->expected_attach_type == BPF_TRACE_ITER ?
1631	&bpf_seq_write_proto :	1725	&bpf_seq_write_proto :
1632	NULL;	1726	NULL;
		1727	case BPF_FUNC_seq_printf_btf:
		1728	return prog->expected_attach_type == BPF_TRACE_ITER ?
		1729	&bpf_seq_printf_btf_proto :
		1730	NULL;
1633	case BPF_FUNC_d_path:	1731	case BPF_FUNC_d_path:
1634	return &bpf_d_path_proto;	1732	return &bpf_d_path_proto;
1635	default:	1733	default:
@@ -1678,6 +1776,7 @@ const struct bpf_verifier_ops raw_tracepoint_verifier_ops = { · 42 unmodified lines
1678	};	1776	};
1679		1777
1680	const struct bpf_prog_ops raw_tracepoint_prog_ops = {	1778	const struct bpf_prog_ops raw_tracepoint_prog_ops = {
		1779	.test_run = bpf_prog_test_run_raw_tp,
1681	};	1780	};
1682		1781
1683	const struct bpf_verifier_ops tracing_verifier_ops = {	1782	const struct bpf_verifier_ops tracing_verifier_ops = {

net/bpf/test_run.c+88−0

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	#include <net/sock.h>	11	#include <net/sock.h>
12	#include <net/tcp.h>	12	#include <net/tcp.h>
13	#include <linux/error-injection.h>	13	#include <linux/error-injection.h>
		14	#include <linux/smp.h>
14		15
15	#define CREATE_TRACE_POINTS	16	#define CREATE_TRACE_POINTS
16	#include <trace/events/bpf_test_run.h>	17	#include <trace/events/bpf_test_run.h>
@@ -204,6 +205,9 @@ int bpf_prog_test_run_tracing(struct bpf_prog *prog, · 187 unmodified lines
204	int b = 2, err = -EFAULT;	205	int b = 2, err = -EFAULT;
205	u32 retval = 0;	206	u32 retval = 0;
206		207
		208	if (kattr->test.flags \|\| kattr->test.cpu)
		209	return -EINVAL;
		210
207	switch (prog->expected_attach_type) {	211	switch (prog->expected_attach_type) {
208	case BPF_TRACE_FENTRY:	212	case BPF_TRACE_FENTRY:
209	case BPF_TRACE_FEXIT:	213	case BPF_TRACE_FEXIT:
@@ -236,6 +240,84 @@ out: · 26 unmodified lines
236	return err;	240	return err;
237	}	241	}
238		242
		243	struct bpf_raw_tp_test_run_info {
		244	struct bpf_prog *prog;
		245	void *ctx;
		246	u32 retval;
		247	};
		248
		249	static void
		250	__bpf_prog_test_run_raw_tp(void *data)
		251	{
		252	struct bpf_raw_tp_test_run_info *info = data;
		253
		254	rcu_read_lock();
		255	info->retval = BPF_PROG_RUN(info->prog, info->ctx);
		256	rcu_read_unlock();
		257	}
		258
		259	int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
		260	const union bpf_attr *kattr,
		261	union bpf_attr __user *uattr)
		262	{
		263	void __user *ctx_in = u64_to_user_ptr(kattr->test.ctx_in);
		264	__u32 ctx_size_in = kattr->test.ctx_size_in;
		265	struct bpf_raw_tp_test_run_info info;
		266	int cpu = kattr->test.cpu, err = 0;
		267	int current_cpu;
		268
		269	/* doesn't support data_in/out, ctx_out, duration, or repeat */
		270	if (kattr->test.data_in \|\| kattr->test.data_out \|\|
		271	kattr->test.ctx_out \|\| kattr->test.duration \|\|
		272	kattr->test.repeat)
		273	return -EINVAL;
		274
		275	if (ctx_size_in < prog->aux->max_ctx_offset)
		276	return -EINVAL;
		277
		278	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
		279	return -EINVAL;
		280
		281	if (ctx_size_in) {
		282	info.ctx = kzalloc(ctx_size_in, GFP_USER);
		283	if (!info.ctx)
		284	return -ENOMEM;
		285	if (copy_from_user(info.ctx, ctx_in, ctx_size_in)) {
		286	err = -EFAULT;
		287	goto out;
		288	}
		289	} else {
		290	info.ctx = NULL;
		291	}
		292
		293	info.prog = prog;
		294
		295	current_cpu = get_cpu();
		296	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 \|\|
		297	cpu == current_cpu) {
		298	__bpf_prog_test_run_raw_tp(&info);
		299	} else if (cpu >= nr_cpu_ids \|\| !cpu_online(cpu)) {
		300	/* smp_call_function_single() also checks cpu_online()
		301	* after csd_lock(). However, since cpu is from user
		302	* space, let's do an extra quick check to filter out
		303	* invalid value before smp_call_function_single().
		304	*/
		305	err = -ENXIO;
		306	} else {
		307	err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
		308	&info, 1);
		309	}
		310	put_cpu();
		311
		312	if (!err &&
		313	copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
		314	err = -EFAULT;
		315
		316	out:
		317	kfree(info.ctx);
		318	return err;
		319	}
		320
239	static void bpf_ctx_init(const union bpf_attr kattr, u32 max_size)	321	static void bpf_ctx_init(const union bpf_attr kattr, u32 max_size)
240	{	322	{
241	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);	323	void __user *data_in = u64_to_user_ptr(kattr->test.ctx_in);
@@ -410,6 +492,9 @@ int bpf_prog_test_run_skb(struct bpf_prog prog, const union bpf_attr kattr, · 168 unmodified lines
410	void *data;	492	void *data;
411	int ret;	493	int ret;
412		494
		495	if (kattr->test.flags \|\| kattr->test.cpu)
		496	return -EINVAL;
		497
413	data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,	498	data = bpf_test_init(kattr, size, NET_SKB_PAD + NET_IP_ALIGN,
414	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));	499	SKB_DATA_ALIGN(sizeof(struct skb_shared_info)));
415	if (IS_ERR(data))	500	if (IS_ERR(data))
@@ -607,6 +692,9 @@ int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, · 191 unmodified lines
607	if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)	692	if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
608	return -EINVAL;	693	return -EINVAL;
609		694
		695	if (kattr->test.flags \|\| kattr->test.cpu)
		696	return -EINVAL;
		697
610	if (size < ETH_HLEN)	698	if (size < ETH_HLEN)
611	return -EINVAL;	699	return -EINVAL;
612		700

net/core/bpf_sk_storage.c+6−23

 {
 	struct bpf_local_storage_data *sdata;
-	if (flags > BPF_SK_STORAGE_GET_F_CREATE)
+	if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
 		return (unsigned long)NULL;
 	sdata = sk_storage_lookup(sk, map, true);
 BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
 {
+	if (!sk || !sk_fullsock(sk))
+		return -EINVAL;
 	if (refcount_inc_not_zero(&sk->sk_refcnt)) {
 		int err;
 	.gpl_only	= false,
 	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
 	.arg1_type	= ARG_CONST_MAP_PTR,
-	.arg2_type	= ARG_PTR_TO_SOCKET,
+	.arg2_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
 	.arg4_type	= ARG_ANYTHING,
 };
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_CONST_MAP_PTR,
-	.arg2_type	= ARG_PTR_TO_SOCKET,
+	.arg2_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
-};
-const struct bpf_func_proto sk_storage_get_btf_proto = {
-	.func		= bpf_sk_storage_get,
-	.gpl_only	= false,
-	.ret_type	= RET_PTR_TO_MAP_VALUE_OR_NULL,
-	.arg1_type	= ARG_CONST_MAP_PTR,
-	.arg2_type	= ARG_PTR_TO_BTF_ID,
-	.arg2_btf_id	= &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
-	.arg3_type	= ARG_PTR_TO_MAP_VALUE_OR_NULL,
-	.arg4_type	= ARG_ANYTHING,
-};
-const struct bpf_func_proto sk_storage_delete_btf_proto = {
-	.func		= bpf_sk_storage_delete,
-	.gpl_only	= false,
-	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_CONST_MAP_PTR,
-	.arg2_type	= ARG_PTR_TO_BTF_ID,
-	.arg2_btf_id	= &btf_sock_ids[BTF_SOCK_TYPE_SOCK],
 };
 struct bpf_sk_storage_diag {

net/core/filter.c+356−62

 #include <net/transp_v6.h>
 #include <linux/btf_ids.h>
+static const struct bpf_func_proto *
+bpf_sk_base_func_proto(enum bpf_func_id func_id);
 int copy_bpf_fprog_from_user(struct sock_fprog *dst, sockptr_t src, int len)
 {
 	if (in_compat_syscall()) {
 		return __bpf_redirect_no_mac(skb, dev, flags);
 }
+#if IS_ENABLED(CONFIG_IPV6)
+static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb)
+{
+	struct dst_entry *dst = skb_dst(skb);
+	struct net_device *dev = dst->dev;
+	u32 hh_len = LL_RESERVED_SPACE(dev);
+	const struct in6_addr *nexthop;
+	struct neighbour *neigh;
+	if (dev_xmit_recursion()) {
+		net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
+		goto out_drop;
+	}
+	skb->dev = dev;
+	skb->tstamp = 0;
+	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
+		struct sk_buff *skb2;
+		skb2 = skb_realloc_headroom(skb, hh_len);
+		if (unlikely(!skb2)) {
+			kfree_skb(skb);
+			return -ENOMEM;
+		}
+		if (skb->sk)
+			skb_set_owner_w(skb2, skb->sk);
+		consume_skb(skb);
+		skb = skb2;
+	}
+	rcu_read_lock_bh();
+	nexthop = rt6_nexthop(container_of(dst, struct rt6_info, dst),
+			      &ipv6_hdr(skb)->daddr);
+	neigh = ip_neigh_gw6(dev, nexthop);
+	if (likely(!IS_ERR(neigh))) {
+		int ret;
+		sock_confirm_neigh(skb, neigh);
+		dev_xmit_recursion_inc();
+		ret = neigh_output(neigh, skb, false);
+		dev_xmit_recursion_dec();
+		rcu_read_unlock_bh();
+		return ret;
+	}
+	rcu_read_unlock_bh();
+	IP6_INC_STATS(dev_net(dst->dev),
+		      ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
+out_drop:
+	kfree_skb(skb);
+	return -ENETDOWN;
+}
+static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev)
+{
+	const struct ipv6hdr *ip6h = ipv6_hdr(skb);
+	struct net *net = dev_net(dev);
+	int err, ret = NET_XMIT_DROP;
+	struct dst_entry *dst;
+	struct flowi6 fl6 = {
+		.flowi6_flags	= FLOWI_FLAG_ANYSRC,
+		.flowi6_mark	= skb->mark,
+		.flowlabel	= ip6_flowinfo(ip6h),
+		.flowi6_oif	= dev->ifindex,
+		.flowi6_proto	= ip6h->nexthdr,
+		.daddr		= ip6h->daddr,
+		.saddr		= ip6h->saddr,
+	};
+	dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &fl6, NULL);
+	if (IS_ERR(dst))
+		goto out_drop;
+	skb_dst_set(skb, dst);
+	err = bpf_out_neigh_v6(net, skb);
+	if (unlikely(net_xmit_eval(err)))
+		dev->stats.tx_errors++;
+	else
+		ret = NET_XMIT_SUCCESS;
+	goto out_xmit;
+out_drop:
+	dev->stats.tx_errors++;
+	kfree_skb(skb);
+out_xmit:
+	return ret;
+}
+#else
+static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev)
+{
+	kfree_skb(skb);
+	return NET_XMIT_DROP;
+}
+#endif /* CONFIG_IPV6 */
+#if IS_ENABLED(CONFIG_INET)
+static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb)
+{
+	struct dst_entry *dst = skb_dst(skb);
+	struct rtable *rt = container_of(dst, struct rtable, dst);
+	struct net_device *dev = dst->dev;
+	u32 hh_len = LL_RESERVED_SPACE(dev);
+	struct neighbour *neigh;
+	bool is_v6gw = false;
+	if (dev_xmit_recursion()) {
+		net_crit_ratelimited("bpf: recursion limit reached on datapath, buggy bpf program?\n");
+		goto out_drop;
+	}
+	skb->dev = dev;
+	skb->tstamp = 0;
+	if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) {
+		struct sk_buff *skb2;
+		skb2 = skb_realloc_headroom(skb, hh_len);
+		if (unlikely(!skb2)) {
+			kfree_skb(skb);
+			return -ENOMEM;
+		}
+		if (skb->sk)
+			skb_set_owner_w(skb2, skb->sk);
+		consume_skb(skb);
+		skb = skb2;
+	}
+	rcu_read_lock_bh();
+	neigh = ip_neigh_for_gw(rt, skb, &is_v6gw);
+	if (likely(!IS_ERR(neigh))) {
+		int ret;
+		sock_confirm_neigh(skb, neigh);
+		dev_xmit_recursion_inc();
+		ret = neigh_output(neigh, skb, is_v6gw);
+		dev_xmit_recursion_dec();
+		rcu_read_unlock_bh();
+		return ret;
+	}
+	rcu_read_unlock_bh();
+out_drop:
+	kfree_skb(skb);
+	return -ENETDOWN;
+}
+static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev)
+{
+	const struct iphdr *ip4h = ip_hdr(skb);
+	struct net *net = dev_net(dev);
+	int err, ret = NET_XMIT_DROP;
+	struct rtable *rt;
+	struct flowi4 fl4 = {
+		.flowi4_flags	= FLOWI_FLAG_ANYSRC,
+		.flowi4_mark	= skb->mark,
+		.flowi4_tos	= RT_TOS(ip4h->tos),
+		.flowi4_oif	= dev->ifindex,
+		.flowi4_proto	= ip4h->protocol,
+		.daddr		= ip4h->daddr,
+		.saddr		= ip4h->saddr,
+	};
+	rt = ip_route_output_flow(net, &fl4, NULL);
+	if (IS_ERR(rt))
+		goto out_drop;
+	if (rt->rt_type != RTN_UNICAST && rt->rt_type != RTN_LOCAL) {
+		ip_rt_put(rt);
+		goto out_drop;
+	}
+	skb_dst_set(skb, &rt->dst);
+	err = bpf_out_neigh_v4(net, skb);
+	if (unlikely(net_xmit_eval(err)))
+		dev->stats.tx_errors++;
+	else
+		ret = NET_XMIT_SUCCESS;
+	goto out_xmit;
+out_drop:
+	dev->stats.tx_errors++;
+	kfree_skb(skb);
+out_xmit:
+	return ret;
+}
+#else
+static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev)
+{
+	kfree_skb(skb);
+	return NET_XMIT_DROP;
+}
+#endif /* CONFIG_INET */
+static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev)
+{
+	struct ethhdr *ethh = eth_hdr(skb);
+	if (unlikely(skb->mac_header >= skb->network_header))
+		goto out;
+	bpf_push_mac_rcsum(skb);
+	if (is_multicast_ether_addr(ethh->h_dest))
+		goto out;
+	skb_pull(skb, sizeof(*ethh));
+	skb_unset_mac_header(skb);
+	skb_reset_network_header(skb);
+	if (skb->protocol == htons(ETH_P_IP))
+		return __bpf_redirect_neigh_v4(skb, dev);
+	else if (skb->protocol == htons(ETH_P_IPV6))
+		return __bpf_redirect_neigh_v6(skb, dev);
+out:
+	kfree_skb(skb);
+	return -ENOTSUPP;
+}
+/* Internal, non-exposed redirect flags. */
+enum {
+	BPF_F_NEIGH = (1ULL << 1),
+#define BPF_F_REDIRECT_INTERNAL	(BPF_F_NEIGH)
+};
 BPF_CALL_3(bpf_clone_redirect, struct sk_buff *, skb, u32, ifindex, u64, flags)
 {
 	struct net_device *dev;
 	struct sk_buff *clone;
 	int ret;
-	if (unlikely(flags & ~(BPF_F_INGRESS)))
+	if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
 		return -EINVAL;
 	dev = dev_get_by_index_rcu(dev_net(skb->dev), ifindex);
 DEFINE_PER_CPU(struct bpf_redirect_info, bpf_redirect_info);
 EXPORT_PER_CPU_SYMBOL_GPL(bpf_redirect_info);
-BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
-{
-	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
-	if (unlikely(flags & ~(BPF_F_INGRESS)))
-		return TC_ACT_SHOT;
-	ri->flags = flags;
-	ri->tgt_index = ifindex;
-	return TC_ACT_REDIRECT;
-}
 int skb_do_redirect(struct sk_buff *skb)
 {
 	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
 	struct net_device *dev;
+	u32 flags = ri->flags;
 	dev = dev_get_by_index_rcu(dev_net(skb->dev), ri->tgt_index);
 	ri->tgt_index = 0;
 		return -EINVAL;
 	}
-	return __bpf_redirect(skb, dev, ri->flags);
+	return flags & BPF_F_NEIGH ?
+	       __bpf_redirect_neigh(skb, dev) :
+	       __bpf_redirect(skb, dev, flags);
+}
+BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags)
+{
+	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
+	if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))
+		return TC_ACT_SHOT;
+	ri->flags = flags;
+	ri->tgt_index = ifindex;
+	return TC_ACT_REDIRECT;
 }
 static const struct bpf_func_proto bpf_redirect_proto = {
 	.arg2_type      = ARG_ANYTHING,
 };
+BPF_CALL_2(bpf_redirect_neigh, u32, ifindex, u64, flags)
+{
+	struct bpf_redirect_info *ri = this_cpu_ptr(&bpf_redirect_info);
+	if (unlikely(flags))
+		return TC_ACT_SHOT;
+	ri->flags = BPF_F_NEIGH;
+	ri->tgt_index = ifindex;
+	return TC_ACT_REDIRECT;
+}
+static const struct bpf_func_proto bpf_redirect_neigh_proto = {
+	.func		= bpf_redirect_neigh,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_ANYTHING,
+	.arg2_type	= ARG_ANYTHING,
+};
 BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg *, msg, u32, bytes)
 {
 	msg->apply_bytes = bytes;
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
 };
+BPF_CALL_1(bpf_skb_cgroup_classid, const struct sk_buff *, skb)
+{
+	struct sock *sk = skb_to_full_sk(skb);
+	if (!sk || !sk_fullsock(sk))
+		return 0;
+	return sock_cgroup_classid(&sk->sk_cgrp_data);
+}
+static const struct bpf_func_proto bpf_skb_cgroup_classid_proto = {
+	.func		= bpf_skb_cgroup_classid,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_PTR_TO_CTX,
+};
 #endif
 BPF_CALL_1(bpf_get_cgroup_classid, const struct sk_buff *, skb)
 {
 	struct cgroup *cgrp;
+	sk = sk_to_full_sk(sk);
+	if (!sk || !sk_fullsock(sk))
+		return 0;
 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
 	return cgroup_id(cgrp);
 }
 BPF_CALL_1(bpf_skb_cgroup_id, const struct sk_buff *, skb)
 {
-	struct sock *sk = skb_to_full_sk(skb);
+	return __bpf_sk_cgroup_id(skb->sk);
-	if (!sk || !sk_fullsock(sk))
-		return 0;
-	return __bpf_sk_cgroup_id(sk);
 }
 static const struct bpf_func_proto bpf_skb_cgroup_id_proto = {
 	struct cgroup *ancestor;
 	struct cgroup *cgrp;
+	sk = sk_to_full_sk(sk);
+	if (!sk || !sk_fullsock(sk))
+		return 0;
 	cgrp = sock_cgroup_ptr(&sk->sk_cgrp_data);
 	ancestor = cgroup_ancestor(cgrp, ancestor_level);
 	if (!ancestor)
 BPF_CALL_2(bpf_skb_ancestor_cgroup_id, const struct sk_buff *, skb, int,
 	   ancestor_level)
 {
-	struct sock *sk = skb_to_full_sk(skb);
+	return __bpf_sk_ancestor_cgroup_id(skb->sk, ancestor_level);
-	if (!sk || !sk_fullsock(sk))
-		return 0;
-	return __bpf_sk_ancestor_cgroup_id(sk, ancestor_level);
 }
 static const struct bpf_func_proto bpf_skb_ancestor_cgroup_id_proto = {
 	.func           = bpf_sk_cgroup_id,
 	.gpl_only       = false,
 	.ret_type       = RET_INTEGER,
-	.arg1_type      = ARG_PTR_TO_SOCKET,
+	.arg1_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 };
 BPF_CALL_2(bpf_sk_ancestor_cgroup_id, struct sock *, sk, int, ancestor_level)
 	.func           = bpf_sk_ancestor_cgroup_id,
 	.gpl_only       = false,
 	.ret_type       = RET_INTEGER,
-	.arg1_type      = ARG_PTR_TO_SOCKET,
+	.arg1_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 	.arg2_type      = ARG_ANYTHING,
 };
 #endif
 BPF_CALL_1(bpf_get_socket_cookie, struct sk_buff *, skb)
 {
-	return skb->sk ? sock_gen_cookie(skb->sk) : 0;
+	return skb->sk ? __sock_gen_cookie(skb->sk) : 0;
 }
 static const struct bpf_func_proto bpf_get_socket_cookie_proto = {
 BPF_CALL_1(bpf_get_socket_cookie_sock_addr, struct bpf_sock_addr_kern *, ctx)
 {
-	return sock_gen_cookie(ctx->sk);
+	return __sock_gen_cookie(ctx->sk);
 }
 static const struct bpf_func_proto bpf_get_socket_cookie_sock_addr_proto = {
 BPF_CALL_1(bpf_get_socket_cookie_sock, struct sock *, ctx)
 {
-	return sock_gen_cookie(ctx);
+	return __sock_gen_cookie(ctx);
 }
 static const struct bpf_func_proto bpf_get_socket_cookie_sock_proto = {
 BPF_CALL_1(bpf_get_socket_cookie_sock_ops, struct bpf_sock_ops_kern *, ctx)
 {
-	return sock_gen_cookie(ctx->sk);
+	return __sock_gen_cookie(ctx->sk);
 }
 static const struct bpf_func_proto bpf_get_socket_cookie_sock_ops_proto = {
 static u64 __bpf_get_netns_cookie(struct sock *sk)
 {
 #ifdef CONFIG_NET_NS
-	return net_gen_cookie(sk ? sk->sk_net.net : &init_net);
+	return __net_gen_cookie(sk ? sk->sk_net.net : &init_net);
 #else
 	return 0;
 #endif
 BPF_CALL_1(bpf_sk_release, struct sock *, sk)
 {
-	if (sk_is_refcounted(sk))
+	if (sk && sk_is_refcounted(sk))
 		sock_gen_put(sk);
 	return 0;
 }
 	.func		= bpf_sk_release,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_SOCK_COMMON,
+	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 };
 BPF_CALL_5(bpf_xdp_sk_lookup_udp, struct xdp_buff *, ctx,
 	u32 cookie;
 	int ret;
-	if (unlikely(th_len < sizeof(*th)))
+	if (unlikely(!sk || th_len < sizeof(*th)))
 		return -EINVAL;
 	/* sk_listener() allows TCP_NEW_SYN_RECV, which makes no sense here. */
 	.gpl_only	= true,
 	.pkt_access	= true,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_SOCK_COMMON,
+	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 	.arg2_type	= ARG_PTR_TO_MEM,
 	.arg3_type	= ARG_CONST_SIZE,
 	.arg4_type	= ARG_PTR_TO_MEM,
 	u32 cookie;
 	u16 mss;
-	if (unlikely(th_len < sizeof(*th) || th_len != th->doff * 4))
+	if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4))
 		return -EINVAL;
 	if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN)
 	.gpl_only	= true, /* __cookie_v*_init_sequence() is GPL */
 	.pkt_access	= true,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_SOCK_COMMON,
+	.arg1_type	= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 	.arg2_type	= ARG_PTR_TO_MEM,
 	.arg3_type	= ARG_CONST_SIZE,
 	.arg4_type	= ARG_PTR_TO_MEM,
 BPF_CALL_3(bpf_sk_assign, struct sk_buff *, skb, struct sock *, sk, u64, flags)
 {
-	if (flags != 0)
+	if (!sk || flags != 0)
 		return -EINVAL;
 	if (!skb_at_tc_ingress(skb))
 		return -EOPNOTSUPP;
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
 	.arg1_type      = ARG_PTR_TO_CTX,
-	.arg2_type      = ARG_PTR_TO_SOCK_COMMON,
+	.arg2_type      = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
 	.arg3_type	= ARG_ANYTHING,
 };
 			return NULL;
 		}
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 	case BPF_FUNC_perf_event_output:
 		return &bpf_skb_event_output_proto;
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 		return bpf_get_skb_set_tunnel_proto(func_id);
 	case BPF_FUNC_redirect:
 		return &bpf_redirect_proto;
+	case BPF_FUNC_redirect_neigh:
+		return &bpf_redirect_neigh_proto;
 	case BPF_FUNC_get_route_realm:
 		return &bpf_get_route_realm_proto;
 	case BPF_FUNC_get_hash_recalc:
 	case BPF_FUNC_skb_get_xfrm_state:
 		return &bpf_skb_get_xfrm_state_proto;
 #endif
+#ifdef CONFIG_CGROUP_NET_CLASSID
+	case BPF_FUNC_skb_cgroup_classid:
+		return &bpf_skb_cgroup_classid_proto;
+#endif
 #ifdef CONFIG_SOCK_CGROUP_DATA
 	case BPF_FUNC_skb_cgroup_id:
 		return &bpf_skb_cgroup_id_proto;
 		return &bpf_sk_assign_proto;
 #endif
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 		return &bpf_tcp_gen_syncookie_proto;
 #endif
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 		return &bpf_tcp_sock_proto;
 #endif /* CONFIG_INET */
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 		return &bpf_get_cgroup_classid_curr_proto;
 #endif
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 		return &bpf_skc_lookup_tcp_proto;
 #endif
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 	case BPF_FUNC_skb_load_bytes:
 		return &bpf_flow_dissector_load_bytes_proto;
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 	case BPF_FUNC_skb_under_cgroup:
 		return &bpf_skb_under_cgroup_proto;
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 	case BPF_FUNC_sk_release:
 		return &bpf_sk_release_proto;
 	default:
-		return bpf_base_func_proto(func_id);
+		return bpf_sk_base_func_proto(func_id);
 	}
 }
 	.func			= bpf_skc_to_tcp6_sock,
 	.gpl_only		= false,
 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
-	.arg1_type		= ARG_PTR_TO_BTF_ID,
+	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
-	.arg1_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP6],
 };
 	.func			= bpf_skc_to_tcp_sock,
 	.gpl_only		= false,
 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
-	.arg1_type		= ARG_PTR_TO_BTF_ID,
+	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
-	.arg1_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP],
 };
 	.func			= bpf_skc_to_tcp_timewait_sock,
 	.gpl_only		= false,
 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
-	.arg1_type		= ARG_PTR_TO_BTF_ID,
+	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
-	.arg1_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP_TW],
 };
 	.func			= bpf_skc_to_tcp_request_sock,
 	.gpl_only		= false,
 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
-	.arg1_type		= ARG_PTR_TO_BTF_ID,
+	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
-	.arg1_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_TCP_REQ],
 };
 	.func			= bpf_skc_to_udp6_sock,
 	.gpl_only		= false,
 	.ret_type		= RET_PTR_TO_BTF_ID_OR_NULL,
-	.arg1_type		= ARG_PTR_TO_BTF_ID,
+	.arg1_type		= ARG_PTR_TO_BTF_ID_SOCK_COMMON,
-	.arg1_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
 	.ret_btf_id		= &btf_sock_ids[BTF_SOCK_TYPE_UDP6],
 };
+static const struct bpf_func_proto *
+bpf_sk_base_func_proto(enum bpf_func_id func_id)
+{
+	const struct bpf_func_proto *func;
+	switch (func_id) {
+	case BPF_FUNC_skc_to_tcp6_sock:
+		func = &bpf_skc_to_tcp6_sock_proto;
+		break;
+	case BPF_FUNC_skc_to_tcp_sock:
+		func = &bpf_skc_to_tcp_sock_proto;
+		break;
+	case BPF_FUNC_skc_to_tcp_timewait_sock:
+		func = &bpf_skc_to_tcp_timewait_sock_proto;
+		break;
+	case BPF_FUNC_skc_to_tcp_request_sock:
+		func = &bpf_skc_to_tcp_request_sock_proto;
+		break;
+	case BPF_FUNC_skc_to_udp6_sock:
+		func = &bpf_skc_to_udp6_sock_proto;
+		break;
+	default:
+		return bpf_base_func_proto(func_id);
+	}
+	if (!perfmon_capable())
+		return NULL;
+	return func;
+}

net/core/net_namespace.c+8−4

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	#include <linux/net_namespace.h>	19	#include <linux/net_namespace.h>
20	#include <linux/sched/task.h>	20	#include <linux/sched/task.h>
21	#include <linux/uidgid.h>	21	#include <linux/uidgid.h>
		22	#include <linux/cookie.h>
22		23
23	#include <net/sock.h>	24	#include <net/sock.h>
24	#include <net/netlink.h>	25	#include <net/netlink.h>
@@ -69,16 +70,16 @@ EXPORT_SYMBOL_GPL(pernet_ops_rwsem); · 44 unmodified lines
69		70
70	static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS;	71	static unsigned int max_gen_ptrs = INITIAL_NET_GEN_PTRS;
71		72
72	static atomic64_t cookie_gen;	73	DEFINE_COOKIE(net_cookie);
73		74
74	u64 net_gen_cookie(struct net *net)	75	u64 __net_gen_cookie(struct net *net)
75	{	76	{
76	while (1) {	77	while (1) {
77	u64 res = atomic64_read(&net->net_cookie);	78	u64 res = atomic64_read(&net->net_cookie);
78		79
79	if (res)	80	if (res)
80	return res;	81	return res;
81	res = atomic64_inc_return(&cookie_gen);	82	res = gen_cookie_next(&net_cookie);
82	atomic64_cmpxchg(&net->net_cookie, 0, res);	83	atomic64_cmpxchg(&net->net_cookie, 0, res);
83	}	84	}
84	}	85	}
@@ -1101,7 +1102,10 @@ static int __init net_ns_init(void) · 1016 unmodified lines
1101	panic("Could not allocate generic netns");	1102	panic("Could not allocate generic netns");
1102		1103
1103	rcu_assign_pointer(init_net.gen, ng);	1104	rcu_assign_pointer(init_net.gen, ng);
1104	net_gen_cookie(&init_net);	1105
		1106	preempt_disable();
		1107	__net_gen_cookie(&init_net);
		1108	preempt_enable();
1105		1109
1106	down_write(&pernet_ops_rwsem);	1110	down_write(&pernet_ops_rwsem);
1107	if (setup_net(&init_net, &init_user_ns))	1111	if (setup_net(&init_net, &init_user_ns))

net/core/sock_diag.c+5−4

@@ -11,7 +11,7 @@ · 10 unmodified lines
11	#include <linux/tcp.h>	11	#include <linux/tcp.h>
12	#include <linux/workqueue.h>	12	#include <linux/workqueue.h>
13	#include <linux/nospec.h>	13	#include <linux/nospec.h>
14		14	#include <linux/cookie.h>
15	#include <linux/inet_diag.h>	15	#include <linux/inet_diag.h>
16	#include <linux/sock_diag.h>	16	#include <linux/sock_diag.h>
17		17
@@ -19,16 +19,17 @@ static const struct sock_diag_handler *sock_diag_handlers[AF_MAX]; · 1 unmodified line
19	static int (inet_rcv_compat)(struct sk_buff skb, struct nlmsghdr *nlh);	19	static int (inet_rcv_compat)(struct sk_buff skb, struct nlmsghdr *nlh);
20	static DEFINE_MUTEX(sock_diag_table_mutex);	20	static DEFINE_MUTEX(sock_diag_table_mutex);
21	static struct workqueue_struct *broadcast_wq;	21	static struct workqueue_struct *broadcast_wq;
22	static atomic64_t cookie_gen;
23		22
24	u64 sock_gen_cookie(struct sock *sk)	23	DEFINE_COOKIE(sock_cookie);
		24
		25	u64 __sock_gen_cookie(struct sock *sk)
25	{	26	{
26	while (1) {	27	while (1) {
27	u64 res = atomic64_read(&sk->sk_cookie);	28	u64 res = atomic64_read(&sk->sk_cookie);
28		29
29	if (res)	30	if (res)
30	return res;	31	return res;
31	res = atomic64_inc_return(&cookie_gen);	32	res = gen_cookie_next(&sock_cookie);
32	atomic64_cmpxchg(&sk->sk_cookie, 0, res);	33	atomic64_cmpxchg(&sk->sk_cookie, 0, res);
33	}	34	}
34	}	35	}

net/core/sock_map.c+5−2

 	if (!sk)
 		return ERR_PTR(-ENOENT);
-	sock_gen_cookie(sk);
+	__sock_gen_cookie(sk);
 	return &sk->sk_cookie;
 }
 	struct sock *sk = (struct sock *)value;
 	int ret;
+	if (unlikely(!sk || !sk_fullsock(sk)))
+		return -EINVAL;
 	if (!sock_map_sk_is_suitable(sk))
 		return -EOPNOTSUPP;
 	if (!sk)
 		return ERR_PTR(-ENOENT);
-	sock_gen_cookie(sk);
+	__sock_gen_cookie(sk);
 	return &sk->sk_cookie;
 }

net/ipv4/bpf_tcp_ca.c+2−21

 static const struct btf_type *tcp_sock_type;
 static u32 tcp_sock_id, sock_id;
-static struct bpf_func_proto btf_sk_storage_get_proto __read_mostly;
-static struct bpf_func_proto btf_sk_storage_delete_proto __read_mostly;
-static void convert_sk_func_proto(struct bpf_func_proto *to, const struct bpf_func_proto *from)
-{
-	int i;
-	*to = *from;
-	for (i = 0; i < ARRAY_SIZE(to->arg_type); i++) {
-		if (to->arg_type[i] == ARG_PTR_TO_SOCKET) {
-			to->arg_type[i] = ARG_PTR_TO_BTF_ID;
-			to->arg_btf_id[i] = &tcp_sock_id;
-		}
-	}
-}
 static int bpf_tcp_ca_init(struct btf *btf)
 {
 	s32 type_id;
 	tcp_sock_id = type_id;
 	tcp_sock_type = btf_type_by_id(btf, tcp_sock_id);
-	convert_sk_func_proto(&btf_sk_storage_get_proto, &bpf_sk_storage_get_proto);
-	convert_sk_func_proto(&btf_sk_storage_delete_proto, &bpf_sk_storage_delete_proto);
 	return 0;
 }
 	case BPF_FUNC_tcp_send_ack:
 		return &bpf_tcp_send_ack_proto;
 	case BPF_FUNC_sk_storage_get:
-		return &btf_sk_storage_get_proto;
+		return &bpf_sk_storage_get_proto;
 	case BPF_FUNC_sk_storage_delete:
-		return &btf_sk_storage_delete_proto;
+		return &bpf_sk_storage_delete_proto;
 	default:
 		return bpf_base_func_proto(func_id);
 	}

net/xdp/xsk.c+1−0

@@ -703,6 +703,7 @@ static int xsk_bind(struct socket sock, struct sockaddr addr, int addr_len) · 702 unmodified lines
703	xs->pool = xp_create_and_assign_umem(xs,	703	xs->pool = xp_create_and_assign_umem(xs,
704	umem_xs->umem);	704	umem_xs->umem);
705	if (!xs->pool) {	705	if (!xs->pool) {
		706	err = -ENOMEM;
706	sockfd_put(sock);	707	sockfd_put(sock);
707	goto out_unlock;	708	goto out_unlock;
708	}	709	}

net/xdp/xsk_queue.h+1−1

@@ -96,7 +96,7 @@ struct xsk_queue { · 95 unmodified lines
96	* seen and read by the consumer.	96	* seen and read by the consumer.
97	*	97	*
98	* The consumer peeks into the ring to see if the producer has written	98	* The consumer peeks into the ring to see if the producer has written
99	* any new entries. If so, the producer can then read these entries	99	* any new entries. If so, the consumer can then read these entries
100	* and when it is done reading them release them back to the producer	100	* and when it is done reading them release them back to the producer
101	* so that the producer can use these slots to fill in new entries.	101	* so that the producer can use these slots to fill in new entries.
102	*	102	*

samples/bpf/sockex3_kern.c+11−9

 #define PARSE_IP 3
 #define PARSE_IPV6 4
-/* protocol dispatch routine.
+/* Protocol dispatch routine. It tail-calls next BPF program depending
- * It tail-calls next BPF program depending on eth proto
+ * on eth proto. Note, we could have used ...
- * Note, we could have used:
+ *
- * bpf_tail_call(skb, &jmp_table, proto);
+ *   bpf_tail_call(skb, &jmp_table, proto);
- * but it would need large prog_array
+ *
+ * ... but it would need large prog_array and cannot be optimised given
+ * the map key is not static.
  */
 static inline void parse_eth_proto(struct __sk_buff *skb, u32 proto)
 {
 	switch (proto) {
 	case ETH_P_8021Q:
 	case ETH_P_8021AD:
-		bpf_tail_call(skb, &jmp_table, PARSE_VLAN);
+		bpf_tail_call_static(skb, &jmp_table, PARSE_VLAN);
 		break;
 	case ETH_P_MPLS_UC:
 	case ETH_P_MPLS_MC:
-		bpf_tail_call(skb, &jmp_table, PARSE_MPLS);
+		bpf_tail_call_static(skb, &jmp_table, PARSE_MPLS);
 		break;
 	case ETH_P_IP:
-		bpf_tail_call(skb, &jmp_table, PARSE_IP);
+		bpf_tail_call_static(skb, &jmp_table, PARSE_IP);
 		break;
 	case ETH_P_IPV6:
-		bpf_tail_call(skb, &jmp_table, PARSE_IPV6);
+		bpf_tail_call_static(skb, &jmp_table, PARSE_IPV6);
 		break;
 	}
 }

scripts/bpf_helpers_doc.py+2−0

             'struct sk_msg_md',
             'struct xdp_md',
             'struct path',
+            'struct btf_ptr',
     ]
     known_types = {
             '...',
             'struct udp6_sock',
             'struct task_struct',
             'struct path',
+            'struct btf_ptr',
     }
     mapped_types = {
             'u8': '__u8',

tools/include/uapi/linux/bpf.h+125−9

@@ -414,6 +414,9 @@ enum { · 413 unmodified lines
414		414
415	/* Enable memory-mapping BPF map */	415	/* Enable memory-mapping BPF map */
416	BPF_F_MMAPABLE = (1U << 10),	416	BPF_F_MMAPABLE = (1U << 10),
		417
		418	/* Share perf_event among processes */
		419	BPF_F_PRESERVE_ELEMS = (1U << 11),
417	};	420	};
418		421
419	/* Flags for BPF_PROG_QUERY. */	422	/* Flags for BPF_PROG_QUERY. */
@@ -424,6 +427,11 @@ enum { · 4 unmodified lines
424	*/	427	*/
425	#define BPF_F_QUERY_EFFECTIVE (1U << 0)	428	#define BPF_F_QUERY_EFFECTIVE (1U << 0)
426		429
		430	/* Flags for BPF_PROG_TEST_RUN */
		431
		432	/* If set, run the test on the cpu specified by bpf_attr.test.cpu */
		433	#define BPF_F_TEST_RUN_ON_CPU (1U << 0)
		434
427	/* type for BPF_ENABLE_STATS */	435	/* type for BPF_ENABLE_STATS */
428	enum bpf_stats_type {	436	enum bpf_stats_type {
429	/* enabled run_time_ns and run_cnt */	437	/* enabled run_time_ns and run_cnt */
@@ -566,6 +574,8 @@ union bpf_attr { · 136 unmodified lines
566	*/	574	*/
567	__aligned_u64 ctx_in;	575	__aligned_u64 ctx_in;
568	__aligned_u64 ctx_out;	576	__aligned_u64 ctx_out;
		577	__u32 flags;
		578	__u32 cpu;
569	} test;	579	} test;
570		580
571	struct { /* anonymous struct used by BPF__GET__ID */	581	struct { /* anonymous struct used by BPF__GET__ID */
@@ -632,8 +642,13 @@ union bpf_attr { · 60 unmodified lines
632	};	642	};
633	__u32 attach_type; /* attach type */	643	__u32 attach_type; /* attach type */
634	__u32 flags; /* extra flags */	644	__u32 flags; /* extra flags */
635	__aligned_u64 iter_info; /* extra bpf_iter_link_info */	645	union {
636	__u32 iter_info_len; /* iter_info length */	646	__u32 target_btf_id; /* btf_id of target to attach to */
		647	struct {
		648	__aligned_u64 iter_info; /* extra bpf_iter_link_info */
		649	__u32 iter_info_len; /* iter_info length */
		650	};
		651	};
637	} link_create;	652	} link_create;
638		653
639	struct { /* struct used by BPF_LINK_UPDATE command */	654	struct { /* struct used by BPF_LINK_UPDATE command */
@@ -2512,7 +2527,7 @@ union bpf_attr { · 1872 unmodified lines
2512	* result is from reuse\ ->socks\ [] using the hash of the	2527	* result is from reuse\ ->socks\ [] using the hash of the
2513	* tuple.	2528	* tuple.
2514	*	2529	*
2515	* long bpf_sk_release(struct bpf_sock *sock)	2530	* long bpf_sk_release(void *sock)
2516	* Description	2531	* Description
2517	* Release the reference held by sock. sock must be a	2532	* Release the reference held by sock. sock must be a
2518	* non-NULL pointer that was returned from	2533	* non-NULL pointer that was returned from
@@ -2692,7 +2707,7 @@ union bpf_attr { · 173 unmodified lines
2692	* result is from reuse\ ->socks\ [] using the hash of the	2707	* result is from reuse\ ->socks\ [] using the hash of the
2693	* tuple.	2708	* tuple.
2694	*	2709	*
2695	* long bpf_tcp_check_syncookie(struct bpf_sock sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)	2710	* long bpf_tcp_check_syncookie(void sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)
2696	* Description	2711	* Description
2697	* Check whether iph and th contain a valid SYN cookie ACK for	2712	* Check whether iph and th contain a valid SYN cookie ACK for
2698	* the listening socket in sk.	2713	* the listening socket in sk.
@@ -2861,6 +2876,7 @@ union bpf_attr { · 162 unmodified lines
2861	* 0 on success.	2876	* 0 on success.
2862	*	2877	*
2863	* -ENOENT if the bpf-local-storage cannot be found.	2878	* -ENOENT if the bpf-local-storage cannot be found.
		2879	* -EINVAL if sk is not a fullsock (e.g. a request_sock).
2864	*	2880	*
2865	* long bpf_send_signal(u32 sig)	2881	* long bpf_send_signal(u32 sig)
2866	* Description	2882	* Description
@@ -2877,7 +2893,7 @@ union bpf_attr { · 10 unmodified lines
2877	*	2893	*
2878	* -EAGAIN if bpf program can try again.	2894	* -EAGAIN if bpf program can try again.
2879	*	2895	*
2880	* s64 bpf_tcp_gen_syncookie(struct bpf_sock sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)	2896	* s64 bpf_tcp_gen_syncookie(void sk, void iph, u32 iph_len, struct tcphdr *th, u32 th_len)
2881	* Description	2897	* Description
2882	* Try to issue a SYN cookie for the packet with corresponding	2898	* Try to issue a SYN cookie for the packet with corresponding
2883	* IP/TCP headers, iph and th, on the listening socket in sk.	2899	* IP/TCP headers, iph and th, on the listening socket in sk.
@@ -3106,7 +3122,7 @@ union bpf_attr { · 222 unmodified lines
3106	* Return	3122	* Return
3107	* The id is returned or 0 in case the id could not be retrieved.	3123	* The id is returned or 0 in case the id could not be retrieved.
3108	*	3124	*
3109	* long bpf_sk_assign(struct sk_buff skb, struct bpf_sock sk, u64 flags)	3125	* long bpf_sk_assign(struct sk_buff skb, void sk, u64 flags)
3110	* Description	3126	* Description
3111	* Helper is overloaded depending on BPF program type. This	3127	* Helper is overloaded depending on BPF program type. This
3112	* description applies to BPF_PROG_TYPE_SCHED_CLS and	3128	* description applies to BPF_PROG_TYPE_SCHED_CLS and
@@ -3234,11 +3250,11 @@ union bpf_attr { · 121 unmodified lines
3234	*	3250	*
3235	* -EOVERFLOW if an overflow happened: The same object will be tried again.	3251	* -EOVERFLOW if an overflow happened: The same object will be tried again.
3236	*	3252	*
3237	* u64 bpf_sk_cgroup_id(struct bpf_sock *sk)	3253	* u64 bpf_sk_cgroup_id(void *sk)
3238	* Description	3254	* Description
3239	* Return the cgroup v2 id of the socket sk.	3255	* Return the cgroup v2 id of the socket sk.
3240	*	3256	*
3241	* sk must be a non-NULL pointer to a full socket, e.g. one	3257	* sk must be a non-NULL pointer to a socket, e.g. one
3242	* returned from bpf_sk_lookup_xxx\ (),	3258	* returned from bpf_sk_lookup_xxx\ (),
3243	* bpf_sk_fullsock\ (), etc. The format of returned id is	3259	* bpf_sk_fullsock\ (), etc. The format of returned id is
3244	* same as in bpf_skb_cgroup_id\ ().	3260	* same as in bpf_skb_cgroup_id\ ().
@@ -3248,7 +3264,7 @@ union bpf_attr { · 3 unmodified lines
3248	* Return	3264	* Return
3249	* The id is returned or 0 in case the id could not be retrieved.	3265	* The id is returned or 0 in case the id could not be retrieved.
3250	*	3266	*
3251	* u64 bpf_sk_ancestor_cgroup_id(struct bpf_sock *sk, int ancestor_level)	3267	* u64 bpf_sk_ancestor_cgroup_id(void *sk, int ancestor_level)
3252	* Description	3268	* Description
3253	* Return id of cgroup v2 that is ancestor of cgroup associated	3269	* Return id of cgroup v2 that is ancestor of cgroup associated
3254	* with the sk at the ancestor_level. The root cgroup is at	3270	* with the sk at the ancestor_level. The root cgroup is at
@@ -3586,6 +3602,72 @@ union bpf_attr { · 331 unmodified lines
3586	* the data in dst. This is a wrapper of copy_from_user\ ().	3602	* the data in dst. This is a wrapper of copy_from_user\ ().
3587	* Return	3603	* Return
3588	* 0 on success, or a negative error in case of failure.	3604	* 0 on success, or a negative error in case of failure.
		3605	*
		3606	* long bpf_snprintf_btf(char str, u32 str_size, struct btf_ptr ptr, u32 btf_ptr_size, u64 flags)
		3607	* Description
		3608	* Use BTF to store a string representation of ptr->ptr in str,
		3609	* using ptr->type_id. This value should specify the type
		3610	* that ptr->ptr points to. LLVM __builtin_btf_type_id(type, 1)
		3611	* can be used to look up vmlinux BTF type ids. Traversing the
		3612	* data structure using BTF, the type information and values are
		3613	* stored in the first str_size - 1 bytes of str. Safe copy of
		3614	* the pointer data is carried out to avoid kernel crashes during
		3615	* operation. Smaller types can use string space on the stack;
		3616	* larger programs can use map data to store the string
		3617	* representation.
		3618	*
		3619	* The string can be subsequently shared with userspace via
		3620	* bpf_perf_event_output() or ring buffer interfaces.
		3621	* bpf_trace_printk() is to be avoided as it places too small
		3622	* a limit on string size to be useful.
		3623	*
		3624	* flags is a combination of
		3625	*
		3626	* BTF_F_COMPACT
		3627	* no formatting around type information
		3628	* BTF_F_NONAME
		3629	* no struct/union member names/types
		3630	* BTF_F_PTR_RAW
		3631	* show raw (unobfuscated) pointer values;
		3632	* equivalent to printk specifier %px.
		3633	* BTF_F_ZERO
		3634	* show zero-valued struct/union members; they
		3635	* are not displayed by default
		3636	*
		3637	* Return
		3638	* The number of bytes that were written (or would have been
		3639	* written if output had to be truncated due to string size),
		3640	* or a negative error in cases of failure.
		3641	*
		3642	* long bpf_seq_printf_btf(struct seq_file m, struct btf_ptr ptr, u32 ptr_size, u64 flags)
		3643	* Description
		3644	* Use BTF to write to seq_write a string representation of
		3645	* ptr->ptr, using ptr->type_id as per bpf_snprintf_btf().
		3646	* flags are identical to those used for bpf_snprintf_btf.
		3647	* Return
		3648	* 0 on success or a negative error in case of failure.
		3649	*
		3650	* u64 bpf_skb_cgroup_classid(struct sk_buff *skb)
		3651	* Description
		3652	* See bpf_get_cgroup_classid\ () for the main description.
		3653	* This helper differs from bpf_get_cgroup_classid\ () in that
		3654	* the cgroup v1 net_cls class is retrieved only from the skb's
		3655	* associated socket instead of the current process.
		3656	* Return
		3657	* The id is returned or 0 in case the id could not be retrieved.
		3658	*
		3659	* long bpf_redirect_neigh(u32 ifindex, u64 flags)
		3660	* Description
		3661	* Redirect the packet to another net device of index ifindex
		3662	* and fill in L2 addresses from neighboring subsystem. This helper
		3663	* is somewhat similar to bpf_redirect\ (), except that it
		3664	* fills in e.g. MAC addresses based on the L3 information from
		3665	* the packet. This helper is supported for IPv4 and IPv6 protocols.
		3666	* The flags argument is reserved and must be 0. The helper is
		3667	* currently only supported for tc BPF program types.
		3668	* Return
		3669	* The helper returns TC_ACT_REDIRECT on success or
		3670	* TC_ACT_SHOT on error.
3589	*/	3671	*/
3590	#define __BPF_FUNC_MAPPER(FN) \	3672	#define __BPF_FUNC_MAPPER(FN) \
3591	FN(unspec), \	3673	FN(unspec), \
@@ -3737,6 +3819,10 @@ union bpf_attr { · 145 unmodified lines
3737	FN(inode_storage_delete), \	3819	FN(inode_storage_delete), \
3738	FN(d_path), \	3820	FN(d_path), \
3739	FN(copy_from_user), \	3821	FN(copy_from_user), \
		3822	FN(snprintf_btf), \
		3823	FN(seq_printf_btf), \
		3824	FN(skb_cgroup_classid), \
		3825	FN(redirect_neigh), \
3740	/* */	3826	/* */
3741		3827
3742	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	3828	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -4845,4 +4931,34 @@ struct bpf_sk_lookup { · 1102 unmodified lines
4845	__u32 local_port; /* Host byte order */	4931	__u32 local_port; /* Host byte order */
4846	};	4932	};
4847		4933
		4934	/*
		4935	* struct btf_ptr is used for typed pointer representation; the
		4936	* type id is used to render the pointer data as the appropriate type
		4937	* via the bpf_snprintf_btf() helper described above. A flags field -
		4938	* potentially to specify additional details about the BTF pointer
		4939	* (rather than its mode of display) - is included for future use.
		4940	* Display flags - BTF_F_* - are passed to bpf_snprintf_btf separately.
		4941	*/
		4942	struct btf_ptr {
		4943	void *ptr;
		4944	__u32 type_id;
		4945	__u32 flags; /* BTF ptr flags; unused at present. */
		4946	};
		4947
		4948	/*
		4949	* Flags to control bpf_snprintf_btf() behaviour.
		4950	* - BTF_F_COMPACT: no formatting around type information
		4951	* - BTF_F_NONAME: no struct/union member names/types
		4952	* - BTF_F_PTR_RAW: show raw (unobfuscated) pointer values;
		4953	* equivalent to %px.
		4954	* - BTF_F_ZERO: show zero-valued struct/union members; they
		4955	* are not displayed by default
		4956	*/
		4957	enum {
		4958	BTF_F_COMPACT = (1ULL << 0),
		4959	BTF_F_NONAME = (1ULL << 1),
		4960	BTF_F_PTR_RAW = (1ULL << 2),
		4961	BTF_F_ZERO = (1ULL << 3),
		4962	};
		4963
4848	#endif /* _UAPI__LINUX_BPF_H__ */	4964	#endif /* _UAPI__LINUX_BPF_H__ */

tools/lib/bpf/Makefile+2−3

 ifdef EXTRA_CFLAGS
   CFLAGS := $(EXTRA_CFLAGS)
 else
-  CFLAGS := -g -Wall
+  CFLAGS := -g -O2
 endif
 # Append required CFLAGS
 override CFLAGS += $(EXTRA_WARNINGS) -Wno-switch-enum
 override CFLAGS += -Werror -Wall
-override CFLAGS += -fPIC
 override CFLAGS += $(INCLUDES)
 override CFLAGS += -fvisibility=hidden
 override CFLAGS += -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64
 # flags specific for shared library
-SHLIB_FLAGS := -DSHARED
+SHLIB_FLAGS := -DSHARED -fPIC
 ifeq ($(VERBOSE),1)
   Q =

tools/lib/bpf/bpf.c+47−4

@@ -586,19 +586,31 @@ int bpf_link_create(int prog_fd, int target_fd, · 585 unmodified lines
586	enum bpf_attach_type attach_type,	586	enum bpf_attach_type attach_type,
587	const struct bpf_link_create_opts *opts)	587	const struct bpf_link_create_opts *opts)
588	{	588	{
		589	__u32 target_btf_id, iter_info_len;
589	union bpf_attr attr;	590	union bpf_attr attr;
590		591
591	if (!OPTS_VALID(opts, bpf_link_create_opts))	592	if (!OPTS_VALID(opts, bpf_link_create_opts))
592	return -EINVAL;	593	return -EINVAL;
593		594
		595	iter_info_len = OPTS_GET(opts, iter_info_len, 0);
		596	target_btf_id = OPTS_GET(opts, target_btf_id, 0);
		597
		598	if (iter_info_len && target_btf_id)
		599	return -EINVAL;
		600
594	memset(&attr, 0, sizeof(attr));	601	memset(&attr, 0, sizeof(attr));
595	attr.link_create.prog_fd = prog_fd;	602	attr.link_create.prog_fd = prog_fd;
596	attr.link_create.target_fd = target_fd;	603	attr.link_create.target_fd = target_fd;
597	attr.link_create.attach_type = attach_type;	604	attr.link_create.attach_type = attach_type;
598	attr.link_create.flags = OPTS_GET(opts, flags, 0);	605	attr.link_create.flags = OPTS_GET(opts, flags, 0);
599	attr.link_create.iter_info =	606
600	ptr_to_u64(OPTS_GET(opts, iter_info, (void *)0));	607	if (iter_info_len) {
601	attr.link_create.iter_info_len = OPTS_GET(opts, iter_info_len, 0);	608	attr.link_create.iter_info =
		609	ptr_to_u64(OPTS_GET(opts, iter_info, (void *)0));
		610	attr.link_create.iter_info_len = iter_info_len;
		611	} else if (target_btf_id) {
		612	attr.link_create.target_btf_id = target_btf_id;
		613	}
602		614
603	return sys_bpf(BPF_LINK_CREATE, &attr, sizeof(attr));	615	return sys_bpf(BPF_LINK_CREATE, &attr, sizeof(attr));
604	}	616	}
@@ -712,6 +724,37 @@ int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr) · 107 unmodified lines
712	return ret;	724	return ret;
713	}	725	}
714		726
		727	int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts)
		728	{
		729	union bpf_attr attr;
		730	int ret;
		731
		732	if (!OPTS_VALID(opts, bpf_test_run_opts))
		733	return -EINVAL;
		734
		735	memset(&attr, 0, sizeof(attr));
		736	attr.test.prog_fd = prog_fd;
		737	attr.test.cpu = OPTS_GET(opts, cpu, 0);
		738	attr.test.flags = OPTS_GET(opts, flags, 0);
		739	attr.test.repeat = OPTS_GET(opts, repeat, 0);
		740	attr.test.duration = OPTS_GET(opts, duration, 0);
		741	attr.test.ctx_size_in = OPTS_GET(opts, ctx_size_in, 0);
		742	attr.test.ctx_size_out = OPTS_GET(opts, ctx_size_out, 0);
		743	attr.test.data_size_in = OPTS_GET(opts, data_size_in, 0);
		744	attr.test.data_size_out = OPTS_GET(opts, data_size_out, 0);
		745	attr.test.ctx_in = ptr_to_u64(OPTS_GET(opts, ctx_in, NULL));
		746	attr.test.ctx_out = ptr_to_u64(OPTS_GET(opts, ctx_out, NULL));
		747	attr.test.data_in = ptr_to_u64(OPTS_GET(opts, data_in, NULL));
		748	attr.test.data_out = ptr_to_u64(OPTS_GET(opts, data_out, NULL));
		749
		750	ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
		751	OPTS_SET(opts, data_size_out, attr.test.data_size_out);
		752	OPTS_SET(opts, ctx_size_out, attr.test.ctx_size_out);
		753	OPTS_SET(opts, duration, attr.test.duration);
		754	OPTS_SET(opts, retval, attr.test.retval);
		755	return ret;
		756	}
		757
715	static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd)	758	static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd)
716	{	759	{
717	union bpf_attr attr;	760	union bpf_attr attr;
@@ -815,7 +858,7 @@ int bpf_raw_tracepoint_open(const char *name, int prog_fd) · 97 unmodified lines
815	return sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr));	858	return sys_bpf(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr));
816	}	859	}
817		860
818	int bpf_load_btf(void btf, __u32 btf_size, char log_buf, __u32 log_buf_size,	861	int bpf_load_btf(const void btf, __u32 btf_size, char log_buf, __u32 log_buf_size,
819	bool do_log)	862	bool do_log)
820	{	863	{
821	union bpf_attr attr = {};	864	union bpf_attr attr = {};

tools/lib/bpf/bpf.h+29−2

@@ -174,8 +174,9 @@ struct bpf_link_create_opts { · 173 unmodified lines
174	__u32 flags;	174	__u32 flags;
175	union bpf_iter_link_info *iter_info;	175	union bpf_iter_link_info *iter_info;
176	__u32 iter_info_len;	176	__u32 iter_info_len;
		177	__u32 target_btf_id;
177	};	178	};
178	#define bpf_link_create_opts__last_field iter_info_len	179	#define bpf_link_create_opts__last_field target_btf_id
179		180
180	LIBBPF_API int bpf_link_create(int prog_fd, int target_fd,	181	LIBBPF_API int bpf_link_create(int prog_fd, int target_fd,
181	enum bpf_attach_type attach_type,	182	enum bpf_attach_type attach_type,
@@ -234,7 +235,7 @@ LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, · 52 unmodified lines
234	__u32 query_flags, __u32 *attach_flags,	235	__u32 query_flags, __u32 *attach_flags,
235	__u32 prog_ids, __u32 prog_cnt);	236	__u32 prog_ids, __u32 prog_cnt);
236	LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd);	237	LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd);
237	LIBBPF_API int bpf_load_btf(void btf, __u32 btf_size, char log_buf,	238	LIBBPF_API int bpf_load_btf(const void btf, __u32 btf_size, char log_buf,
238	__u32 log_buf_size, bool do_log);	239	__u32 log_buf_size, bool do_log);
239	LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf,	240	LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf,
240	__u32 buf_len, __u32 prog_id, __u32 *fd_type,	241	__u32 buf_len, __u32 prog_id, __u32 *fd_type,
@@ -251,6 +252,32 @@ struct bpf_prog_bind_opts { · 10 unmodified lines
251		252
252	LIBBPF_API int bpf_prog_bind_map(int prog_fd, int map_fd,	253	LIBBPF_API int bpf_prog_bind_map(int prog_fd, int map_fd,
253	const struct bpf_prog_bind_opts *opts);	254	const struct bpf_prog_bind_opts *opts);
		255
		256	struct bpf_test_run_opts {
		257	size_t sz; /* size of this struct for forward/backward compatibility */
		258	const void data_in; / optional */
		259	void data_out; / optional */
		260	__u32 data_size_in;
		261	__u32 data_size_out; /* in: max length of data_out
		262	* out: length of data_out
		263	*/
		264	const void ctx_in; / optional */
		265	void ctx_out; / optional */
		266	__u32 ctx_size_in;
		267	__u32 ctx_size_out; /* in: max length of ctx_out
		268	* out: length of cxt_out
		269	*/
		270	__u32 retval; /* out: return code of the BPF program */
		271	int repeat;
		272	__u32 duration; /* out: average per repetition in ns */
		273	__u32 flags;
		274	__u32 cpu;
		275	};
		276	#define bpf_test_run_opts__last_field cpu
		277
		278	LIBBPF_API int bpf_prog_test_run_opts(int prog_fd,
		279	struct bpf_test_run_opts *opts);
		280
254	#ifdef __cplusplus	281	#ifdef __cplusplus
255	} /* extern "C" */	282	} /* extern "C" */
256	#endif	283	#endif

tools/lib/bpf/bpf_helpers.h+46−0

@@ -53,6 +53,52 @@ · 52 unmodified lines
53	})	53	})
54	#endif	54	#endif
55		55
		56	/*
		57	* Helper macro to throw a compilation error if __bpf_unreachable() gets
		58	* built into the resulting code. This works given BPF back end does not
		59	* implement __builtin_trap(). This is useful to assert that certain paths
		60	* of the program code are never used and hence eliminated by the compiler.
		61	*
		62	* For example, consider a switch statement that covers known cases used by
		63	* the program. __bpf_unreachable() can then reside in the default case. If
		64	* the program gets extended such that a case is not covered in the switch
		65	* statement, then it will throw a build error due to the default case not
		66	* being compiled out.
		67	*/
		68	#ifndef __bpf_unreachable
		69	# define __bpf_unreachable() __builtin_trap()
		70	#endif
		71
		72	/*
		73	* Helper function to perform a tail call with a constant/immediate map slot.
		74	*/
		75	static __always_inline void
		76	bpf_tail_call_static(void ctx, const void map, const __u32 slot)
		77	{
		78	if (!__builtin_constant_p(slot))
		79	__bpf_unreachable();
		80
		81	/*
		82	* Provide a hard guarantee that LLVM won't optimize setting r2 (map
		83	* pointer) and r3 (constant map index) from _different paths_ ending
		84	* up at the _same_ call insn as otherwise we won't be able to use the
		85	* jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel
		86	* given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key
		87	* tracking for prog array pokes") for details on verifier tracking.
		88	*
		89	* Note on clobber list: we need to stay in-line with BPF calling
		90	* convention, so even if we don't end up using r0, r4, r5, we need
		91	* to mark them as clobber so that LLVM doesn't end up using them
		92	* before / after the call.
		93	*/
		94	asm volatile("r1 = %[ctx]\n\t"
		95	"r2 = %[map]\n\t"
		96	"r3 = %[slot]\n\t"
		97	"call 12"
		98	:: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
		99	: "r0", "r1", "r2", "r3", "r4", "r5");
		100	}
		101
56	/*	102	/*
57	* Helper structure used by eBPF C program	103	* Helper structure used by eBPF C program
58	* to describe BPF map attributes to libbpf loader	104	* to describe BPF map attributes to libbpf loader

tools/lib/bpf/btf.c+1595−273

@@ -1,6 +1,7 @@
1	// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)	1	// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2	/* Copyright (c) 2018 Facebook */	2	/* Copyright (c) 2018 Facebook */
3		3
		4	#include <byteswap.h>
4	#include <endian.h>	5	#include <endian.h>
5	#include <stdio.h>	6	#include <stdio.h>
6	#include <stdlib.h>	7	#include <stdlib.h>
@@ -27,17 +28,72 @@ · 20 unmodified lines
27	static struct btf_type btf_void;	28	static struct btf_type btf_void;
28		29
29	struct btf {	30	struct btf {
30	union {	31	/* raw BTF data in native endianness */
31	struct btf_header *hdr;	32	void *raw_data;
32	void *data;	33	/* raw BTF data in non-native endianness */
33	};	34	void *raw_data_swapped;
34	struct btf_type **types;	35	__u32 raw_size;
35	const char *strings;	36	/* whether target endianness differs from the native one */
36	void *nohdr_data;	37	bool swapped_endian;
		38
		39	/*
		40	* When BTF is loaded from an ELF or raw memory it is stored
		41	* in a contiguous memory block. The hdr, type_data, and, strs_data
		42	* point inside that memory region to their respective parts of BTF
		43	* representation:
		44	*
		45	* +--------------------------------+
		46	* \| Header \| Types \| Strings \|
		47	* +--------------------------------+
		48	* ^ ^ ^
		49	* \| \| \|
		50	* hdr \| \|
		51	* types_data-+ \|
		52	* strs_data------------+
		53	*
		54	* If BTF data is later modified, e.g., due to types added or
		55	* removed, BTF deduplication performed, etc, this contiguous
		56	* representation is broken up into three independently allocated
		57	* memory regions to be able to modify them independently.
		58	* raw_data is nulled out at that point, but can be later allocated
		59	* and cached again if user calls btf__get_raw_data(), at which point
		60	* raw_data will contain a contiguous copy of header, types, and
		61	* strings:
		62	*
		63	* +----------+ +---------+ +-----------+
		64	* \| Header \| \| Types \| \| Strings \|
		65	* +----------+ +---------+ +-----------+
		66	* ^ ^ ^
		67	* \| \| \|
		68	* hdr \| \|
		69	* types_data----+ \|
		70	* strs_data------------------+
		71	*
		72	* +----------+---------+-----------+
		73	* \| Header \| Types \| Strings \|
		74	* raw_data----->+----------+---------+-----------+
		75	*/
		76	struct btf_header *hdr;
		77
		78	void *types_data;
		79	size_t types_data_cap; /* used size stored in hdr->type_len */
		80
		81	/* type ID to `struct btf_type ` lookup index /
		82	__u32 *type_offs;
		83	size_t type_offs_cap;
37	__u32 nr_types;	84	__u32 nr_types;
38	__u32 types_size;	85
39	__u32 data_size;	86	void *strs_data;
		87	size_t strs_data_cap; /* used size stored in hdr->str_len */
		88
		89	/* lookup index for each unique string in strings section */
		90	struct hashmap *strs_hash;
		91	/* whether strings are already deduplicated */
		92	bool strs_deduped;
		93	/* BTF object FD, if loaded into kernel */
40	int fd;	94	int fd;
		95
		96	/* Pointer size (in bytes) for a target architecture of this BTF */
41	int ptr_sz;	97	int ptr_sz;
42	};	98	};
43		99
@@ -46,60 +102,114 @@ static inline __u64 ptr_to_u64(const void *ptr) · 2 unmodified lines
46	return (__u64) (unsigned long) ptr;	102	return (__u64) (unsigned long) ptr;
47	}	103	}
48		104
49	static int btf_add_type(struct btf btf, struct btf_type t)	105	/* Ensure given dynamically allocated memory region pointed to by data with
		106	* capacity of cap_cnt elements each taking elem_sz bytes has enough
		107	* memory to accomodate add_cnt new elements, assuming cur_cnt elements
		108	* are already used. At most max_cnt elements can be ever allocated.
		109	* If necessary, memory is reallocated and all existing data is copied over,
		110	* new pointer to the memory region is stored at *data, new memory region
		111	* capacity (in number of elements) is stored in *cap.
		112	* On success, memory pointer to the beginning of unused memory is returned.
		113	* On error, NULL is returned.
		114	*/
		115	void btf_add_mem(void data, size_t cap_cnt, size_t elem_sz,
		116	size_t cur_cnt, size_t max_cnt, size_t add_cnt)
50	{	117	{
51	if (btf->types_size - btf->nr_types < 2) {	118	size_t new_cnt;
52	struct btf_type **new_types;	119	void *new_data;
53	__u32 expand_by, new_size;
54		120
55	if (btf->types_size == BTF_MAX_NR_TYPES)	121	if (cur_cnt + add_cnt <= *cap_cnt)
56	return -E2BIG;	122	return data + cur_cnt elem_sz;
57		123
58	expand_by = max(btf->types_size >> 2, 16U);	124	/* requested more than the set limit */
59	new_size = min(BTF_MAX_NR_TYPES, btf->types_size + expand_by);	125	if (cur_cnt + add_cnt > max_cnt)
		126	return NULL;
60		127
61	new_types = libbpf_reallocarray(btf->types, new_size, sizeof(*new_types));	128	new_cnt = *cap_cnt;
62	if (!new_types)	129	new_cnt += new_cnt / 4; /* expand by 25% */
63	return -ENOMEM;	130	if (new_cnt < 16) /* but at least 16 elements */
		131	new_cnt = 16;
		132	if (new_cnt > max_cnt) /* but not exceeding a set limit */
		133	new_cnt = max_cnt;
		134	if (new_cnt < cur_cnt + add_cnt) /* also ensure we have enough memory */
		135	new_cnt = cur_cnt + add_cnt;
		136
		137	new_data = libbpf_reallocarray(*data, new_cnt, elem_sz);
		138	if (!new_data)
		139	return NULL;
64		140
65	if (btf->nr_types == 0)	141	/* zero out newly allocated portion of memory */
66	new_types[0] = &btf_void;	142	memset(new_data + (cap_cnt) elem_sz, 0, (new_cnt - cap_cnt) elem_sz);
67		143
68	btf->types = new_types;	144	*data = new_data;
69	btf->types_size = new_size;	145	*cap_cnt = new_cnt;
70	}	146	return new_data + cur_cnt * elem_sz;
		147	}
		148
		149	/* Ensure given dynamically allocated memory region has enough allocated space
		150	* to accommodate need_cnt elements of size elem_sz bytes each
		151	*/
		152	int btf_ensure_mem(void *data, size_t cap_cnt, size_t elem_sz, size_t need_cnt)
		153	{
		154	void *p;
		155
		156	if (need_cnt <= *cap_cnt)
		157	return 0;
		158
		159	p = btf_add_mem(data, cap_cnt, elem_sz, cap_cnt, SIZE_MAX, need_cnt - cap_cnt);
		160	if (!p)
		161	return -ENOMEM;
		162
		163	return 0;
		164	}
		165
		166	static int btf_add_type_idx_entry(struct btf *btf, __u32 type_off)
		167	{
		168	__u32 *p;
71		169
72	btf->types[++(btf->nr_types)] = t;	170	p = btf_add_mem((void **)&btf->type_offs, &btf->type_offs_cap, sizeof(__u32),
		171	btf->nr_types + 1, BTF_MAX_NR_TYPES, 1);
		172	if (!p)
		173	return -ENOMEM;
73		174
		175	*p = type_off;
74	return 0;	176	return 0;
75	}	177	}
76		178
		179	static void btf_bswap_hdr(struct btf_header *h)
		180	{
		181	h->magic = bswap_16(h->magic);
		182	h->hdr_len = bswap_32(h->hdr_len);
		183	h->type_off = bswap_32(h->type_off);
		184	h->type_len = bswap_32(h->type_len);
		185	h->str_off = bswap_32(h->str_off);
		186	h->str_len = bswap_32(h->str_len);
		187	}
		188
77	static int btf_parse_hdr(struct btf *btf)	189	static int btf_parse_hdr(struct btf *btf)
78	{	190	{
79	const struct btf_header *hdr = btf->hdr;	191	struct btf_header *hdr = btf->hdr;
80	__u32 meta_left;	192	__u32 meta_left;
81		193
82	if (btf->data_size < sizeof(struct btf_header)) {	194	if (btf->raw_size < sizeof(struct btf_header)) {
83	pr_debug("BTF header not found\n");	195	pr_debug("BTF header not found\n");
84	return -EINVAL;	196	return -EINVAL;
85	}	197	}
86		198
87	if (hdr->magic != BTF_MAGIC) {	199	if (hdr->magic == bswap_16(BTF_MAGIC)) {
		200	btf->swapped_endian = true;
		201	if (bswap_32(hdr->hdr_len) != sizeof(struct btf_header)) {
		202	pr_warn("Can't load BTF with non-native endianness due to unsupported header length %u\n",
		203	bswap_32(hdr->hdr_len));
		204	return -ENOTSUP;
		205	}
		206	btf_bswap_hdr(hdr);
		207	} else if (hdr->magic != BTF_MAGIC) {
88	pr_debug("Invalid BTF magic:%x\n", hdr->magic);	208	pr_debug("Invalid BTF magic:%x\n", hdr->magic);
89	return -EINVAL;	209	return -EINVAL;
90	}	210	}
91		211
92	if (hdr->version != BTF_VERSION) {	212	meta_left = btf->raw_size - sizeof(*hdr);
93	pr_debug("Unsupported BTF version:%u\n", hdr->version);
94	return -ENOTSUP;
95	}
96
97	if (hdr->flags) {
98	pr_debug("Unsupported BTF flags:%x\n", hdr->flags);
99	return -ENOTSUP;
100	}
101
102	meta_left = btf->data_size - sizeof(*hdr);
103	if (!meta_left) {	213	if (!meta_left) {
104	pr_debug("BTF has no data\n");	214	pr_debug("BTF has no data\n");
105	return -EINVAL;	215	return -EINVAL;
@@ -125,15 +235,13 @@ static int btf_parse_hdr(struct btf *btf) · 19 unmodified lines
125	return -EINVAL;	235	return -EINVAL;
126	}	236	}
127		237
128	btf->nohdr_data = btf->hdr + 1;
129
130	return 0;	238	return 0;
131	}	239	}
132		240
133	static int btf_parse_str_sec(struct btf *btf)	241	static int btf_parse_str_sec(struct btf *btf)
134	{	242	{
135	const struct btf_header *hdr = btf->hdr;	243	const struct btf_header *hdr = btf->hdr;
136	const char *start = btf->nohdr_data + hdr->str_off;	244	const char *start = btf->strs_data;
137	const char *end = start + btf->hdr->str_len;	245	const char *end = start + btf->hdr->str_len;
138		246
139	if (!hdr->str_len \|\| hdr->str_len - 1 > BTF_MAX_STR_OFFSET \|\|	247	if (!hdr->str_len \|\| hdr->str_len - 1 > BTF_MAX_STR_OFFSET \|\|
@@ -142,14 +250,12 @@ static int btf_parse_str_sec(struct btf *btf) · 2 unmodified lines
142	return -EINVAL;	250	return -EINVAL;
143	}	251	}
144		252
145	btf->strings = start;
146
147	return 0;	253	return 0;
148	}	254	}
149		255
150	static int btf_type_size(struct btf_type *t)	256	static int btf_type_size(const struct btf_type *t)
151	{	257	{
152	int base_size = sizeof(struct btf_type);	258	const int base_size = sizeof(struct btf_type);
153	__u16 vlen = btf_vlen(t);	259	__u16 vlen = btf_vlen(t);
154		260
155	switch (btf_kind(t)) {	261	switch (btf_kind(t)) {
@@ -182,25 +288,120 @@ static int btf_type_size(struct btf_type *t) · 26 unmodified lines
182	}	288	}
183	}	289	}
184		290
		291	static void btf_bswap_type_base(struct btf_type *t)
		292	{
		293	t->name_off = bswap_32(t->name_off);
		294	t->info = bswap_32(t->info);
		295	t->type = bswap_32(t->type);
		296	}
		297
		298	static int btf_bswap_type_rest(struct btf_type *t)
		299	{
		300	struct btf_var_secinfo *v;
		301	struct btf_member *m;
		302	struct btf_array *a;
		303	struct btf_param *p;
		304	struct btf_enum *e;
		305	__u16 vlen = btf_vlen(t);
		306	int i;
		307
		308	switch (btf_kind(t)) {
		309	case BTF_KIND_FWD:
		310	case BTF_KIND_CONST:
		311	case BTF_KIND_VOLATILE:
		312	case BTF_KIND_RESTRICT:
		313	case BTF_KIND_PTR:
		314	case BTF_KIND_TYPEDEF:
		315	case BTF_KIND_FUNC:
		316	return 0;
		317	case BTF_KIND_INT:
		318	(__u32 )(t + 1) = bswap_32((__u32 )(t + 1));
		319	return 0;
		320	case BTF_KIND_ENUM:
		321	for (i = 0, e = btf_enum(t); i < vlen; i++, e++) {
		322	e->name_off = bswap_32(e->name_off);
		323	e->val = bswap_32(e->val);
		324	}
		325	return 0;
		326	case BTF_KIND_ARRAY:
		327	a = btf_array(t);
		328	a->type = bswap_32(a->type);
		329	a->index_type = bswap_32(a->index_type);
		330	a->nelems = bswap_32(a->nelems);
		331	return 0;
		332	case BTF_KIND_STRUCT:
		333	case BTF_KIND_UNION:
		334	for (i = 0, m = btf_members(t); i < vlen; i++, m++) {
		335	m->name_off = bswap_32(m->name_off);
		336	m->type = bswap_32(m->type);
		337	m->offset = bswap_32(m->offset);
		338	}
		339	return 0;
		340	case BTF_KIND_FUNC_PROTO:
		341	for (i = 0, p = btf_params(t); i < vlen; i++, p++) {
		342	p->name_off = bswap_32(p->name_off);
		343	p->type = bswap_32(p->type);
		344	}
		345	return 0;
		346	case BTF_KIND_VAR:
		347	btf_var(t)->linkage = bswap_32(btf_var(t)->linkage);
		348	return 0;
		349	case BTF_KIND_DATASEC:
		350	for (i = 0, v = btf_var_secinfos(t); i < vlen; i++, v++) {
		351	v->type = bswap_32(v->type);
		352	v->offset = bswap_32(v->offset);
		353	v->size = bswap_32(v->size);
		354	}
		355	return 0;
		356	default:
		357	pr_debug("Unsupported BTF_KIND:%u\n", btf_kind(t));
		358	return -EINVAL;
		359	}
		360	}
		361
185	static int btf_parse_type_sec(struct btf *btf)	362	static int btf_parse_type_sec(struct btf *btf)
186	{	363	{
187	struct btf_header *hdr = btf->hdr;	364	struct btf_header *hdr = btf->hdr;
188	void *nohdr_data = btf->nohdr_data;	365	void *next_type = btf->types_data;
189	void *next_type = nohdr_data + hdr->type_off;	366	void *end_type = next_type + hdr->type_len;
190	void *end_type = nohdr_data + hdr->str_off;	367	int err, i = 0, type_size;
		368
		369	/* VOID (type_id == 0) is specially handled by btf__get_type_by_id(),
		370	* so ensure we can never properly use its offset from index by
		371	* setting it to a large value
		372	*/
		373	err = btf_add_type_idx_entry(btf, UINT_MAX);
		374	if (err)
		375	return err;
191		376
192	while (next_type < end_type) {	377	while (next_type + sizeof(struct btf_type) <= end_type) {
193	struct btf_type *t = next_type;	378	i++;
194	int type_size;
195	int err;
196		379
197	type_size = btf_type_size(t);	380	if (btf->swapped_endian)
		381	btf_bswap_type_base(next_type);
		382
		383	type_size = btf_type_size(next_type);
198	if (type_size < 0)	384	if (type_size < 0)
199	return type_size;	385	return type_size;
200	next_type += type_size;	386	if (next_type + type_size > end_type) {
201	err = btf_add_type(btf, t);	387	pr_warn("BTF type [%d] is malformed\n", i);
		388	return -EINVAL;
		389	}
		390
		391	if (btf->swapped_endian && btf_bswap_type_rest(next_type))
		392	return -EINVAL;
		393
		394	err = btf_add_type_idx_entry(btf, next_type - btf->types_data);
202	if (err)	395	if (err)
203	return err;	396	return err;
		397
		398	next_type += type_size;
		399	btf->nr_types++;
		400	}
		401
		402	if (next_type != end_type) {
		403	pr_warn("BTF types data is malformed\n");
		404	return -EINVAL;
204	}	405	}
205		406
206	return 0;	407	return 0;
@@ -211,12 +412,20 @@ __u32 btf__get_nr_types(const struct btf *btf) · 4 unmodified lines
211	return btf->nr_types;	412	return btf->nr_types;
212	}	413	}
213		414
		415	/* internal helper returning non-const pointer to a type */
		416	static struct btf_type btf_type_by_id(struct btf btf, __u32 type_id)
		417	{
		418	if (type_id == 0)
		419	return &btf_void;
		420
		421	return btf->types_data + btf->type_offs[type_id];
		422	}
		423
214	const struct btf_type btf__type_by_id(const struct btf btf, __u32 type_id)	424	const struct btf_type btf__type_by_id(const struct btf btf, __u32 type_id)
215	{	425	{
216	if (type_id > btf->nr_types)	426	if (type_id > btf->nr_types)
217	return NULL;	427	return NULL;
218		428	return btf_type_by_id((struct btf *)btf, type_id);
219	return btf->types[type_id];
220	}	429	}
221		430
222	static int determine_ptr_size(const struct btf *btf)	431	static int determine_ptr_size(const struct btf *btf)
@@ -283,6 +492,38 @@ int btf__set_pointer_size(struct btf *btf, size_t ptr_sz) · 60 unmodified lines
283	return 0;	492	return 0;
284	}	493	}
285		494
		495	static bool is_host_big_endian(void)
		496	{
		497	#if __BYTE_ORDER == __LITTLE_ENDIAN
		498	return false;
		499	#elif __BYTE_ORDER == __BIG_ENDIAN
		500	return true;
		501	#else
		502	# error "Unrecognized __BYTE_ORDER__"
		503	#endif
		504	}
		505
		506	enum btf_endianness btf__endianness(const struct btf *btf)
		507	{
		508	if (is_host_big_endian())
		509	return btf->swapped_endian ? BTF_LITTLE_ENDIAN : BTF_BIG_ENDIAN;
		510	else
		511	return btf->swapped_endian ? BTF_BIG_ENDIAN : BTF_LITTLE_ENDIAN;
		512	}
		513
		514	int btf__set_endianness(struct btf *btf, enum btf_endianness endian)
		515	{
		516	if (endian != BTF_LITTLE_ENDIAN && endian != BTF_BIG_ENDIAN)
		517	return -EINVAL;
		518
		519	btf->swapped_endian = is_host_big_endian() != (endian == BTF_BIG_ENDIAN);
		520	if (!btf->swapped_endian) {
		521	free(btf->raw_data_swapped);
		522	btf->raw_data_swapped = NULL;
		523	}
		524	return 0;
		525	}
		526
286	static bool btf_type_is_void(const struct btf_type *t)	527	static bool btf_type_is_void(const struct btf_type *t)
287	{	528	{
288	return t == &btf_void \|\| btf_is_fwd(t);	529	return t == &btf_void \|\| btf_is_fwd(t);
@@ -414,7 +655,7 @@ __s32 btf__find_by_name(const struct btf btf, const char type_name) · 125 unmodified lines
414	return 0;	655	return 0;
415		656
416	for (i = 1; i <= btf->nr_types; i++) {	657	for (i = 1; i <= btf->nr_types; i++) {
417	const struct btf_type *t = btf->types[i];	658	const struct btf_type *t = btf__type_by_id(btf, i);
418	const char *name = btf__name_by_offset(btf, t->name_off);	659	const char *name = btf__name_by_offset(btf, t->name_off);
419		660
420	if (name && !strcmp(type_name, name))	661	if (name && !strcmp(type_name, name))
@@ -433,7 +674,7 @@ __s32 btf__find_by_name_kind(const struct btf btf, const char type_name, · 12 unmodified lines
433	return 0;	674	return 0;
434		675
435	for (i = 1; i <= btf->nr_types; i++) {	676	for (i = 1; i <= btf->nr_types; i++) {
436	const struct btf_type *t = btf->types[i];	677	const struct btf_type *t = btf__type_by_id(btf, i);
437	const char *name;	678	const char *name;
438		679
439	if (btf_kind(t) != kind)	680	if (btf_kind(t) != kind)
@@ -446,6 +687,11 @@ __s32 btf__find_by_name_kind(const struct btf btf, const char type_name, · 6 unmodified lines
446	return -ENOENT;	687	return -ENOENT;
447	}	688	}
448		689
		690	static bool btf_is_modifiable(const struct btf *btf)
		691	{
		692	return (void *)btf->hdr != btf->raw_data;
		693	}
		694
449	void btf__free(struct btf *btf)	695	void btf__free(struct btf *btf)
450	{	696	{
451	if (IS_ERR_OR_NULL(btf))	697	if (IS_ERR_OR_NULL(btf))
@@ -454,11 +700,55 @@ void btf__free(struct btf *btf) · 2 unmodified lines
454	if (btf->fd >= 0)	700	if (btf->fd >= 0)
455	close(btf->fd);	701	close(btf->fd);
456		702
457	free(btf->data);	703	if (btf_is_modifiable(btf)) {
458	free(btf->types);	704	/* if BTF was modified after loading, it will have a split
		705	* in-memory representation for header, types, and strings
		706	* sections, so we need to free all of them individually. It
		707	* might still have a cached contiguous raw data present,
		708	* which will be unconditionally freed below.
		709	*/
		710	free(btf->hdr);
		711	free(btf->types_data);
		712	free(btf->strs_data);
		713	}
		714	free(btf->raw_data);
		715	free(btf->raw_data_swapped);
		716	free(btf->type_offs);
459	free(btf);	717	free(btf);
460	}	718	}
461		719
		720	struct btf *btf__new_empty(void)
		721	{
		722	struct btf *btf;
		723
		724	btf = calloc(1, sizeof(*btf));
		725	if (!btf)
		726	return ERR_PTR(-ENOMEM);
		727
		728	btf->fd = -1;
		729	btf->ptr_sz = sizeof(void *);
		730	btf->swapped_endian = false;
		731
		732	/* +1 for empty string at offset 0 */
		733	btf->raw_size = sizeof(struct btf_header) + 1;
		734	btf->raw_data = calloc(1, btf->raw_size);
		735	if (!btf->raw_data) {
		736	free(btf);
		737	return ERR_PTR(-ENOMEM);
		738	}
		739
		740	btf->hdr = btf->raw_data;
		741	btf->hdr->hdr_len = sizeof(struct btf_header);
		742	btf->hdr->magic = BTF_MAGIC;
		743	btf->hdr->version = BTF_VERSION;
		744
		745	btf->types_data = btf->raw_data + btf->hdr->hdr_len;
		746	btf->strs_data = btf->raw_data + btf->hdr->hdr_len;
		747	btf->hdr->str_len = 1; /* empty string at offset 0 */
		748
		749	return btf;
		750	}
		751
462	struct btf btf__new(const void data, __u32 size)	752	struct btf btf__new(const void data, __u32 size)
463	{	753	{
464	struct btf *btf;	754	struct btf *btf;
@@ -468,26 +758,28 @@ struct btf btf__new(const void data, __u32 size) · 3 unmodified lines
468	if (!btf)	758	if (!btf)
469	return ERR_PTR(-ENOMEM);	759	return ERR_PTR(-ENOMEM);
470		760
471	btf->fd = -1;	761	btf->raw_data = malloc(size);
472		762	if (!btf->raw_data) {
473	btf->data = malloc(size);
474	if (!btf->data) {
475	err = -ENOMEM;	763	err = -ENOMEM;
476	goto done;	764	goto done;
477	}	765	}
		766	memcpy(btf->raw_data, data, size);
		767	btf->raw_size = size;
478		768
479	memcpy(btf->data, data, size);	769	btf->hdr = btf->raw_data;
480	btf->data_size = size;
481
482	err = btf_parse_hdr(btf);	770	err = btf_parse_hdr(btf);
483	if (err)	771	if (err)
484	goto done;	772	goto done;
485		773
		774	btf->strs_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->str_off;
		775	btf->types_data = btf->raw_data + btf->hdr->hdr_len + btf->hdr->type_off;
		776
486	err = btf_parse_str_sec(btf);	777	err = btf_parse_str_sec(btf);
		778	err = err ?: btf_parse_type_sec(btf);
487	if (err)	779	if (err)
488	goto done;	780	goto done;
489		781
490	err = btf_parse_type_sec(btf);	782	btf->fd = -1;
491		783
492	done:	784	done:
493	if (err) {	785	if (err) {
@@ -498,17 +790,6 @@ done: · 4 unmodified lines
498	return btf;	790	return btf;
499	}	791	}
500		792
501	static bool btf_check_endianness(const GElf_Ehdr *ehdr)
502	{
503	#if __BYTE_ORDER == __LITTLE_ENDIAN
504	return ehdr->e_ident[EI_DATA] == ELFDATA2LSB;
505	#elif __BYTE_ORDER == __BIG_ENDIAN
506	return ehdr->e_ident[EI_DATA] == ELFDATA2MSB;
507	#else
508	# error "Unrecognized __BYTE_ORDER__"
509	#endif
510	}
511
512	struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext)	793	struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext)
513	{	794	{
514	Elf_Data btf_data = NULL, btf_ext_data = NULL;	795	Elf_Data btf_data = NULL, btf_ext_data = NULL;
@@ -541,10 +822,6 @@ struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext) · 26 unmodified lines
541	pr_warn("failed to get EHDR from %s\n", path);	822	pr_warn("failed to get EHDR from %s\n", path);
542	goto done;	823	goto done;
543	}	824	}
544	if (!btf_check_endianness(&ehdr)) {
545	pr_warn("non-native ELF endianness is not supported\n");
546	goto done;
547	}
548	if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {	825	if (!elf_rawdata(elf_getscn(elf, ehdr.e_shstrndx), NULL)) {
549	pr_warn("failed to get e_shstrndx from %s\n", path);	826	pr_warn("failed to get e_shstrndx from %s\n", path);
550	goto done;	827	goto done;
@@ -656,7 +933,7 @@ struct btf btf__parse_raw(const char path) · 105 unmodified lines
656	err = -EIO;	933	err = -EIO;
657	goto err_out;	934	goto err_out;
658	}	935	}
659	if (magic != BTF_MAGIC) {	936	if (magic != BTF_MAGIC && magic != bswap_16(BTF_MAGIC)) {
660	/* definitely not a raw BTF */	937	/* definitely not a raw BTF */
661	err = -EPROTO;	938	err = -EPROTO;
662	goto err_out;	939	goto err_out;
@@ -789,7 +1066,7 @@ int btf__finalize_data(struct bpf_object obj, struct btf btf) · 126 unmodified lines
789	__u32 i;	1066	__u32 i;
790		1067
791	for (i = 1; i <= btf->nr_types; i++) {	1068	for (i = 1; i <= btf->nr_types; i++) {
792	struct btf_type *t = btf->types[i];	1069	struct btf_type *t = btf_type_by_id(btf, i);
793		1070
794	/* Loader needs to fix up some of the things compiler	1071	/* Loader needs to fix up some of the things compiler
795	* couldn't get its hands on while emitting BTF. This	1072	* couldn't get its hands on while emitting BTF. This
@@ -806,202 +1083,1245 @@ int btf__finalize_data(struct bpf_object obj, struct btf btf) · 10 unmodified lines
806	return err;	1083	return err;
807	}	1084	}
808		1085
		1086	static void btf_get_raw_data(const struct btf btf, __u32 *size, bool swap_endian);
		1087
809	int btf__load(struct btf *btf)	1088	int btf__load(struct btf *btf)
810	{	1089	{
811	__u32 log_buf_size = 0;	1090	__u32 log_buf_size = 0, raw_size;
812	char *log_buf = NULL;	1091	char *log_buf = NULL;
		1092	void *raw_data;
813	int err = 0;	1093	int err = 0;
814		1094
815	if (btf->fd >= 0)	1095	if (btf->fd >= 0)
816	return -EEXIST;	1096	return -EEXIST;
		1097
		1098	retry_load:
		1099	if (log_buf_size) {
		1100	log_buf = malloc(log_buf_size);
		1101	if (!log_buf)
		1102	return -ENOMEM;
		1103
		1104	*log_buf = 0;
		1105	}
		1106
		1107	raw_data = btf_get_raw_data(btf, &raw_size, false);
		1108	if (!raw_data) {
		1109	err = -ENOMEM;
		1110	goto done;
		1111	}
		1112	/* cache native raw data representation */
		1113	btf->raw_size = raw_size;
		1114	btf->raw_data = raw_data;
		1115
		1116	btf->fd = bpf_load_btf(raw_data, raw_size, log_buf, log_buf_size, false);
		1117	if (btf->fd < 0) {
		1118	if (!log_buf \|\| errno == ENOSPC) {
		1119	log_buf_size = max((__u32)BPF_LOG_BUF_SIZE,
		1120	log_buf_size << 1);
		1121	free(log_buf);
		1122	goto retry_load;
		1123	}
		1124
		1125	err = -errno;
		1126	pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno);
		1127	if (*log_buf)
		1128	pr_warn("%s\n", log_buf);
		1129	goto done;
		1130	}
		1131
		1132	done:
		1133	free(log_buf);
		1134	return err;
		1135	}
		1136
		1137	int btf__fd(const struct btf *btf)
		1138	{
		1139	return btf->fd;
		1140	}
		1141
		1142	void btf__set_fd(struct btf *btf, int fd)
		1143	{
		1144	btf->fd = fd;
		1145	}
		1146
		1147	static void btf_get_raw_data(const struct btf btf, __u32 *size, bool swap_endian)
		1148	{
		1149	struct btf_header *hdr = btf->hdr;
		1150	struct btf_type *t;
		1151	void data, p;
		1152	__u32 data_sz;
		1153	int i;
		1154
		1155	data = swap_endian ? btf->raw_data_swapped : btf->raw_data;
		1156	if (data) {
		1157	*size = btf->raw_size;
		1158	return data;
		1159	}
		1160
		1161	data_sz = hdr->hdr_len + hdr->type_len + hdr->str_len;
		1162	data = calloc(1, data_sz);
		1163	if (!data)
		1164	return NULL;
		1165	p = data;
		1166
		1167	memcpy(p, hdr, hdr->hdr_len);
		1168	if (swap_endian)
		1169	btf_bswap_hdr(p);
		1170	p += hdr->hdr_len;
		1171
		1172	memcpy(p, btf->types_data, hdr->type_len);
		1173	if (swap_endian) {
		1174	for (i = 1; i <= btf->nr_types; i++) {
		1175	t = p + btf->type_offs[i];
		1176	/* btf_bswap_type_rest() relies on native t->info, so
		1177	* we swap base type info after we swapped all the
		1178	* additional information
		1179	*/
		1180	if (btf_bswap_type_rest(t))
		1181	goto err_out;
		1182	btf_bswap_type_base(t);
		1183	}
		1184	}
		1185	p += hdr->type_len;
		1186
		1187	memcpy(p, btf->strs_data, hdr->str_len);
		1188	p += hdr->str_len;
		1189
		1190	*size = data_sz;
		1191	return data;
		1192	err_out:
		1193	free(data);
		1194	return NULL;
		1195	}
		1196
		1197	const void btf__get_raw_data(const struct btf btf_ro, __u32 *size)
		1198	{
		1199	struct btf btf = (struct btf )btf_ro;
		1200	__u32 data_sz;
		1201	void *data;
		1202
		1203	data = btf_get_raw_data(btf, &data_sz, btf->swapped_endian);
		1204	if (!data)
		1205	return NULL;
		1206
		1207	btf->raw_size = data_sz;
		1208	if (btf->swapped_endian)
		1209	btf->raw_data_swapped = data;
		1210	else
		1211	btf->raw_data = data;
		1212	*size = data_sz;
		1213	return data;
		1214	}
		1215
		1216	const char btf__str_by_offset(const struct btf btf, __u32 offset)
		1217	{
		1218	if (offset < btf->hdr->str_len)
		1219	return btf->strs_data + offset;
		1220	else
		1221	return NULL;
		1222	}
		1223
		1224	const char btf__name_by_offset(const struct btf btf, __u32 offset)
		1225	{
		1226	return btf__str_by_offset(btf, offset);
		1227	}
		1228
		1229	int btf__get_from_id(__u32 id, struct btf **btf)
		1230	{
		1231	struct bpf_btf_info btf_info = { 0 };
		1232	__u32 len = sizeof(btf_info);
		1233	__u32 last_size;
		1234	int btf_fd;
		1235	void *ptr;
		1236	int err;
		1237
		1238	err = 0;
		1239	*btf = NULL;
		1240	btf_fd = bpf_btf_get_fd_by_id(id);
		1241	if (btf_fd < 0)
		1242	return 0;
		1243
		1244	/* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so
		1245	* let's start with a sane default - 4KiB here - and resize it only if
		1246	* bpf_obj_get_info_by_fd() needs a bigger buffer.
		1247	*/
		1248	btf_info.btf_size = 4096;
		1249	last_size = btf_info.btf_size;
		1250	ptr = malloc(last_size);
		1251	if (!ptr) {
		1252	err = -ENOMEM;
		1253	goto exit_free;
		1254	}
		1255
		1256	memset(ptr, 0, last_size);
		1257	btf_info.btf = ptr_to_u64(ptr);
		1258	err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
		1259
		1260	if (!err && btf_info.btf_size > last_size) {
		1261	void *temp_ptr;
		1262
		1263	last_size = btf_info.btf_size;
		1264	temp_ptr = realloc(ptr, last_size);
		1265	if (!temp_ptr) {
		1266	err = -ENOMEM;
		1267	goto exit_free;
		1268	}
		1269	ptr = temp_ptr;
		1270	memset(ptr, 0, last_size);
		1271	btf_info.btf = ptr_to_u64(ptr);
		1272	err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
		1273	}
		1274
		1275	if (err \|\| btf_info.btf_size > last_size) {
		1276	err = errno;
		1277	goto exit_free;
		1278	}
		1279
		1280	btf = btf__new((__u8 )(long)btf_info.btf, btf_info.btf_size);
		1281	if (IS_ERR(*btf)) {
		1282	err = PTR_ERR(*btf);
		1283	*btf = NULL;
		1284	}
		1285
		1286	exit_free:
		1287	close(btf_fd);
		1288	free(ptr);
		1289
		1290	return err;
		1291	}
		1292
		1293	int btf__get_map_kv_tids(const struct btf btf, const char map_name,
		1294	__u32 expected_key_size, __u32 expected_value_size,
		1295	__u32 key_type_id, __u32 value_type_id)
		1296	{
		1297	const struct btf_type *container_type;
		1298	const struct btf_member key, value;
		1299	const size_t max_name = 256;
		1300	char container_name[max_name];
		1301	__s64 key_size, value_size;
		1302	__s32 container_id;
		1303
		1304	if (snprintf(container_name, max_name, "____btf_map_%s", map_name) ==
		1305	max_name) {
		1306	pr_warn("map:%s length of '____btf_map_%s' is too long\n",
		1307	map_name, map_name);
		1308	return -EINVAL;
		1309	}
		1310
		1311	container_id = btf__find_by_name(btf, container_name);
		1312	if (container_id < 0) {
		1313	pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
		1314	map_name, container_name);
		1315	return container_id;
		1316	}
		1317
		1318	container_type = btf__type_by_id(btf, container_id);
		1319	if (!container_type) {
		1320	pr_warn("map:%s cannot find BTF type for container_id:%u\n",
		1321	map_name, container_id);
		1322	return -EINVAL;
		1323	}
		1324
		1325	if (!btf_is_struct(container_type) \|\| btf_vlen(container_type) < 2) {
		1326	pr_warn("map:%s container_name:%s is an invalid container struct\n",
		1327	map_name, container_name);
		1328	return -EINVAL;
		1329	}
		1330
		1331	key = btf_members(container_type);
		1332	value = key + 1;
		1333
		1334	key_size = btf__resolve_size(btf, key->type);
		1335	if (key_size < 0) {
		1336	pr_warn("map:%s invalid BTF key_type_size\n", map_name);
		1337	return key_size;
		1338	}
		1339
		1340	if (expected_key_size != key_size) {
		1341	pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
		1342	map_name, (__u32)key_size, expected_key_size);
		1343	return -EINVAL;
		1344	}
		1345
		1346	value_size = btf__resolve_size(btf, value->type);
		1347	if (value_size < 0) {
		1348	pr_warn("map:%s invalid BTF value_type_size\n", map_name);
		1349	return value_size;
		1350	}
		1351
		1352	if (expected_value_size != value_size) {
		1353	pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
		1354	map_name, (__u32)value_size, expected_value_size);
		1355	return -EINVAL;
		1356	}
		1357
		1358	*key_type_id = key->type;
		1359	*value_type_id = value->type;
		1360
		1361	return 0;
		1362	}
		1363
		1364	static size_t strs_hash_fn(const void key, void ctx)
		1365	{
		1366	struct btf *btf = ctx;
		1367	const char *str = btf->strs_data + (long)key;
		1368
		1369	return str_hash(str);
		1370	}
		1371
		1372	static bool strs_hash_equal_fn(const void key1, const void key2, void *ctx)
		1373	{
		1374	struct btf *btf = ctx;
		1375	const char *str1 = btf->strs_data + (long)key1;
		1376	const char *str2 = btf->strs_data + (long)key2;
		1377
		1378	return strcmp(str1, str2) == 0;
		1379	}
		1380
		1381	static void btf_invalidate_raw_data(struct btf *btf)
		1382	{
		1383	if (btf->raw_data) {
		1384	free(btf->raw_data);
		1385	btf->raw_data = NULL;
		1386	}
		1387	if (btf->raw_data_swapped) {
		1388	free(btf->raw_data_swapped);
		1389	btf->raw_data_swapped = NULL;
		1390	}
		1391	}
		1392
		1393	/* Ensure BTF is ready to be modified (by splitting into a three memory
		1394	* regions for header, types, and strings). Also invalidate cached
		1395	* raw_data, if any.
		1396	*/
		1397	static int btf_ensure_modifiable(struct btf *btf)
		1398	{
		1399	void hdr, types, strs, strs_end, *s;
		1400	struct hashmap *hash = NULL;
		1401	long off;
		1402	int err;
		1403
		1404	if (btf_is_modifiable(btf)) {
		1405	/* any BTF modification invalidates raw_data */
		1406	btf_invalidate_raw_data(btf);
		1407	return 0;
		1408	}
		1409
		1410	/* split raw data into three memory regions */
		1411	hdr = malloc(btf->hdr->hdr_len);
		1412	types = malloc(btf->hdr->type_len);
		1413	strs = malloc(btf->hdr->str_len);
		1414	if (!hdr \|\| !types \|\| !strs)
		1415	goto err_out;
		1416
		1417	memcpy(hdr, btf->hdr, btf->hdr->hdr_len);
		1418	memcpy(types, btf->types_data, btf->hdr->type_len);
		1419	memcpy(strs, btf->strs_data, btf->hdr->str_len);
		1420
		1421	/* build lookup index for all strings */
		1422	hash = hashmap__new(strs_hash_fn, strs_hash_equal_fn, btf);
		1423	if (IS_ERR(hash)) {
		1424	err = PTR_ERR(hash);
		1425	hash = NULL;
		1426	goto err_out;
		1427	}
		1428
		1429	strs_end = strs + btf->hdr->str_len;
		1430	for (off = 0, s = strs; s < strs_end; off += strlen(s) + 1, s = strs + off) {
		1431	/* hashmap__add() returns EEXIST if string with the same
		1432	* content already is in the hash map
		1433	*/
		1434	err = hashmap__add(hash, (void )off, (void )off);
		1435	if (err == -EEXIST)
		1436	continue; /* duplicate */
		1437	if (err)
		1438	goto err_out;
		1439	}
		1440
		1441	/* only when everything was successful, update internal state */
		1442	btf->hdr = hdr;
		1443	btf->types_data = types;
		1444	btf->types_data_cap = btf->hdr->type_len;
		1445	btf->strs_data = strs;
		1446	btf->strs_data_cap = btf->hdr->str_len;
		1447	btf->strs_hash = hash;
		1448	/* if BTF was created from scratch, all strings are guaranteed to be
		1449	* unique and deduplicated
		1450	*/
		1451	btf->strs_deduped = btf->hdr->str_len <= 1;
		1452
		1453	/* invalidate raw_data representation */
		1454	btf_invalidate_raw_data(btf);
		1455
		1456	return 0;
		1457
		1458	err_out:
		1459	hashmap__free(hash);
		1460	free(hdr);
		1461	free(types);
		1462	free(strs);
		1463	return -ENOMEM;
		1464	}
		1465
		1466	static void btf_add_str_mem(struct btf btf, size_t add_sz)
		1467	{
		1468	return btf_add_mem(&btf->strs_data, &btf->strs_data_cap, 1,
		1469	btf->hdr->str_len, BTF_MAX_STR_OFFSET, add_sz);
		1470	}
		1471
		1472	/* Find an offset in BTF string section that corresponds to a given string s.
		1473	* Returns:
		1474	* - >0 offset into string section, if string is found;
		1475	* - -ENOENT, if string is not in the string section;
		1476	* - <0, on any other error.
		1477	*/
		1478	int btf__find_str(struct btf btf, const char s)
		1479	{
		1480	long old_off, new_off, len;
		1481	void *p;
		1482
		1483	/* BTF needs to be in a modifiable state to build string lookup index */
		1484	if (btf_ensure_modifiable(btf))
		1485	return -ENOMEM;
		1486
		1487	/* see btf__add_str() for why we do this */
		1488	len = strlen(s) + 1;
		1489	p = btf_add_str_mem(btf, len);
		1490	if (!p)
		1491	return -ENOMEM;
		1492
		1493	new_off = btf->hdr->str_len;
		1494	memcpy(p, s, len);
		1495
		1496	if (hashmap__find(btf->strs_hash, (void )new_off, (void *)&old_off))
		1497	return old_off;
		1498
		1499	return -ENOENT;
		1500	}
		1501
		1502	/* Add a string s to the BTF string section.
		1503	* Returns:
		1504	* - > 0 offset into string section, on success;
		1505	* - < 0, on error.
		1506	*/
		1507	int btf__add_str(struct btf btf, const char s)
		1508	{
		1509	long old_off, new_off, len;
		1510	void *p;
		1511	int err;
		1512
		1513	if (btf_ensure_modifiable(btf))
		1514	return -ENOMEM;
		1515
		1516	/* Hashmap keys are always offsets within btf->strs_data, so to even
		1517	* look up some string from the "outside", we need to first append it
		1518	* at the end, so that it can be addressed with an offset. Luckily,
		1519	* until btf->hdr->str_len is incremented, that string is just a piece
		1520	* of garbage for the rest of BTF code, so no harm, no foul. On the
		1521	* other hand, if the string is unique, it's already appended and
		1522	* ready to be used, only a simple btf->hdr->str_len increment away.
		1523	*/
		1524	len = strlen(s) + 1;
		1525	p = btf_add_str_mem(btf, len);
		1526	if (!p)
		1527	return -ENOMEM;
		1528
		1529	new_off = btf->hdr->str_len;
		1530	memcpy(p, s, len);
		1531
		1532	/* Now attempt to add the string, but only if the string with the same
		1533	* contents doesn't exist already (HASHMAP_ADD strategy). If such
		1534	* string exists, we'll get its offset in old_off (that's old_key).
		1535	*/
		1536	err = hashmap__insert(btf->strs_hash, (void )new_off, (void )new_off,
		1537	HASHMAP_ADD, (const void **)&old_off, NULL);
		1538	if (err == -EEXIST)
		1539	return old_off; /* duplicated string, return existing offset */
		1540	if (err)
		1541	return err;
		1542
		1543	btf->hdr->str_len += len; /* new unique string, adjust data length */
		1544	return new_off;
		1545	}
		1546
		1547	static void btf_add_type_mem(struct btf btf, size_t add_sz)
		1548	{
		1549	return btf_add_mem(&btf->types_data, &btf->types_data_cap, 1,
		1550	btf->hdr->type_len, UINT_MAX, add_sz);
		1551	}
		1552
		1553	static __u32 btf_type_info(int kind, int vlen, int kflag)
		1554	{
		1555	return (kflag << 31) \| (kind << 24) \| vlen;
		1556	}
		1557
		1558	static void btf_type_inc_vlen(struct btf_type *t)
		1559	{
		1560	t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, btf_kflag(t));
		1561	}
		1562
		1563	/*
		1564	* Append new BTF_KIND_INT type with:
		1565	* - name - non-empty, non-NULL type name;
		1566	* - sz - power-of-2 (1, 2, 4, ..) size of the type, in bytes;
		1567	* - encoding is a combination of BTF_INT_SIGNED, BTF_INT_CHAR, BTF_INT_BOOL.
		1568	* Returns:
		1569	* - >0, type ID of newly added BTF type;
		1570	* - <0, on error.
		1571	*/
		1572	int btf__add_int(struct btf btf, const char name, size_t byte_sz, int encoding)
		1573	{
		1574	struct btf_type *t;
		1575	int sz, err, name_off;
		1576
		1577	/* non-empty name */
		1578	if (!name \|\| !name[0])
		1579	return -EINVAL;
		1580	/* byte_sz must be power of 2 */
		1581	if (!byte_sz \|\| (byte_sz & (byte_sz - 1)) \|\| byte_sz > 16)
		1582	return -EINVAL;
		1583	if (encoding & ~(BTF_INT_SIGNED \| BTF_INT_CHAR \| BTF_INT_BOOL))
		1584	return -EINVAL;
		1585
		1586	/* deconstruct BTF, if necessary, and invalidate raw_data */
		1587	if (btf_ensure_modifiable(btf))
		1588	return -ENOMEM;
		1589
		1590	sz = sizeof(struct btf_type) + sizeof(int);
		1591	t = btf_add_type_mem(btf, sz);
		1592	if (!t)
		1593	return -ENOMEM;
		1594
		1595	/* if something goes wrong later, we might end up with an extra string,
		1596	* but that shouldn't be a problem, because BTF can't be constructed
		1597	* completely anyway and will most probably be just discarded
		1598	*/
		1599	name_off = btf__add_str(btf, name);
		1600	if (name_off < 0)
		1601	return name_off;
		1602
		1603	t->name_off = name_off;
		1604	t->info = btf_type_info(BTF_KIND_INT, 0, 0);
		1605	t->size = byte_sz;
		1606	/* set INT info, we don't allow setting legacy bit offset/size */
		1607	(__u32 )(t + 1) = (encoding << 24) \| (byte_sz * 8);
		1608
		1609	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
		1610	if (err)
		1611	return err;
		1612
		1613	btf->hdr->type_len += sz;
		1614	btf->hdr->str_off += sz;
		1615	btf->nr_types++;
		1616	return btf->nr_types;
		1617	}
		1618
		1619	/* it's completely legal to append BTF types with type IDs pointing forward to
		1620	* types that haven't been appended yet, so we only make sure that id looks
		1621	* sane, we can't guarantee that ID will always be valid
		1622	*/
		1623	static int validate_type_id(int id)
		1624	{
		1625	if (id < 0 \|\| id > BTF_MAX_NR_TYPES)
		1626	return -EINVAL;
		1627	return 0;
		1628	}
		1629
		1630	/* generic append function for PTR, TYPEDEF, CONST/VOLATILE/RESTRICT */
		1631	static int btf_add_ref_kind(struct btf btf, int kind, const char name, int ref_type_id)
		1632	{
		1633	struct btf_type *t;
		1634	int sz, name_off = 0, err;
		1635
		1636	if (validate_type_id(ref_type_id))
		1637	return -EINVAL;
		1638
		1639	if (btf_ensure_modifiable(btf))
		1640	return -ENOMEM;
		1641
		1642	sz = sizeof(struct btf_type);
		1643	t = btf_add_type_mem(btf, sz);
		1644	if (!t)
		1645	return -ENOMEM;
		1646
		1647	if (name && name[0]) {
		1648	name_off = btf__add_str(btf, name);
		1649	if (name_off < 0)
		1650	return name_off;
		1651	}
		1652
		1653	t->name_off = name_off;
		1654	t->info = btf_type_info(kind, 0, 0);
		1655	t->type = ref_type_id;
		1656
		1657	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
		1658	if (err)
		1659	return err;
		1660
		1661	btf->hdr->type_len += sz;
		1662	btf->hdr->str_off += sz;
		1663	btf->nr_types++;
		1664	return btf->nr_types;
		1665	}
		1666
		1667	/*
		1668	* Append new BTF_KIND_PTR type with:
		1669	* - ref_type_id - referenced type ID, it might not exist yet;
		1670	* Returns:
		1671	* - >0, type ID of newly added BTF type;
		1672	* - <0, on error.
		1673	*/
		1674	int btf__add_ptr(struct btf *btf, int ref_type_id)
		1675	{
		1676	return btf_add_ref_kind(btf, BTF_KIND_PTR, NULL, ref_type_id);
		1677	}
		1678
		1679	/*
		1680	* Append new BTF_KIND_ARRAY type with:
		1681	* - index_type_id - type ID of the type describing array index;
		1682	* - elem_type_id - type ID of the type describing array element;
		1683	* - nr_elems - the size of the array;
		1684	* Returns:
		1685	* - >0, type ID of newly added BTF type;
		1686	* - <0, on error.
		1687	*/
		1688	int btf__add_array(struct btf *btf, int index_type_id, int elem_type_id, __u32 nr_elems)
		1689	{
		1690	struct btf_type *t;
		1691	struct btf_array *a;
		1692	int sz, err;
		1693
		1694	if (validate_type_id(index_type_id) \|\| validate_type_id(elem_type_id))
		1695	return -EINVAL;
		1696
		1697	if (btf_ensure_modifiable(btf))
		1698	return -ENOMEM;
		1699
		1700	sz = sizeof(struct btf_type) + sizeof(struct btf_array);
		1701	t = btf_add_type_mem(btf, sz);
		1702	if (!t)
		1703	return -ENOMEM;
		1704
		1705	t->name_off = 0;
		1706	t->info = btf_type_info(BTF_KIND_ARRAY, 0, 0);
		1707	t->size = 0;
		1708
		1709	a = btf_array(t);
		1710	a->type = elem_type_id;
		1711	a->index_type = index_type_id;
		1712	a->nelems = nr_elems;
		1713
		1714	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
		1715	if (err)
		1716	return err;
		1717
		1718	btf->hdr->type_len += sz;
		1719	btf->hdr->str_off += sz;
		1720	btf->nr_types++;
		1721	return btf->nr_types;
		1722	}
		1723
		1724	/* generic STRUCT/UNION append function */
		1725	static int btf_add_composite(struct btf btf, int kind, const char name, __u32 bytes_sz)
		1726	{
		1727	struct btf_type *t;
		1728	int sz, err, name_off = 0;
		1729
		1730	if (btf_ensure_modifiable(btf))
		1731	return -ENOMEM;
		1732
		1733	sz = sizeof(struct btf_type);
		1734	t = btf_add_type_mem(btf, sz);
		1735	if (!t)
		1736	return -ENOMEM;
		1737
		1738	if (name && name[0]) {
		1739	name_off = btf__add_str(btf, name);
		1740	if (name_off < 0)
		1741	return name_off;
		1742	}
		1743
		1744	/* start out with vlen=0 and no kflag; this will be adjusted when
		1745	* adding each member
		1746	*/
		1747	t->name_off = name_off;
		1748	t->info = btf_type_info(kind, 0, 0);
		1749	t->size = bytes_sz;
		1750
		1751	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
		1752	if (err)
		1753	return err;
		1754
		1755	btf->hdr->type_len += sz;
		1756	btf->hdr->str_off += sz;
		1757	btf->nr_types++;
		1758	return btf->nr_types;
		1759	}
		1760
		1761	/*
		1762	* Append new BTF_KIND_STRUCT type with:
		1763	* - name - name of the struct, can be NULL or empty for anonymous structs;
		1764	* - byte_sz - size of the struct, in bytes;
		1765	*
		1766	* Struct initially has no fields in it. Fields can be added by
		1767	* btf__add_field() right after btf__add_struct() succeeds.
		1768	*
		1769	* Returns:
		1770	* - >0, type ID of newly added BTF type;
		1771	* - <0, on error.
		1772	*/
		1773	int btf__add_struct(struct btf btf, const char name, __u32 byte_sz)
		1774	{
		1775	return btf_add_composite(btf, BTF_KIND_STRUCT, name, byte_sz);
		1776	}
		1777
		1778	/*
		1779	* Append new BTF_KIND_UNION type with:
		1780	* - name - name of the union, can be NULL or empty for anonymous union;
		1781	* - byte_sz - size of the union, in bytes;
		1782	*
		1783	* Union initially has no fields in it. Fields can be added by
		1784	* btf__add_field() right after btf__add_union() succeeds. All fields
		1785	* should have bit_offset of 0.
		1786	*
		1787	* Returns:
		1788	* - >0, type ID of newly added BTF type;
		1789	* - <0, on error.
		1790	*/
		1791	int btf__add_union(struct btf btf, const char name, __u32 byte_sz)
		1792	{
		1793	return btf_add_composite(btf, BTF_KIND_UNION, name, byte_sz);
		1794	}
		1795
		1796	/*
		1797	* Append new field for the current STRUCT/UNION type with:
		1798	* - name - name of the field, can be NULL or empty for anonymous field;
		1799	* - type_id - type ID for the type describing field type;
		1800	* - bit_offset - bit offset of the start of the field within struct/union;
		1801	* - bit_size - bit size of a bitfield, 0 for non-bitfield fields;
		1802	* Returns:
		1803	* - 0, on success;
		1804	* - <0, on error.
		1805	*/
		1806	int btf__add_field(struct btf btf, const char name, int type_id,
		1807	__u32 bit_offset, __u32 bit_size)
		1808	{
		1809	struct btf_type *t;
		1810	struct btf_member *m;
		1811	bool is_bitfield;
		1812	int sz, name_off = 0;
		1813
		1814	/* last type should be union/struct */
		1815	if (btf->nr_types == 0)
		1816	return -EINVAL;
		1817	t = btf_type_by_id(btf, btf->nr_types);
		1818	if (!btf_is_composite(t))
		1819	return -EINVAL;
		1820
		1821	if (validate_type_id(type_id))
		1822	return -EINVAL;
		1823	/* best-effort bit field offset/size enforcement */
		1824	is_bitfield = bit_size \|\| (bit_offset % 8 != 0);
		1825	if (is_bitfield && (bit_size == 0 \|\| bit_size > 255 \|\| bit_offset > 0xffffff))
		1826	return -EINVAL;
		1827
		1828	/* only offset 0 is allowed for unions */
		1829	if (btf_is_union(t) && bit_offset)
		1830	return -EINVAL;
		1831
		1832	/* decompose and invalidate raw data */
		1833	if (btf_ensure_modifiable(btf))
		1834	return -ENOMEM;
		1835
		1836	sz = sizeof(struct btf_member);
		1837	m = btf_add_type_mem(btf, sz);
		1838	if (!m)
		1839	return -ENOMEM;
		1840
		1841	if (name && name[0]) {
		1842	name_off = btf__add_str(btf, name);
		1843	if (name_off < 0)
		1844	return name_off;
		1845	}
		1846
		1847	m->name_off = name_off;
		1848	m->type = type_id;
		1849	m->offset = bit_offset \| (bit_size << 24);
		1850
		1851	/* btf_add_type_mem can invalidate t pointer */
		1852	t = btf_type_by_id(btf, btf->nr_types);
		1853	/* update parent type's vlen and kflag */
		1854	t->info = btf_type_info(btf_kind(t), btf_vlen(t) + 1, is_bitfield \|\| btf_kflag(t));
		1855
		1856	btf->hdr->type_len += sz;
		1857	btf->hdr->str_off += sz;
		1858	return 0;
		1859	}
		1860
		1861	/*
		1862	* Append new BTF_KIND_ENUM type with:
		1863	* - name - name of the enum, can be NULL or empty for anonymous enums;
		1864	* - byte_sz - size of the enum, in bytes.
		1865	*
		1866	* Enum initially has no enum values in it (and corresponds to enum forward
		1867	* declaration). Enumerator values can be added by btf__add_enum_value()
		1868	* immediately after btf__add_enum() succeeds.
		1869	*
		1870	* Returns:
		1871	* - >0, type ID of newly added BTF type;
		1872	* - <0, on error.
		1873	*/
		1874	int btf__add_enum(struct btf btf, const char name, __u32 byte_sz)
		1875	{
		1876	struct btf_type *t;
		1877	int sz, err, name_off = 0;
		1878
		1879	/* byte_sz must be power of 2 */
		1880	if (!byte_sz \|\| (byte_sz & (byte_sz - 1)) \|\| byte_sz > 8)
		1881	return -EINVAL;
		1882
		1883	if (btf_ensure_modifiable(btf))
		1884	return -ENOMEM;
		1885
		1886	sz = sizeof(struct btf_type);
		1887	t = btf_add_type_mem(btf, sz);
		1888	if (!t)
		1889	return -ENOMEM;
		1890
		1891	if (name && name[0]) {
		1892	name_off = btf__add_str(btf, name);
		1893	if (name_off < 0)
		1894	return name_off;
		1895	}
		1896
		1897	/* start out with vlen=0; it will be adjusted when adding enum values */
		1898	t->name_off = name_off;
		1899	t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);
		1900	t->size = byte_sz;
		1901
		1902	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
		1903	if (err)
		1904	return err;
		1905
		1906	btf->hdr->type_len += sz;
		1907	btf->hdr->str_off += sz;
		1908	btf->nr_types++;
		1909	return btf->nr_types;
		1910	}
		1911
		1912	/*
		1913	* Append new enum value for the current ENUM type with:
		1914	* - name - name of the enumerator value, can't be NULL or empty;
		1915	* - value - integer value corresponding to enum value name;
		1916	* Returns:
		1917	* - 0, on success;
		1918	* - <0, on error.
		1919	*/
		1920	int btf__add_enum_value(struct btf btf, const char name, __s64 value)
		1921	{
		1922	struct btf_type *t;
		1923	struct btf_enum *v;
		1924	int sz, name_off;
		1925
		1926	/* last type should be BTF_KIND_ENUM */
		1927	if (btf->nr_types == 0)
		1928	return -EINVAL;
		1929	t = btf_type_by_id(btf, btf->nr_types);
		1930	if (!btf_is_enum(t))
		1931	return -EINVAL;
		1932
		1933	/* non-empty name */
		1934	if (!name \|\| !name[0])
		1935	return -EINVAL;
		1936	if (value < INT_MIN \|\| value > UINT_MAX)
		1937	return -E2BIG;
		1938
		1939	/* decompose and invalidate raw data */
		1940	if (btf_ensure_modifiable(btf))
		1941	return -ENOMEM;
		1942
		1943	sz = sizeof(struct btf_enum);
		1944	v = btf_add_type_mem(btf, sz);
		1945	if (!v)
		1946	return -ENOMEM;
817		1947
818	retry_load:	1948	name_off = btf__add_str(btf, name);
819	if (log_buf_size) {	1949	if (name_off < 0)
820	log_buf = malloc(log_buf_size);	1950	return name_off;
821	if (!log_buf)
822	return -ENOMEM;
823		1951
824	*log_buf = 0;	1952	v->name_off = name_off;
825	}	1953	v->val = value;
826		1954
827	btf->fd = bpf_load_btf(btf->data, btf->data_size,	1955	/* update parent type's vlen */
828	log_buf, log_buf_size, false);	1956	t = btf_type_by_id(btf, btf->nr_types);
829	if (btf->fd < 0) {	1957	btf_type_inc_vlen(t);
830	if (!log_buf \|\| errno == ENOSPC) {
831	log_buf_size = max((__u32)BPF_LOG_BUF_SIZE,
832	log_buf_size << 1);
833	free(log_buf);
834	goto retry_load;
835	}
836		1958
837	err = -errno;	1959	btf->hdr->type_len += sz;
838	pr_warn("Error loading BTF: %s(%d)\n", strerror(errno), errno);	1960	btf->hdr->str_off += sz;
839	if (*log_buf)	1961	return 0;
840	pr_warn("%s\n", log_buf);	1962	}
841	goto done;
842	}
843		1963
844	done:	1964	/*
845	free(log_buf);	1965	* Append new BTF_KIND_FWD type with:
846	return err;	1966	* - name, non-empty/non-NULL name;
		1967	* - fwd_kind, kind of forward declaration, one of BTF_FWD_STRUCT,
		1968	* BTF_FWD_UNION, or BTF_FWD_ENUM;
		1969	* Returns:
		1970	* - >0, type ID of newly added BTF type;
		1971	* - <0, on error.
		1972	*/
		1973	int btf__add_fwd(struct btf btf, const char name, enum btf_fwd_kind fwd_kind)
		1974	{
		1975	if (!name \|\| !name[0])
		1976	return -EINVAL;
		1977
		1978	switch (fwd_kind) {
		1979	case BTF_FWD_STRUCT:
		1980	case BTF_FWD_UNION: {
		1981	struct btf_type *t;
		1982	int id;
		1983
		1984	id = btf_add_ref_kind(btf, BTF_KIND_FWD, name, 0);
		1985	if (id <= 0)
		1986	return id;
		1987	t = btf_type_by_id(btf, id);
		1988	t->info = btf_type_info(BTF_KIND_FWD, 0, fwd_kind == BTF_FWD_UNION);
		1989	return id;
		1990	}
		1991	case BTF_FWD_ENUM:
		1992	/* enum forward in BTF currently is just an enum with no enum
		1993	* values; we also assume a standard 4-byte size for it
		1994	*/
		1995	return btf__add_enum(btf, name, sizeof(int));
		1996	default:
		1997	return -EINVAL;
		1998	}
847	}	1999	}
848		2000
849	int btf__fd(const struct btf *btf)	2001	/*
		2002	* Append new BTF_KING_TYPEDEF type with:
		2003	* - name, non-empty/non-NULL name;
		2004	* - ref_type_id - referenced type ID, it might not exist yet;
		2005	* Returns:
		2006	* - >0, type ID of newly added BTF type;
		2007	* - <0, on error.
		2008	*/
		2009	int btf__add_typedef(struct btf btf, const char name, int ref_type_id)
850	{	2010	{
851	return btf->fd;	2011	if (!name \|\| !name[0])
		2012	return -EINVAL;
		2013
		2014	return btf_add_ref_kind(btf, BTF_KIND_TYPEDEF, name, ref_type_id);
852	}	2015	}
853		2016
854	void btf__set_fd(struct btf *btf, int fd)	2017	/*
		2018	* Append new BTF_KIND_VOLATILE type with:
		2019	* - ref_type_id - referenced type ID, it might not exist yet;
		2020	* Returns:
		2021	* - >0, type ID of newly added BTF type;
		2022	* - <0, on error.
		2023	*/
		2024	int btf__add_volatile(struct btf *btf, int ref_type_id)
855	{	2025	{
856	btf->fd = fd;	2026	return btf_add_ref_kind(btf, BTF_KIND_VOLATILE, NULL, ref_type_id);
857	}	2027	}
858		2028
859	const void btf__get_raw_data(const struct btf btf, __u32 *size)	2029	/*
		2030	* Append new BTF_KIND_CONST type with:
		2031	* - ref_type_id - referenced type ID, it might not exist yet;
		2032	* Returns:
		2033	* - >0, type ID of newly added BTF type;
		2034	* - <0, on error.
		2035	*/
		2036	int btf__add_const(struct btf *btf, int ref_type_id)
860	{	2037	{
861	*size = btf->data_size;	2038	return btf_add_ref_kind(btf, BTF_KIND_CONST, NULL, ref_type_id);
862	return btf->data;
863	}	2039	}
864		2040
865	const char btf__name_by_offset(const struct btf btf, __u32 offset)	2041	/*
		2042	* Append new BTF_KIND_RESTRICT type with:
		2043	* - ref_type_id - referenced type ID, it might not exist yet;
		2044	* Returns:
		2045	* - >0, type ID of newly added BTF type;
		2046	* - <0, on error.
		2047	*/
		2048	int btf__add_restrict(struct btf *btf, int ref_type_id)
866	{	2049	{
867	if (offset < btf->hdr->str_len)	2050	return btf_add_ref_kind(btf, BTF_KIND_RESTRICT, NULL, ref_type_id);
868	return &btf->strings[offset];
869	else
870	return NULL;
871	}	2051	}
872		2052
873	int btf__get_from_id(__u32 id, struct btf **btf)	2053	/*
		2054	* Append new BTF_KIND_FUNC type with:
		2055	* - name, non-empty/non-NULL name;
		2056	* - proto_type_id - FUNC_PROTO's type ID, it might not exist yet;
		2057	* Returns:
		2058	* - >0, type ID of newly added BTF type;
		2059	* - <0, on error.
		2060	*/
		2061	int btf__add_func(struct btf btf, const char name,
		2062	enum btf_func_linkage linkage, int proto_type_id)
874	{	2063	{
875	struct bpf_btf_info btf_info = { 0 };	2064	int id;
876	__u32 len = sizeof(btf_info);
877	__u32 last_size;
878	int btf_fd;
879	void *ptr;
880	int err;
881		2065
882	err = 0;	2066	if (!name \|\| !name[0])
883	*btf = NULL;	2067	return -EINVAL;
884	btf_fd = bpf_btf_get_fd_by_id(id);	2068	if (linkage != BTF_FUNC_STATIC && linkage != BTF_FUNC_GLOBAL &&
885	if (btf_fd < 0)	2069	linkage != BTF_FUNC_EXTERN)
886	return 0;	2070	return -EINVAL;
887		2071
888	/* we won't know btf_size until we call bpf_obj_get_info_by_fd(). so	2072	id = btf_add_ref_kind(btf, BTF_KIND_FUNC, name, proto_type_id);
889	* let's start with a sane default - 4KiB here - and resize it only if	2073	if (id > 0) {
890	* bpf_obj_get_info_by_fd() needs a bigger buffer.	2074	struct btf_type *t = btf_type_by_id(btf, id);
891	*/	2075
892	btf_info.btf_size = 4096;	2076	t->info = btf_type_info(BTF_KIND_FUNC, linkage, 0);
893	last_size = btf_info.btf_size;
894	ptr = malloc(last_size);
895	if (!ptr) {
896	err = -ENOMEM;
897	goto exit_free;
898	}	2077	}
		2078	return id;
		2079	}
899		2080
900	memset(ptr, 0, last_size);	2081	/*
901	btf_info.btf = ptr_to_u64(ptr);	2082	* Append new BTF_KIND_FUNC_PROTO with:
902	err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);	2083	* - ret_type_id - type ID for return result of a function.
		2084	*
		2085	* Function prototype initially has no arguments, but they can be added by
		2086	* btf__add_func_param() one by one, immediately after
		2087	* btf__add_func_proto() succeeded.
		2088	*
		2089	* Returns:
		2090	* - >0, type ID of newly added BTF type;
		2091	* - <0, on error.
		2092	*/
		2093	int btf__add_func_proto(struct btf *btf, int ret_type_id)
		2094	{
		2095	struct btf_type *t;
		2096	int sz, err;
903		2097
904	if (!err && btf_info.btf_size > last_size) {	2098	if (validate_type_id(ret_type_id))
905	void *temp_ptr;	2099	return -EINVAL;
906		2100
907	last_size = btf_info.btf_size;	2101	if (btf_ensure_modifiable(btf))
908	temp_ptr = realloc(ptr, last_size);	2102	return -ENOMEM;
909	if (!temp_ptr) {
910	err = -ENOMEM;
911	goto exit_free;
912	}
913	ptr = temp_ptr;
914	memset(ptr, 0, last_size);
915	btf_info.btf = ptr_to_u64(ptr);
916	err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len);
917	}
918		2103
919	if (err \|\| btf_info.btf_size > last_size) {	2104	sz = sizeof(struct btf_type);
920	err = errno;	2105	t = btf_add_type_mem(btf, sz);
921	goto exit_free;	2106	if (!t)
922	}	2107	return -ENOMEM;
923		2108
924	btf = btf__new((__u8 )(long)btf_info.btf, btf_info.btf_size);	2109	/* start out with vlen=0; this will be adjusted when adding enum
925	if (IS_ERR(*btf)) {	2110	* values, if necessary
926	err = PTR_ERR(*btf);	2111	*/
927	*btf = NULL;	2112	t->name_off = 0;
928	}	2113	t->info = btf_type_info(BTF_KIND_FUNC_PROTO, 0, 0);
		2114	t->type = ret_type_id;
929		2115
930	exit_free:	2116	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
931	close(btf_fd);	2117	if (err)
932	free(ptr);	2118	return err;
933		2119
934	return err;	2120	btf->hdr->type_len += sz;
		2121	btf->hdr->str_off += sz;
		2122	btf->nr_types++;
		2123	return btf->nr_types;
935	}	2124	}
936		2125
937	int btf__get_map_kv_tids(const struct btf btf, const char map_name,	2126	/*
938	__u32 expected_key_size, __u32 expected_value_size,	2127	* Append new function parameter for current FUNC_PROTO type with:
939	__u32 key_type_id, __u32 value_type_id)	2128	* - name - parameter name, can be NULL or empty;
		2129	* - type_id - type ID describing the type of the parameter.
		2130	* Returns:
		2131	* - 0, on success;
		2132	* - <0, on error.
		2133	*/
		2134	int btf__add_func_param(struct btf btf, const char name, int type_id)
940	{	2135	{
941	const struct btf_type *container_type;	2136	struct btf_type *t;
942	const struct btf_member key, value;	2137	struct btf_param *p;
943	const size_t max_name = 256;	2138	int sz, name_off = 0;
944	char container_name[max_name];
945	__s64 key_size, value_size;
946	__s32 container_id;
947		2139
948	if (snprintf(container_name, max_name, "____btf_map_%s", map_name) ==	2140	if (validate_type_id(type_id))
949	max_name) {
950	pr_warn("map:%s length of '____btf_map_%s' is too long\n",
951	map_name, map_name);
952	return -EINVAL;	2141	return -EINVAL;
953	}
954
955	container_id = btf__find_by_name(btf, container_name);
956	if (container_id < 0) {
957	pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
958	map_name, container_name);
959	return container_id;
960	}
961		2142
962	container_type = btf__type_by_id(btf, container_id);	2143	/* last type should be BTF_KIND_FUNC_PROTO */
963	if (!container_type) {	2144	if (btf->nr_types == 0)
964	pr_warn("map:%s cannot find BTF type for container_id:%u\n",	2145	return -EINVAL;
965	map_name, container_id);	2146	t = btf_type_by_id(btf, btf->nr_types);
		2147	if (!btf_is_func_proto(t))
966	return -EINVAL;	2148	return -EINVAL;
		2149
		2150	/* decompose and invalidate raw data */
		2151	if (btf_ensure_modifiable(btf))
		2152	return -ENOMEM;
		2153
		2154	sz = sizeof(struct btf_param);
		2155	p = btf_add_type_mem(btf, sz);
		2156	if (!p)
		2157	return -ENOMEM;
		2158
		2159	if (name && name[0]) {
		2160	name_off = btf__add_str(btf, name);
		2161	if (name_off < 0)
		2162	return name_off;
967	}	2163	}
968		2164
969	if (!btf_is_struct(container_type) \|\| btf_vlen(container_type) < 2) {	2165	p->name_off = name_off;
970	pr_warn("map:%s container_name:%s is an invalid container struct\n",	2166	p->type = type_id;
971	map_name, container_name);	2167
		2168	/* update parent type's vlen */
		2169	t = btf_type_by_id(btf, btf->nr_types);
		2170	btf_type_inc_vlen(t);
		2171
		2172	btf->hdr->type_len += sz;
		2173	btf->hdr->str_off += sz;
		2174	return 0;
		2175	}
		2176
		2177	/*
		2178	* Append new BTF_KIND_VAR type with:
		2179	* - name - non-empty/non-NULL name;
		2180	* - linkage - variable linkage, one of BTF_VAR_STATIC,
		2181	* BTF_VAR_GLOBAL_ALLOCATED, or BTF_VAR_GLOBAL_EXTERN;
		2182	* - type_id - type ID of the type describing the type of the variable.
		2183	* Returns:
		2184	* - >0, type ID of newly added BTF type;
		2185	* - <0, on error.
		2186	*/
		2187	int btf__add_var(struct btf btf, const char name, int linkage, int type_id)
		2188	{
		2189	struct btf_type *t;
		2190	struct btf_var *v;
		2191	int sz, err, name_off;
		2192
		2193	/* non-empty name */
		2194	if (!name \|\| !name[0])
		2195	return -EINVAL;
		2196	if (linkage != BTF_VAR_STATIC && linkage != BTF_VAR_GLOBAL_ALLOCATED &&
		2197	linkage != BTF_VAR_GLOBAL_EXTERN)
		2198	return -EINVAL;
		2199	if (validate_type_id(type_id))
972	return -EINVAL;	2200	return -EINVAL;
973	}
974		2201
975	key = btf_members(container_type);	2202	/* deconstruct BTF, if necessary, and invalidate raw_data */
976	value = key + 1;	2203	if (btf_ensure_modifiable(btf))
		2204	return -ENOMEM;
977		2205
978	key_size = btf__resolve_size(btf, key->type);	2206	sz = sizeof(struct btf_type) + sizeof(struct btf_var);
979	if (key_size < 0) {	2207	t = btf_add_type_mem(btf, sz);
980	pr_warn("map:%s invalid BTF key_type_size\n", map_name);	2208	if (!t)
981	return key_size;	2209	return -ENOMEM;
982	}
983		2210
984	if (expected_key_size != key_size) {	2211	name_off = btf__add_str(btf, name);
985	pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",	2212	if (name_off < 0)
986	map_name, (__u32)key_size, expected_key_size);	2213	return name_off;
		2214
		2215	t->name_off = name_off;
		2216	t->info = btf_type_info(BTF_KIND_VAR, 0, 0);
		2217	t->type = type_id;
		2218
		2219	v = btf_var(t);
		2220	v->linkage = linkage;
		2221
		2222	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
		2223	if (err)
		2224	return err;
		2225
		2226	btf->hdr->type_len += sz;
		2227	btf->hdr->str_off += sz;
		2228	btf->nr_types++;
		2229	return btf->nr_types;
		2230	}
		2231
		2232	/*
		2233	* Append new BTF_KIND_DATASEC type with:
		2234	* - name - non-empty/non-NULL name;
		2235	* - byte_sz - data section size, in bytes.
		2236	*
		2237	* Data section is initially empty. Variables info can be added with
		2238	* btf__add_datasec_var_info() calls, after btf__add_datasec() succeeds.
		2239	*
		2240	* Returns:
		2241	* - >0, type ID of newly added BTF type;
		2242	* - <0, on error.
		2243	*/
		2244	int btf__add_datasec(struct btf btf, const char name, __u32 byte_sz)
		2245	{
		2246	struct btf_type *t;
		2247	int sz, err, name_off;
		2248
		2249	/* non-empty name */
		2250	if (!name \|\| !name[0])
987	return -EINVAL;	2251	return -EINVAL;
988	}
989		2252
990	value_size = btf__resolve_size(btf, value->type);	2253	if (btf_ensure_modifiable(btf))
991	if (value_size < 0) {	2254	return -ENOMEM;
992	pr_warn("map:%s invalid BTF value_type_size\n", map_name);
993	return value_size;
994	}
995		2255
996	if (expected_value_size != value_size) {	2256	sz = sizeof(struct btf_type);
997	pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",	2257	t = btf_add_type_mem(btf, sz);
998	map_name, (__u32)value_size, expected_value_size);	2258	if (!t)
		2259	return -ENOMEM;
		2260
		2261	name_off = btf__add_str(btf, name);
		2262	if (name_off < 0)
		2263	return name_off;
		2264
		2265	/* start with vlen=0, which will be update as var_secinfos are added */
		2266	t->name_off = name_off;
		2267	t->info = btf_type_info(BTF_KIND_DATASEC, 0, 0);
		2268	t->size = byte_sz;
		2269
		2270	err = btf_add_type_idx_entry(btf, btf->hdr->type_len);
		2271	if (err)
		2272	return err;
		2273
		2274	btf->hdr->type_len += sz;
		2275	btf->hdr->str_off += sz;
		2276	btf->nr_types++;
		2277	return btf->nr_types;
		2278	}
		2279
		2280	/*
		2281	* Append new data section variable information entry for current DATASEC type:
		2282	* - var_type_id - type ID, describing type of the variable;
		2283	* - offset - variable offset within data section, in bytes;
		2284	* - byte_sz - variable size, in bytes.
		2285	*
		2286	* Returns:
		2287	* - 0, on success;
		2288	* - <0, on error.
		2289	*/
		2290	int btf__add_datasec_var_info(struct btf *btf, int var_type_id, __u32 offset, __u32 byte_sz)
		2291	{
		2292	struct btf_type *t;
		2293	struct btf_var_secinfo *v;
		2294	int sz;
		2295
		2296	/* last type should be BTF_KIND_DATASEC */
		2297	if (btf->nr_types == 0)
		2298	return -EINVAL;
		2299	t = btf_type_by_id(btf, btf->nr_types);
		2300	if (!btf_is_datasec(t))
999	return -EINVAL;	2301	return -EINVAL;
1000	}
1001		2302
1002	*key_type_id = key->type;	2303	if (validate_type_id(var_type_id))
1003	*value_type_id = value->type;	2304	return -EINVAL;
		2305
		2306	/* decompose and invalidate raw data */
		2307	if (btf_ensure_modifiable(btf))
		2308	return -ENOMEM;
		2309
		2310	sz = sizeof(struct btf_var_secinfo);
		2311	v = btf_add_type_mem(btf, sz);
		2312	if (!v)
		2313	return -ENOMEM;
		2314
		2315	v->type = var_type_id;
		2316	v->offset = offset;
		2317	v->size = byte_sz;
1004		2318
		2319	/* update parent type's vlen */
		2320	t = btf_type_by_id(btf, btf->nr_types);
		2321	btf_type_inc_vlen(t);
		2322
		2323	btf->hdr->type_len += sz;
		2324	btf->hdr->str_off += sz;
1005	return 0;	2325	return 0;
1006	}	2326	}
1007		2327
@@ -1151,7 +2471,10 @@ static int btf_ext_parse_hdr(__u8 *data, __u32 data_size) · 143 unmodified lines
1151	return -EINVAL;	2471	return -EINVAL;
1152	}	2472	}
1153		2473
1154	if (hdr->magic != BTF_MAGIC) {	2474	if (hdr->magic == bswap_16(BTF_MAGIC)) {
		2475	pr_warn("BTF.ext in non-native endianness is not supported\n");
		2476	return -ENOTSUP;
		2477	} else if (hdr->magic != BTF_MAGIC) {
1155	pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);	2478	pr_debug("Invalid BTF.ext magic:%x\n", hdr->magic);
1156	return -EINVAL;	2479	return -EINVAL;
1157	}	2480	}
@@ -1471,6 +2794,9 @@ int btf__dedup(struct btf btf, struct btf_ext btf_ext, · 313 unmodified lines
1471	return -EINVAL;	2794	return -EINVAL;
1472	}	2795	}
1473		2796
		2797	if (btf_ensure_modifiable(btf))
		2798	return -ENOMEM;
		2799
1474	err = btf_dedup_strings(d);	2800	err = btf_dedup_strings(d);
1475	if (err < 0) {	2801	if (err < 0) {
1476	pr_debug("btf_dedup_strings failed:%d\n", err);	2802	pr_debug("btf_dedup_strings failed:%d\n", err);
@@ -1655,7 +2981,7 @@ static struct btf_dedup btf_dedup_new(struct btf btf, struct btf_ext *btf_ext, · 178 unmodified lines
1655	/* special BTF "void" type is made canonical immediately */	2981	/* special BTF "void" type is made canonical immediately */
1656	d->map[0] = 0;	2982	d->map[0] = 0;
1657	for (i = 1; i <= btf->nr_types; i++) {	2983	for (i = 1; i <= btf->nr_types; i++) {
1658	struct btf_type *t = d->btf->types[i];	2984	struct btf_type *t = btf_type_by_id(d->btf, i);
1659		2985
1660	/* VAR and DATASEC are never deduped and are self-canonical */	2986	/* VAR and DATASEC are never deduped and are self-canonical */
1661	if (btf_is_var(t) \|\| btf_is_datasec(t))	2987	if (btf_is_var(t) \|\| btf_is_datasec(t))
@@ -1694,7 +3020,7 @@ static int btf_for_each_str_off(struct btf_dedup d, str_off_fn_t fn, void ctx) · 32 unmodified lines
1694	struct btf_type *t;	3020	struct btf_type *t;
1695		3021
1696	for (i = 1; i <= d->btf->nr_types; i++) {	3022	for (i = 1; i <= d->btf->nr_types; i++) {
1697	t = d->btf->types[i];	3023	t = btf_type_by_id(d->btf, i);
1698	r = fn(&t->name_off, ctx);	3024	r = fn(&t->name_off, ctx);
1699	if (r)	3025	if (r)
1700	return r;	3026	return r;
@@ -1848,8 +3174,7 @@ static int btf_str_remap_offset(__u32 str_off_ptr, void ctx) · 147 unmodified lines
1848	*/	3174	*/
1849	static int btf_dedup_strings(struct btf_dedup *d)	3175	static int btf_dedup_strings(struct btf_dedup *d)
1850	{	3176	{
1851	const struct btf_header *hdr = d->btf->hdr;	3177	char *start = d->btf->strs_data;
1852	char start = (char )d->btf->nohdr_data + hdr->str_off;
1853	char *end = start + d->btf->hdr->str_len;	3178	char *end = start + d->btf->hdr->str_len;
1854	char p = start, tmp_strs = NULL;	3179	char p = start, tmp_strs = NULL;
1855	struct btf_str_ptrs strs = {	3180	struct btf_str_ptrs strs = {
@@ -1861,6 +3186,9 @@ static int btf_dedup_strings(struct btf_dedup *d) · 5 unmodified lines
1861	int i, j, err = 0, grp_idx;	3186	int i, j, err = 0, grp_idx;
1862	bool grp_used;	3187	bool grp_used;
1863		3188
		3189	if (d->btf->strs_deduped)
		3190	return 0;
		3191
1864	/* build index of all strings */	3192	/* build index of all strings */
1865	while (p < end) {	3193	while (p < end) {
1866	if (strs.cnt + 1 > strs.cap) {	3194	if (strs.cnt + 1 > strs.cap) {
@@ -1953,6 +3281,7 @@ static int btf_dedup_strings(struct btf_dedup *d) · 86 unmodified lines
1953	goto done;	3281	goto done;
1954		3282
1955	d->btf->hdr->str_len = end - start;	3283	d->btf->hdr->str_len = end - start;
		3284	d->btf->strs_deduped = true;
1956		3285
1957	done:	3286	done:
1958	free(tmp_strs);	3287	free(tmp_strs);
@@ -2229,7 +3558,7 @@ static bool btf_compat_fnproto(struct btf_type t1, struct btf_type t2) · 270 unmodified lines
2229	*/	3558	*/
2230	static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)	3559	static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id)
2231	{	3560	{
2232	struct btf_type *t = d->btf->types[type_id];	3561	struct btf_type *t = btf_type_by_id(d->btf, type_id);
2233	struct hashmap_entry *hash_entry;	3562	struct hashmap_entry *hash_entry;
2234	struct btf_type *cand;	3563	struct btf_type *cand;
2235	/* if we don't find equivalent type, then we are canonical */	3564	/* if we don't find equivalent type, then we are canonical */
@@ -2256,7 +3585,7 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) · 20 unmodified lines
2256	h = btf_hash_int(t);	3585	h = btf_hash_int(t);
2257	for_each_dedup_cand(d, hash_entry, h) {	3586	for_each_dedup_cand(d, hash_entry, h) {
2258	cand_id = (__u32)(long)hash_entry->value;	3587	cand_id = (__u32)(long)hash_entry->value;
2259	cand = d->btf->types[cand_id];	3588	cand = btf_type_by_id(d->btf, cand_id);
2260	if (btf_equal_int(t, cand)) {	3589	if (btf_equal_int(t, cand)) {
2261	new_id = cand_id;	3590	new_id = cand_id;
2262	break;	3591	break;
@@ -2268,7 +3597,7 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) · 5 unmodified lines
2268	h = btf_hash_enum(t);	3597	h = btf_hash_enum(t);
2269	for_each_dedup_cand(d, hash_entry, h) {	3598	for_each_dedup_cand(d, hash_entry, h) {
2270	cand_id = (__u32)(long)hash_entry->value;	3599	cand_id = (__u32)(long)hash_entry->value;
2271	cand = d->btf->types[cand_id];	3600	cand = btf_type_by_id(d->btf, cand_id);
2272	if (btf_equal_enum(t, cand)) {	3601	if (btf_equal_enum(t, cand)) {
2273	new_id = cand_id;	3602	new_id = cand_id;
2274	break;	3603	break;
@@ -2291,7 +3620,7 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) · 16 unmodified lines
2291	h = btf_hash_common(t);	3620	h = btf_hash_common(t);
2292	for_each_dedup_cand(d, hash_entry, h) {	3621	for_each_dedup_cand(d, hash_entry, h) {
2293	cand_id = (__u32)(long)hash_entry->value;	3622	cand_id = (__u32)(long)hash_entry->value;
2294	cand = d->btf->types[cand_id];	3623	cand = btf_type_by_id(d->btf, cand_id);
2295	if (btf_equal_common(t, cand)) {	3624	if (btf_equal_common(t, cand)) {
2296	new_id = cand_id;	3625	new_id = cand_id;
2297	break;	3626	break;
@@ -2350,13 +3679,13 @@ static uint32_t resolve_fwd_id(struct btf_dedup *d, uint32_t type_id) · 52 unmodified lines
2350	{	3679	{
2351	__u32 orig_type_id = type_id;	3680	__u32 orig_type_id = type_id;
2352		3681
2353	if (!btf_is_fwd(d->btf->types[type_id]))	3682	if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
2354	return type_id;	3683	return type_id;
2355		3684
2356	while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)	3685	while (is_type_mapped(d, type_id) && d->map[type_id] != type_id)
2357	type_id = d->map[type_id];	3686	type_id = d->map[type_id];
2358		3687
2359	if (!btf_is_fwd(d->btf->types[type_id]))	3688	if (!btf_is_fwd(btf__type_by_id(d->btf, type_id)))
2360	return type_id;	3689	return type_id;
2361		3690
2362	return orig_type_id;	3691	return orig_type_id;
@@ -2484,8 +3813,8 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, · 121 unmodified lines
2484	if (btf_dedup_hypot_map_add(d, canon_id, cand_id))	3813	if (btf_dedup_hypot_map_add(d, canon_id, cand_id))
2485	return -ENOMEM;	3814	return -ENOMEM;
2486		3815
2487	cand_type = d->btf->types[cand_id];	3816	cand_type = btf_type_by_id(d->btf, cand_id);
2488	canon_type = d->btf->types[canon_id];	3817	canon_type = btf_type_by_id(d->btf, canon_id);
2489	cand_kind = btf_kind(cand_type);	3818	cand_kind = btf_kind(cand_type);
2490	canon_kind = btf_kind(canon_type);	3819	canon_kind = btf_kind(canon_type);
2491		3820
@@ -2636,8 +3965,8 @@ static void btf_dedup_merge_hypot_map(struct btf_dedup *d) · 144 unmodified lines
2636	targ_type_id = d->hypot_map[cand_type_id];	3965	targ_type_id = d->hypot_map[cand_type_id];
2637	t_id = resolve_type_id(d, targ_type_id);	3966	t_id = resolve_type_id(d, targ_type_id);
2638	c_id = resolve_type_id(d, cand_type_id);	3967	c_id = resolve_type_id(d, cand_type_id);
2639	t_kind = btf_kind(d->btf->types[t_id]);	3968	t_kind = btf_kind(btf__type_by_id(d->btf, t_id));
2640	c_kind = btf_kind(d->btf->types[c_id]);	3969	c_kind = btf_kind(btf__type_by_id(d->btf, c_id));
2641	/*	3970	/*
2642	* Resolve FWD into STRUCT/UNION.	3971	* Resolve FWD into STRUCT/UNION.
2643	* It's ok to resolve FWD into STRUCT/UNION that's not yet	3972	* It's ok to resolve FWD into STRUCT/UNION that's not yet
@@ -2705,7 +4034,7 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) · 61 unmodified lines
2705	if (d->map[type_id] <= BTF_MAX_NR_TYPES)	4034	if (d->map[type_id] <= BTF_MAX_NR_TYPES)
2706	return 0;	4035	return 0;
2707		4036
2708	t = d->btf->types[type_id];	4037	t = btf_type_by_id(d->btf, type_id);
2709	kind = btf_kind(t);	4038	kind = btf_kind(t);
2710		4039
2711	if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)	4040	if (kind != BTF_KIND_STRUCT && kind != BTF_KIND_UNION)
@@ -2726,7 +4055,7 @@ static int btf_dedup_struct_type(struct btf_dedup *d, __u32 type_id) · 14 unmodified lines
2726	* creating a loop (FWD -> STRUCT and STRUCT -> FWD), because	4055	* creating a loop (FWD -> STRUCT and STRUCT -> FWD), because
2727	* FWD and compatible STRUCT/UNION are considered equivalent.	4056	* FWD and compatible STRUCT/UNION are considered equivalent.
2728	*/	4057	*/
2729	cand_type = d->btf->types[cand_id];	4058	cand_type = btf_type_by_id(d->btf, cand_id);
2730	if (!btf_shallow_equal_struct(t, cand_type))	4059	if (!btf_shallow_equal_struct(t, cand_type))
2731	continue;	4060	continue;
2732		4061
@@ -2798,7 +4127,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) · 65 unmodified lines
2798	if (d->map[type_id] <= BTF_MAX_NR_TYPES)	4127	if (d->map[type_id] <= BTF_MAX_NR_TYPES)
2799	return resolve_type_id(d, type_id);	4128	return resolve_type_id(d, type_id);
2800		4129
2801	t = d->btf->types[type_id];	4130	t = btf_type_by_id(d->btf, type_id);
2802	d->map[type_id] = BTF_IN_PROGRESS_ID;	4131	d->map[type_id] = BTF_IN_PROGRESS_ID;
2803		4132
2804	switch (btf_kind(t)) {	4133	switch (btf_kind(t)) {
@@ -2816,7 +4145,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) · 11 unmodified lines
2816	h = btf_hash_common(t);	4145	h = btf_hash_common(t);
2817	for_each_dedup_cand(d, hash_entry, h) {	4146	for_each_dedup_cand(d, hash_entry, h) {
2818	cand_id = (__u32)(long)hash_entry->value;	4147	cand_id = (__u32)(long)hash_entry->value;
2819	cand = d->btf->types[cand_id];	4148	cand = btf_type_by_id(d->btf, cand_id);
2820	if (btf_equal_common(t, cand)) {	4149	if (btf_equal_common(t, cand)) {
2821	new_id = cand_id;	4150	new_id = cand_id;
2822	break;	4151	break;
@@ -2840,7 +4169,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) · 17 unmodified lines
2840	h = btf_hash_array(t);	4169	h = btf_hash_array(t);
2841	for_each_dedup_cand(d, hash_entry, h) {	4170	for_each_dedup_cand(d, hash_entry, h) {
2842	cand_id = (__u32)(long)hash_entry->value;	4171	cand_id = (__u32)(long)hash_entry->value;
2843	cand = d->btf->types[cand_id];	4172	cand = btf_type_by_id(d->btf, cand_id);
2844	if (btf_equal_array(t, cand)) {	4173	if (btf_equal_array(t, cand)) {
2845	new_id = cand_id;	4174	new_id = cand_id;
2846	break;	4175	break;
@@ -2872,7 +4201,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) · 25 unmodified lines
2872	h = btf_hash_fnproto(t);	4201	h = btf_hash_fnproto(t);
2873	for_each_dedup_cand(d, hash_entry, h) {	4202	for_each_dedup_cand(d, hash_entry, h) {
2874	cand_id = (__u32)(long)hash_entry->value;	4203	cand_id = (__u32)(long)hash_entry->value;
2875	cand = d->btf->types[cand_id];	4204	cand = btf_type_by_id(d->btf, cand_id);
2876	if (btf_equal_fnproto(t, cand)) {	4205	if (btf_equal_fnproto(t, cand)) {
2877	new_id = cand_id;	4206	new_id = cand_id;
2878	break;	4207	break;
@@ -2920,9 +4249,9 @@ static int btf_dedup_ref_types(struct btf_dedup *d) · 41 unmodified lines
2920	*/	4249	*/
2921	static int btf_dedup_compact_types(struct btf_dedup *d)	4250	static int btf_dedup_compact_types(struct btf_dedup *d)
2922	{	4251	{
2923	struct btf_type **new_types;	4252	__u32 *new_offs;
2924	__u32 next_type_id = 1;	4253	__u32 next_type_id = 1;
2925	char types_start, p;	4254	void *p;
2926	int i, len;	4255	int i, len;
2927		4256
2928	/* we are going to reuse hypot_map to store compaction remapping */	4257	/* we are going to reuse hypot_map to store compaction remapping */
@@ -2930,41 +4259,34 @@ static int btf_dedup_compact_types(struct btf_dedup *d) · 1 unmodified line
2930	for (i = 1; i <= d->btf->nr_types; i++)	4259	for (i = 1; i <= d->btf->nr_types; i++)
2931	d->hypot_map[i] = BTF_UNPROCESSED_ID;	4260	d->hypot_map[i] = BTF_UNPROCESSED_ID;
2932		4261
2933	types_start = d->btf->nohdr_data + d->btf->hdr->type_off;	4262	p = d->btf->types_data;
2934	p = types_start;
2935		4263
2936	for (i = 1; i <= d->btf->nr_types; i++) {	4264	for (i = 1; i <= d->btf->nr_types; i++) {
2937	if (d->map[i] != i)	4265	if (d->map[i] != i)
2938	continue;	4266	continue;
2939		4267
2940	len = btf_type_size(d->btf->types[i]);	4268	len = btf_type_size(btf__type_by_id(d->btf, i));
2941	if (len < 0)	4269	if (len < 0)
2942	return len;	4270	return len;
2943		4271
2944	memmove(p, d->btf->types[i], len);	4272	memmove(p, btf__type_by_id(d->btf, i), len);
2945	d->hypot_map[i] = next_type_id;	4273	d->hypot_map[i] = next_type_id;
2946	d->btf->types[next_type_id] = (struct btf_type *)p;	4274	d->btf->type_offs[next_type_id] = p - d->btf->types_data;
2947	p += len;	4275	p += len;
2948	next_type_id++;	4276	next_type_id++;
2949	}	4277	}
2950		4278
2951	/* shrink struct btf's internal types index and update btf_header */	4279	/* shrink struct btf's internal types index and update btf_header */
2952	d->btf->nr_types = next_type_id - 1;	4280	d->btf->nr_types = next_type_id - 1;
2953	d->btf->types_size = d->btf->nr_types;	4281	d->btf->type_offs_cap = d->btf->nr_types + 1;
2954	d->btf->hdr->type_len = p - types_start;	4282	d->btf->hdr->type_len = p - d->btf->types_data;
2955	new_types = libbpf_reallocarray(d->btf->types, (1 + d->btf->nr_types),	4283	new_offs = libbpf_reallocarray(d->btf->type_offs, d->btf->type_offs_cap,
2956	sizeof(struct btf_type *));	4284	sizeof(*new_offs));
2957	if (!new_types)	4285	if (!new_offs)
2958	return -ENOMEM;	4286	return -ENOMEM;
2959	d->btf->types = new_types;	4287	d->btf->type_offs = new_offs;
2960		4288	d->btf->hdr->str_off = d->btf->hdr->type_len;
2961	/* make sure string section follows type information without gaps */	4289	d->btf->raw_size = d->btf->hdr->hdr_len + d->btf->hdr->type_len + d->btf->hdr->str_len;
2962	d->btf->hdr->str_off = p - (char *)d->btf->nohdr_data;
2963	memmove(p, d->btf->strings, d->btf->hdr->str_len);
2964	d->btf->strings = p;
2965	p += d->btf->hdr->str_len;
2966
2967	d->btf->data_size = p - (char *)d->btf->data;
2968	return 0;	4290	return 0;
2969	}	4291	}
2970		4292
@@ -2997,7 +4319,7 @@ static int btf_dedup_remap_type_id(struct btf_dedup *d, __u32 type_id) · 26 unmodified lines
2997	*/	4319	*/
2998	static int btf_dedup_remap_type(struct btf_dedup *d, __u32 type_id)	4320	static int btf_dedup_remap_type(struct btf_dedup *d, __u32 type_id)
2999	{	4321	{
3000	struct btf_type *t = d->btf->types[type_id];	4322	struct btf_type *t = btf_type_by_id(d->btf, type_id);
3001	int i, r;	4323	int i, r;
3002		4324
3003	switch (btf_kind(t)) {	4325	switch (btf_kind(t)) {

tools/lib/bpf/btf.h+51−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	#define __LIBBPF_BTF_H	5	#define __LIBBPF_BTF_H
6		6
7	#include <stdarg.h>	7	#include <stdarg.h>
		8	#include <stdbool.h>
8	#include <linux/btf.h>	9	#include <linux/btf.h>
9	#include <linux/types.h>	10	#include <linux/types.h>
10		11
@@ -24,8 +25,14 @@ struct btf_type; · 13 unmodified lines
24		25
25	struct bpf_object;	26	struct bpf_object;
26		27
		28	enum btf_endianness {
		29	BTF_LITTLE_ENDIAN = 0,
		30	BTF_BIG_ENDIAN = 1,
		31	};
		32
27	LIBBPF_API void btf__free(struct btf *btf);	33	LIBBPF_API void btf__free(struct btf *btf);
28	LIBBPF_API struct btf btf__new(const void data, __u32 size);	34	LIBBPF_API struct btf btf__new(const void data, __u32 size);
		35	LIBBPF_API struct btf *btf__new_empty(void);
29	LIBBPF_API struct btf btf__parse(const char path, struct btf_ext **btf_ext);	36	LIBBPF_API struct btf btf__parse(const char path, struct btf_ext **btf_ext);
30	LIBBPF_API struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext);	37	LIBBPF_API struct btf btf__parse_elf(const char path, struct btf_ext **btf_ext);
31	LIBBPF_API struct btf btf__parse_raw(const char path);	38	LIBBPF_API struct btf btf__parse_raw(const char path);
@@ -40,6 +47,8 @@ LIBBPF_API const struct btf_type btf__type_by_id(const struct btf btf, · 8 unmodified lines
40	__u32 id);	47	__u32 id);
41	LIBBPF_API size_t btf__pointer_size(const struct btf *btf);	48	LIBBPF_API size_t btf__pointer_size(const struct btf *btf);
42	LIBBPF_API int btf__set_pointer_size(struct btf *btf, size_t ptr_sz);	49	LIBBPF_API int btf__set_pointer_size(struct btf *btf, size_t ptr_sz);
		50	LIBBPF_API enum btf_endianness btf__endianness(const struct btf *btf);
		51	LIBBPF_API int btf__set_endianness(struct btf *btf, enum btf_endianness endian);
43	LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id);	52	LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id);
44	LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id);	53	LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id);
45	LIBBPF_API int btf__align_of(const struct btf *btf, __u32 id);	54	LIBBPF_API int btf__align_of(const struct btf *btf, __u32 id);
@@ -47,6 +56,7 @@ LIBBPF_API int btf__fd(const struct btf *btf); · 1 unmodified line
47	LIBBPF_API void btf__set_fd(struct btf *btf, int fd);	56	LIBBPF_API void btf__set_fd(struct btf *btf, int fd);
48	LIBBPF_API const void btf__get_raw_data(const struct btf btf, __u32 *size);	57	LIBBPF_API const void btf__get_raw_data(const struct btf btf, __u32 *size);
49	LIBBPF_API const char btf__name_by_offset(const struct btf btf, __u32 offset);	58	LIBBPF_API const char btf__name_by_offset(const struct btf btf, __u32 offset);
		59	LIBBPF_API const char btf__str_by_offset(const struct btf btf, __u32 offset);
50	LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);	60	LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf);
51	LIBBPF_API int btf__get_map_kv_tids(const struct btf btf, const char map_name,	61	LIBBPF_API int btf__get_map_kv_tids(const struct btf btf, const char map_name,
52	__u32 expected_key_size,	62	__u32 expected_key_size,
@@ -72,6 +82,47 @@ LIBBPF_API __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); · 19 unmodified lines
72		82
73	LIBBPF_API struct btf *libbpf_find_kernel_btf(void);	83	LIBBPF_API struct btf *libbpf_find_kernel_btf(void);
74		84
		85	LIBBPF_API int btf__find_str(struct btf btf, const char s);
		86	LIBBPF_API int btf__add_str(struct btf btf, const char s);
		87
		88	LIBBPF_API int btf__add_int(struct btf btf, const char name, size_t byte_sz, int encoding);
		89	LIBBPF_API int btf__add_ptr(struct btf *btf, int ref_type_id);
		90	LIBBPF_API int btf__add_array(struct btf *btf,
		91	int index_type_id, int elem_type_id, __u32 nr_elems);
		92	/* struct/union construction APIs */
		93	LIBBPF_API int btf__add_struct(struct btf btf, const char name, __u32 sz);
		94	LIBBPF_API int btf__add_union(struct btf btf, const char name, __u32 sz);
		95	LIBBPF_API int btf__add_field(struct btf btf, const char name, int field_type_id,
		96	__u32 bit_offset, __u32 bit_size);
		97
		98	/* enum construction APIs */
		99	LIBBPF_API int btf__add_enum(struct btf btf, const char name, __u32 bytes_sz);
		100	LIBBPF_API int btf__add_enum_value(struct btf btf, const char name, __s64 value);
		101
		102	enum btf_fwd_kind {
		103	BTF_FWD_STRUCT = 0,
		104	BTF_FWD_UNION = 1,
		105	BTF_FWD_ENUM = 2,
		106	};
		107
		108	LIBBPF_API int btf__add_fwd(struct btf btf, const char name, enum btf_fwd_kind fwd_kind);
		109	LIBBPF_API int btf__add_typedef(struct btf btf, const char name, int ref_type_id);
		110	LIBBPF_API int btf__add_volatile(struct btf *btf, int ref_type_id);
		111	LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id);
		112	LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id);
		113
		114	/* func and func_proto construction APIs */
		115	LIBBPF_API int btf__add_func(struct btf btf, const char name,
		116	enum btf_func_linkage linkage, int proto_type_id);
		117	LIBBPF_API int btf__add_func_proto(struct btf *btf, int ret_type_id);
		118	LIBBPF_API int btf__add_func_param(struct btf btf, const char name, int type_id);
		119
		120	/* var & datasec construction APIs */
		121	LIBBPF_API int btf__add_var(struct btf btf, const char name, int linkage, int type_id);
		122	LIBBPF_API int btf__add_datasec(struct btf btf, const char name, __u32 byte_sz);
		123	LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id,
		124	__u32 offset, __u32 byte_sz);
		125
75	struct btf_dedup_opts {	126	struct btf_dedup_opts {
76	unsigned int dedup_table_size;	127	unsigned int dedup_table_size;
77	bool dont_resolve_fwds;	128	bool dont_resolve_fwds;

tools/lib/bpf/btf_dump.c+48−30

 	struct btf_dump_opts opts;
 	int ptr_sz;
 	bool strip_mods;
+	int last_id;
 	/* per-type auxiliary state */
 	struct btf_dump_type_aux_state *type_states;
+	size_t type_states_cap;
 	/* per-type optional cached unique name, must be freed, if present */
 	const char **cached_names;
+	size_t cached_names_cap;
 	/* topo-sorted list of dependent type definitions */
 	__u32 *emit_queue;
 static size_t str_hash_fn(const void *key, void *ctx)
 {
-	const char *s = key;
+	return str_hash(key);
-	size_t h = 0;
-	while (*s) {
-		h = h * 31 + *s;
-		s++;
-	}
-	return h;
 }
 static bool str_equal_fn(const void *a, const void *b, void *ctx)
 }
 static int btf_dump_mark_referenced(struct btf_dump *d);
+static int btf_dump_resize(struct btf_dump *d);
 struct btf_dump *btf_dump__new(const struct btf *btf,
 			       const struct btf_ext *btf_ext,
 		d->ident_names = NULL;
 		goto err;
 	}
-	d->type_states = calloc(1 + btf__get_nr_types(d->btf),
-				sizeof(d->type_states[0]));
-	if (!d->type_states) {
-		err = -ENOMEM;
-		goto err;
-	}
-	d->cached_names = calloc(1 + btf__get_nr_types(d->btf),
-				 sizeof(d->cached_names[0]));
-	if (!d->cached_names) {
-		err = -ENOMEM;
-		goto err;
-	}
-	/* VOID is special */
+	err = btf_dump_resize(d);
-	d->type_states[0].order_state = ORDERED;
-	d->type_states[0].emit_state = EMITTED;
-	/* eagerly determine referenced types for anon enums */
-	err = btf_dump_mark_referenced(d);
 	if (err)
 		goto err;
 	return ERR_PTR(err);
 }
+static int btf_dump_resize(struct btf_dump *d)
+{
+	int err, last_id = btf__get_nr_types(d->btf);
+	if (last_id <= d->last_id)
+		return 0;
+	if (btf_ensure_mem((void **)&d->type_states, &d->type_states_cap,
+			   sizeof(*d->type_states), last_id + 1))
+		return -ENOMEM;
+	if (btf_ensure_mem((void **)&d->cached_names, &d->cached_names_cap,
+			   sizeof(*d->cached_names), last_id + 1))
+		return -ENOMEM;
+	if (d->last_id == 0) {
+		/* VOID is special */
+		d->type_states[0].order_state = ORDERED;
+		d->type_states[0].emit_state = EMITTED;
+	}
+	/* eagerly determine referenced types for anon enums */
+	err = btf_dump_mark_referenced(d);
+	if (err)
+		return err;
+	d->last_id = last_id;
+	return 0;
+}
 void btf_dump__free(struct btf_dump *d)
 {
-	int i, cnt;
+	int i;
 	if (IS_ERR_OR_NULL(d))
 		return;
 	free(d->type_states);
 	if (d->cached_names) {
 		/* any set cached name is owned by us and should be freed */
-		for (i = 0, cnt = btf__get_nr_types(d->btf); i <= cnt; i++) {
+		for (i = 0; i <= d->last_id; i++) {
 			if (d->cached_names[i])
 				free((void *)d->cached_names[i]);
 		}
 	if (id > btf__get_nr_types(d->btf))
 		return -EINVAL;
+	err = btf_dump_resize(d);
+	if (err)
+		return err;
 	d->emit_queue_cnt = 0;
 	err = btf_dump_order_type(d, id, false);
 	if (err < 0)
 	const struct btf_type *t;
 	__u16 vlen;
-	for (i = 1; i <= n; i++) {
+	for (i = d->last_id + 1; i <= n; i++) {
 		t = btf__type_by_id(d->btf, i);
 		vlen = btf_vlen(t);
 	}
 	return 0;
 }
 static int btf_dump_add_emit_queue_id(struct btf_dump *d, __u32 id)
 {
 	__u32 *new_queue;
 			     const struct btf_dump_emit_type_decl_opts *opts)
 {
 	const char *fname;
-	int lvl;
+	int lvl, err;
 	if (!OPTS_VALID(opts, btf_dump_emit_type_decl_opts))
 		return -EINVAL;
+	err = btf_dump_resize(d);
+	if (err)
+		return -EINVAL;
 	fname = OPTS_GET(opts, field_name, "");
 	lvl = OPTS_GET(opts, indent_level, 0);
 	d->strip_mods = OPTS_GET(opts, strip_mods, false);

tools/lib/bpf/hashmap.h+12−0

@@ -25,6 +25,18 @@ static inline size_t hash_bits(size_t h, int bits) · 24 unmodified lines
25	#endif	25	#endif
26	}	26	}
27		27
		28	/* generic C-string hashing function */
		29	static inline size_t str_hash(const char *s)
		30	{
		31	size_t h = 0;
		32
		33	while (*s) {
		34	h = h * 31 + *s;
		35	s++;
		36	}
		37	return h;
		38	}
		39
28	typedef size_t (hashmap_hash_fn)(const void key, void *ctx);	40	typedef size_t (hashmap_hash_fn)(const void key, void *ctx);
29	typedef bool (hashmap_equal_fn)(const void key1, const void key2, void ctx);	41	typedef bool (hashmap_equal_fn)(const void key1, const void key2, void ctx);
30		42

tools/lib/bpf/libbpf.c+39−5

 }
 static struct bpf_link *
-bpf_program__attach_fd(struct bpf_program *prog, int target_fd,
+bpf_program__attach_fd(struct bpf_program *prog, int target_fd, int btf_id,
 		       const char *target_name)
 {
+	DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts,
+			    .target_btf_id = btf_id);
 	enum bpf_attach_type attach_type;
 	char errmsg[STRERR_BUFSIZE];
 	struct bpf_link *link;
 	link->detach = &bpf_link__detach_fd;
 	attach_type = bpf_program__get_expected_attach_type(prog);
-	link_fd = bpf_link_create(prog_fd, target_fd, attach_type, NULL);
+	link_fd = bpf_link_create(prog_fd, target_fd, attach_type, &opts);
 	if (link_fd < 0) {
 		link_fd = -errno;
 		free(link);
 struct bpf_link *
 bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd)
 {
-	return bpf_program__attach_fd(prog, cgroup_fd, "cgroup");
+	return bpf_program__attach_fd(prog, cgroup_fd, 0, "cgroup");
 }
 struct bpf_link *
 bpf_program__attach_netns(struct bpf_program *prog, int netns_fd)
 {
-	return bpf_program__attach_fd(prog, netns_fd, "netns");
+	return bpf_program__attach_fd(prog, netns_fd, 0, "netns");
 }
 struct bpf_link *bpf_program__attach_xdp(struct bpf_program *prog, int ifindex)
 {
 	/* target_fd/target_ifindex use the same field in LINK_CREATE */
-	return bpf_program__attach_fd(prog, ifindex, "xdp");
+	return bpf_program__attach_fd(prog, ifindex, 0, "xdp");
+}
+struct bpf_link *bpf_program__attach_freplace(struct bpf_program *prog,
+					      int target_fd,
+					      const char *attach_func_name)
+{
+	int btf_id;
+	if (!!target_fd != !!attach_func_name) {
+		pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
+			prog->name);
+		return ERR_PTR(-EINVAL);
+	}
+	if (prog->type != BPF_PROG_TYPE_EXT) {
+		pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
+			prog->name);
+		return ERR_PTR(-EINVAL);
+	}
+	if (target_fd) {
+		btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd);
+		if (btf_id < 0)
+			return ERR_PTR(btf_id);
+		return bpf_program__attach_fd(prog, target_fd, btf_id, "freplace");
+	} else {
+		/* no target, so use raw_tracepoint_open for compatibility
+		 * with old kernels
+		 */
+		return bpf_program__attach_trace(prog);
+	}
 }
 struct bpf_link *

tools/lib/bpf/libbpf.h+3−0

@@ -261,6 +261,9 @@ LIBBPF_API struct bpf_link * · 260 unmodified lines
261	bpf_program__attach_netns(struct bpf_program *prog, int netns_fd);	261	bpf_program__attach_netns(struct bpf_program *prog, int netns_fd);
262	LIBBPF_API struct bpf_link *	262	LIBBPF_API struct bpf_link *
263	bpf_program__attach_xdp(struct bpf_program *prog, int ifindex);	263	bpf_program__attach_xdp(struct bpf_program *prog, int ifindex);
		264	LIBBPF_API struct bpf_link *
		265	bpf_program__attach_freplace(struct bpf_program *prog,
		266	int target_fd, const char *attach_func_name);
264		267
265	struct bpf_map;	268	struct bpf_map;
266		269

tools/lib/bpf/libbpf.map+27−0

@@ -303,7 +303,34 @@ LIBBPF_0.1.0 { · 302 unmodified lines
303	LIBBPF_0.2.0 {	303	LIBBPF_0.2.0 {
304	global:	304	global:
305	bpf_prog_bind_map;	305	bpf_prog_bind_map;
		306	bpf_prog_test_run_opts;
		307	bpf_program__attach_freplace;
306	bpf_program__section_name;	308	bpf_program__section_name;
		309	btf__add_array;
		310	btf__add_const;
		311	btf__add_enum;
		312	btf__add_enum_value;
		313	btf__add_datasec;
		314	btf__add_datasec_var_info;
		315	btf__add_field;
		316	btf__add_func;
		317	btf__add_func_param;
		318	btf__add_func_proto;
		319	btf__add_fwd;
		320	btf__add_int;
		321	btf__add_ptr;
		322	btf__add_restrict;
		323	btf__add_str;
		324	btf__add_struct;
		325	btf__add_typedef;
		326	btf__add_union;
		327	btf__add_var;
		328	btf__add_volatile;
		329	btf__endianness;
		330	btf__find_str;
		331	btf__new_empty;
		332	btf__set_endianness;
		333	btf__str_by_offset;
307	perf_buffer__buffer_cnt;	334	perf_buffer__buffer_cnt;
308	perf_buffer__buffer_fd;	335	perf_buffer__buffer_fd;
309	perf_buffer__epoll_fd;	336	perf_buffer__epoll_fd;

tools/lib/bpf/libbpf_internal.h+9−0

 	return realloc(ptr, total);
 }
+void *btf_add_mem(void **data, size_t *cap_cnt, size_t elem_sz,
+		  size_t cur_cnt, size_t max_cnt, size_t add_cnt);
+int btf_ensure_mem(void **data, size_t *cap_cnt, size_t elem_sz, size_t need_cnt);
 static inline bool libbpf_validate_opts(const char *opts,
 					size_t opts_sz, size_t user_sz,
 					const char *type_name)
 	((opts) && opts->sz >= offsetofend(typeof(*(opts)), field))
 #define OPTS_GET(opts, field, fallback_value) \
 	(OPTS_HAS(opts, field) ? (opts)->field : fallback_value)
+#define OPTS_SET(opts, field, value)		\
+	do {					\
+		if (OPTS_HAS(opts, field))	\
+			(opts)->field = value;	\
+	} while (0)
 int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz);
 int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz);

tools/testing/selftests/bpf/.gitignore+0−1

@@ -13,7 +13,6 @@ test_verifier_log · 12 unmodified lines
13	feature	13	feature
14	test_sock	14	test_sock
15	test_sock_addr	15	test_sock_addr
16	test_sock_fields
17	urandom_read	16	urandom_read
18	test_sockmap	17	test_sockmap
19	test_lirc_mode2_user	18	test_lirc_mode2_user

tools/testing/selftests/bpf/Makefile+1−1

@@ -35,7 +35,7 @@ TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test · 34 unmodified lines
35	test_verifier_log test_dev_cgroup test_tcpbpf_user \	35	test_verifier_log test_dev_cgroup test_tcpbpf_user \
36	test_sock test_sockmap get_cgroup_id_user test_socket_cookie \	36	test_sock test_sockmap get_cgroup_id_user test_socket_cookie \
37	test_cgroup_storage \	37	test_cgroup_storage \
38	test_netcnt test_tcpnotify_user test_sock_fields test_sysctl \	38	test_netcnt test_tcpnotify_user test_sysctl \
39	test_progs-no_alu32 \	39	test_progs-no_alu32 \
40	test_current_pid_tgid_new_ns	40	test_current_pid_tgid_new_ns
41		41

tools/testing/selftests/bpf/bench.c+0−3

 extern const struct bench bench_rename_rawtp;
 extern const struct bench bench_rename_fentry;
 extern const struct bench bench_rename_fexit;
-extern const struct bench bench_rename_fmodret;
 extern const struct bench bench_trig_base;
 extern const struct bench bench_trig_tp;
 extern const struct bench bench_trig_rawtp;
 	&bench_rename_rawtp,
 	&bench_rename_fentry,
 	&bench_rename_fexit,
-	&bench_rename_fmodret,
 	&bench_trig_base,
 	&bench_trig_tp,
 	&bench_trig_rawtp,
 	return 0;
 }

tools/testing/selftests/bpf/benchs/bench_rename.c+0−17

@@ -106,12 +106,6 @@ static void setup_fexit() · 105 unmodified lines
106	attach_bpf(ctx.skel->progs.prog5);	106	attach_bpf(ctx.skel->progs.prog5);
107	}	107	}
108		108
109	static void setup_fmodret()
110	{
111	setup_ctx();
112	attach_bpf(ctx.skel->progs.prog6);
113	}
114
115	static void consumer(void input)	109	static void consumer(void input)
116	{	110	{
117	return NULL;	111	return NULL;
@@ -182,14 +176,3 @@ const struct bench bench_rename_fexit = { · 64 unmodified lines
182	.report_progress = hits_drops_report_progress,	176	.report_progress = hits_drops_report_progress,
183	.report_final = hits_drops_report_final,	177	.report_final = hits_drops_report_final,
184	};	178	};
185
186	const struct bench bench_rename_fmodret = {
187	.name = "rename-fmodret",
188	.validate = validate,
189	.setup = setup_fmodret,
190	.producer_thread = producer,
191	.consumer_thread = consumer,
192	.measure = measure,
193	.report_progress = hits_drops_report_progress,
194	.report_final = hits_drops_report_final,
195	};

tools/testing/selftests/bpf/bpf_tcp_helpers.h+5−8

 struct sock_common {
 	unsigned char	skc_state;
+	__u16		skc_num;
 } __attribute__((preserve_access_index));
 enum sk_pacing {
 	__u64			  icsk_ca_priv[104 / sizeof(__u64)];
 } __attribute__((preserve_access_index));
+struct request_sock {
+	struct sock_common		__req_common;
+} __attribute__((preserve_access_index));
 struct tcp_sock {
 	struct inet_connection_sock	inet_conn;
 	CA_EVENT_ECN_IS_CE = 5,
 };
-enum tcp_ca_state {
-	TCP_CA_Open = 0,
-	TCP_CA_Disorder = 1,
-	TCP_CA_CWR = 2,
-	TCP_CA_Recovery = 3,
-	TCP_CA_Loss = 4
-};
 struct ack_sample {
 	__u32 pkts_acked;
 	__s32 rtt_us;

tools/testing/selftests/bpf/prog_tests/bpf_iter.c+87−7

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	#include "bpf_iter_task.skel.h"	7	#include "bpf_iter_task.skel.h"
8	#include "bpf_iter_task_stack.skel.h"	8	#include "bpf_iter_task_stack.skel.h"
9	#include "bpf_iter_task_file.skel.h"	9	#include "bpf_iter_task_file.skel.h"
		10	#include "bpf_iter_task_btf.skel.h"
10	#include "bpf_iter_tcp4.skel.h"	11	#include "bpf_iter_tcp4.skel.h"
11	#include "bpf_iter_tcp6.skel.h"	12	#include "bpf_iter_tcp6.skel.h"
12	#include "bpf_iter_udp4.skel.h"	13	#include "bpf_iter_udp4.skel.h"
@@ -167,6 +168,83 @@ done: · 154 unmodified lines
167	bpf_iter_task_file__destroy(skel);	168	bpf_iter_task_file__destroy(skel);
168	}	169	}
169		170
		171	#define TASKBUFSZ 32768
		172
		173	static char taskbuf[TASKBUFSZ];
		174
		175	static int do_btf_read(struct bpf_iter_task_btf *skel)
		176	{
		177	struct bpf_program *prog = skel->progs.dump_task_struct;
		178	struct bpf_iter_task_btf__bss *bss = skel->bss;
		179	int iter_fd = -1, len = 0, bufleft = TASKBUFSZ;
		180	struct bpf_link *link;
		181	char *buf = taskbuf;
		182	int ret = 0;
		183
		184	link = bpf_program__attach_iter(prog, NULL);
		185	if (CHECK(IS_ERR(link), "attach_iter", "attach_iter failed\n"))
		186	return ret;
		187
		188	iter_fd = bpf_iter_create(bpf_link__fd(link));
		189	if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n"))
		190	goto free_link;
		191
		192	do {
		193	len = read(iter_fd, buf, bufleft);
		194	if (len > 0) {
		195	buf += len;
		196	bufleft -= len;
		197	}
		198	} while (len > 0);
		199
		200	if (bss->skip) {
		201	printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
		202	ret = 1;
		203	test__skip();
		204	goto free_link;
		205	}
		206
		207	if (CHECK(len < 0, "read", "read failed: %s\n", strerror(errno)))
		208	goto free_link;
		209
		210	CHECK(strstr(taskbuf, "(struct task_struct)") == NULL,
		211	"check for btf representation of task_struct in iter data",
		212	"struct task_struct not found");
		213	free_link:
		214	if (iter_fd > 0)
		215	close(iter_fd);
		216	bpf_link__destroy(link);
		217	return ret;
		218	}
		219
		220	static void test_task_btf(void)
		221	{
		222	struct bpf_iter_task_btf__bss *bss;
		223	struct bpf_iter_task_btf *skel;
		224	int ret;
		225
		226	skel = bpf_iter_task_btf__open_and_load();
		227	if (CHECK(!skel, "bpf_iter_task_btf__open_and_load",
		228	"skeleton open_and_load failed\n"))
		229	return;
		230
		231	bss = skel->bss;
		232
		233	ret = do_btf_read(skel);
		234	if (ret)
		235	goto cleanup;
		236
		237	if (CHECK(bss->tasks == 0, "check if iterated over tasks",
		238	"no task iteration, did BPF program run?\n"))
		239	goto cleanup;
		240
		241	CHECK(bss->seq_err != 0, "check for unexpected err",
		242	"bpf_seq_printf_btf returned %ld", bss->seq_err);
		243
		244	cleanup:
		245	bpf_iter_task_btf__destroy(skel);
		246	}
		247
170	static void test_tcp4(void)	248	static void test_tcp4(void)
171	{	249	{
172	struct bpf_iter_tcp4 *skel;	250	struct bpf_iter_tcp4 *skel;
@@ -352,7 +430,7 @@ static void test_overflow(bool test_e2big_overflow, bool ret1) · 179 unmodified lines
352	struct bpf_map_info map_info = {};	430	struct bpf_map_info map_info = {};
353	struct bpf_iter_test_kern4 *skel;	431	struct bpf_iter_test_kern4 *skel;
354	struct bpf_link *link;	432	struct bpf_link *link;
355	__u32 page_size;	433	__u32 iter_size;
356	char *buf;	434	char *buf;
357		435
358	skel = bpf_iter_test_kern4__open();	436	skel = bpf_iter_test_kern4__open();
@@ -374,19 +452,19 @@ static void test_overflow(bool test_e2big_overflow, bool ret1) · 15 unmodified lines
374	"map_creation failed: %s\n", strerror(errno)))	452	"map_creation failed: %s\n", strerror(errno)))
375	goto free_map1;	453	goto free_map1;
376		454
377	/* bpf_seq_printf kernel buffer is one page, so one map	455	/* bpf_seq_printf kernel buffer is 8 pages, so one map
378	* bpf_seq_write will mostly fill it, and the other map	456	* bpf_seq_write will mostly fill it, and the other map
379	* will partially fill and then trigger overflow and need	457	* will partially fill and then trigger overflow and need
380	* bpf_seq_read restart.	458	* bpf_seq_read restart.
381	*/	459	*/
382	page_size = sysconf(_SC_PAGE_SIZE);	460	iter_size = sysconf(_SC_PAGE_SIZE) << 3;
383		461
384	if (test_e2big_overflow) {	462	if (test_e2big_overflow) {
385	skel->rodata->print_len = (page_size + 8) / 8;	463	skel->rodata->print_len = (iter_size + 8) / 8;
386	expected_read_len = 2 * (page_size + 8);	464	expected_read_len = 2 * (iter_size + 8);
387	} else if (!ret1) {	465	} else if (!ret1) {
388	skel->rodata->print_len = (page_size - 8) / 8;	466	skel->rodata->print_len = (iter_size - 8) / 8;
389	expected_read_len = 2 * (page_size - 8);	467	expected_read_len = 2 * (iter_size - 8);
390	} else {	468	} else {
391	skel->rodata->print_len = 1;	469	skel->rodata->print_len = 1;
392	expected_read_len = 2 * 8;	470	expected_read_len = 2 * 8;
@@ -957,6 +1035,8 @@ void test_bpf_iter(void) · 564 unmodified lines
957	test_task_stack();	1035	test_task_stack();
958	if (test__start_subtest("task_file"))	1036	if (test__start_subtest("task_file"))
959	test_task_file();	1037	test_task_file();
		1038	if (test__start_subtest("task_btf"))
		1039	test_task_btf();
960	if (test__start_subtest("tcp4"))	1040	if (test__start_subtest("tcp4"))
961	test_tcp4();	1041	test_tcp4();
962	if (test__start_subtest("tcp6"))	1042	if (test__start_subtest("tcp6"))

tools/testing/selftests/bpf/prog_tests/btf_dump.c+105−0

@@ -129,6 +129,109 @@ done: · 128 unmodified lines
129	return err;	129	return err;
130	}	130	}
131		131
		132	static char *dump_buf;
		133	static size_t dump_buf_sz;
		134	static FILE *dump_buf_file;
		135
		136	void test_btf_dump_incremental(void)
		137	{
		138	struct btf *btf = NULL;
		139	struct btf_dump *d = NULL;
		140	struct btf_dump_opts opts;
		141	int id, err, i;
		142
		143	dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz);
		144	if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream"))
		145	return;
		146	btf = btf__new_empty();
		147	if (!ASSERT_OK_PTR(btf, "new_empty"))
		148	goto err_out;
		149	opts.ctx = dump_buf_file;
		150	d = btf_dump__new(btf, NULL, &opts, btf_dump_printf);
		151	if (!ASSERT_OK(libbpf_get_error(d), "btf_dump__new"))
		152	goto err_out;
		153
		154	/* First, generate BTF corresponding to the following C code:
		155	*
		156	* enum { VAL = 1 };
		157	*
		158	* struct s { int x; };
		159	*
		160	*/
		161	id = btf__add_enum(btf, NULL, 4);
		162	ASSERT_EQ(id, 1, "enum_id");
		163	err = btf__add_enum_value(btf, "VAL", 1);
		164	ASSERT_OK(err, "enum_val_ok");
		165
		166	id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED);
		167	ASSERT_EQ(id, 2, "int_id");
		168
		169	id = btf__add_struct(btf, "s", 4);
		170	ASSERT_EQ(id, 3, "struct_id");
		171	err = btf__add_field(btf, "x", 2, 0, 0);
		172	ASSERT_OK(err, "field_ok");
		173
		174	for (i = 1; i <= btf__get_nr_types(btf); i++) {
		175	err = btf_dump__dump_type(d, i);
		176	ASSERT_OK(err, "dump_type_ok");
		177	}
		178
		179	fflush(dump_buf_file);
		180	dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
		181	ASSERT_STREQ(dump_buf,
		182	"enum {\n"
		183	" VAL = 1,\n"
		184	"};\n"
		185	"\n"
		186	"struct s {\n"
		187	" int x;\n"
		188	"};\n\n", "c_dump1");
		189
		190	/* Now, after dumping original BTF, append another struct that embeds
		191	* anonymous enum. It also has a name conflict with the first struct:
		192	*
		193	* struct s___2 {
		194	* enum { VAL___2 = 1 } x;
		195	* struct s s;
		196	* };
		197	*
		198	* This will test that btf_dump'er maintains internal state properly.
		199	* Note that VAL___2 enum value. It's because we've already emitted
		200	* that enum as a global anonymous enum, so btf_dump will ensure that
		201	* enum values don't conflict;
		202	*
		203	*/
		204	fseek(dump_buf_file, 0, SEEK_SET);
		205
		206	id = btf__add_struct(btf, "s", 4);
		207	ASSERT_EQ(id, 4, "struct_id");
		208	err = btf__add_field(btf, "x", 1, 0, 0);
		209	ASSERT_OK(err, "field_ok");
		210	err = btf__add_field(btf, "s", 3, 32, 0);
		211	ASSERT_OK(err, "field_ok");
		212
		213	for (i = 1; i <= btf__get_nr_types(btf); i++) {
		214	err = btf_dump__dump_type(d, i);
		215	ASSERT_OK(err, "dump_type_ok");
		216	}
		217
		218	fflush(dump_buf_file);
		219	dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
		220	ASSERT_STREQ(dump_buf,
		221	"struct s___2 {\n"
		222	" enum {\n"
		223	" VAL___2 = 1,\n"
		224	" } x;\n"
		225	" struct s s;\n"
		226	"};\n\n" , "c_dump1");
		227
		228	err_out:
		229	fclose(dump_buf_file);
		230	free(dump_buf);
		231	btf_dump__free(d);
		232	btf__free(btf);
		233	}
		234
132	void test_btf_dump() {	235	void test_btf_dump() {
133	int i;	236	int i;
134		237
@@ -140,4 +243,6 @@ void test_btf_dump() { · 5 unmodified lines
140		243
141	test_btf_dump_case(i, &btf_dump_test_cases[i]);	244	test_btf_dump_case(i, &btf_dump_test_cases[i]);
142	}	245	}
		246	if (test__start_subtest("btf_dump: incremental"))
		247	test_btf_dump_incremental();
143	}	248	}

tools/testing/selftests/bpf/prog_tests/btf_endian.c+101−0

@@ -0,0 +1,101 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#define _GNU_SOURCE
	4	#include <string.h>
	5	#include <byteswap.h>
	6	#include <test_progs.h>
	7	#include <bpf/btf.h>
	8
	9	static int duration = 0;
	10
	11	void test_btf_endian() {
	12	#if __BYTE_ORDER == __LITTLE_ENDIAN
	13	enum btf_endianness endian = BTF_LITTLE_ENDIAN;
	14	#elif __BYTE_ORDER == __BIG_ENDIAN
	15	enum btf_endianness endian = BTF_BIG_ENDIAN;
	16	#else
	17	#error "Unrecognized __BYTE_ORDER"
	18	#endif
	19	enum btf_endianness swap_endian = 1 - endian;
	20	struct btf btf = NULL, swap_btf = NULL;
	21	const void raw_data, swap_raw_data;
	22	const struct btf_type *t;
	23	const struct btf_header *hdr;
	24	__u32 raw_sz, swap_raw_sz;
	25	int var_id;
	26
	27	/* Load BTF in native endianness */
	28	btf = btf__parse_elf("btf_dump_test_case_syntax.o", NULL);
	29	if (!ASSERT_OK_PTR(btf, "parse_native_btf"))
	30	goto err_out;
	31
	32	ASSERT_EQ(btf__endianness(btf), endian, "endian");
	33	btf__set_endianness(btf, swap_endian);
	34	ASSERT_EQ(btf__endianness(btf), swap_endian, "endian");
	35
	36	/* Get raw BTF data in non-native endianness... */
	37	raw_data = btf__get_raw_data(btf, &raw_sz);
	38	if (!ASSERT_OK_PTR(raw_data, "raw_data_inverted"))
	39	goto err_out;
	40
	41	/* ...and open it as a new BTF instance */
	42	swap_btf = btf__new(raw_data, raw_sz);
	43	if (!ASSERT_OK_PTR(swap_btf, "parse_swap_btf"))
	44	goto err_out;
	45
	46	ASSERT_EQ(btf__endianness(swap_btf), swap_endian, "endian");
	47	ASSERT_EQ(btf__get_nr_types(swap_btf), btf__get_nr_types(btf), "nr_types");
	48
	49	swap_raw_data = btf__get_raw_data(swap_btf, &swap_raw_sz);
	50	if (!ASSERT_OK_PTR(swap_raw_data, "swap_raw_data"))
	51	goto err_out;
	52
	53	/* both raw data should be identical (with non-native endianness) */
	54	ASSERT_OK(memcmp(raw_data, swap_raw_data, raw_sz), "mem_identical");
	55
	56	/* make sure that at least BTF header data is really swapped */
	57	hdr = swap_raw_data;
	58	ASSERT_EQ(bswap_16(hdr->magic), BTF_MAGIC, "btf_magic_swapped");
	59	ASSERT_EQ(raw_sz, swap_raw_sz, "raw_sizes");
	60
	61	/* swap it back to native endianness */
	62	btf__set_endianness(swap_btf, endian);
	63	swap_raw_data = btf__get_raw_data(swap_btf, &swap_raw_sz);
	64	if (!ASSERT_OK_PTR(swap_raw_data, "swap_raw_data"))
	65	goto err_out;
	66
	67	/* now header should have native BTF_MAGIC */
	68	hdr = swap_raw_data;
	69	ASSERT_EQ(hdr->magic, BTF_MAGIC, "btf_magic_native");
	70	ASSERT_EQ(raw_sz, swap_raw_sz, "raw_sizes");
	71
	72	/* now modify original BTF */
	73	var_id = btf__add_var(btf, "some_var", BTF_VAR_GLOBAL_ALLOCATED, 1);
	74	CHECK(var_id <= 0, "var_id", "failed %d\n", var_id);
	75
	76	btf__free(swap_btf);
	77	swap_btf = NULL;
	78
	79	btf__set_endianness(btf, swap_endian);
	80	raw_data = btf__get_raw_data(btf, &raw_sz);
	81	if (!ASSERT_OK_PTR(raw_data, "raw_data_inverted"))
	82	goto err_out;
	83
	84	/* and re-open swapped raw data again */
	85	swap_btf = btf__new(raw_data, raw_sz);
	86	if (!ASSERT_OK_PTR(swap_btf, "parse_swap_btf"))
	87	goto err_out;
	88
	89	ASSERT_EQ(btf__endianness(swap_btf), swap_endian, "endian");
	90	ASSERT_EQ(btf__get_nr_types(swap_btf), btf__get_nr_types(btf), "nr_types");
	91
	92	/* the type should appear as if it was stored in native endianness */
	93	t = btf__type_by_id(swap_btf, var_id);
	94	ASSERT_STREQ(btf__str_by_offset(swap_btf, t->name_off), "some_var", "var_name");
	95	ASSERT_EQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_ALLOCATED, "var_linkage");
	96	ASSERT_EQ(t->type, 1, "var_type");
	97
	98	err_out:
	99	btf__free(btf);
	100	btf__free(swap_btf);
	101	}

tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c+234−0

@@ -0,0 +1,234 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3
	4	#define _GNU_SOURCE
	5	#include <netinet/in.h>
	6	#include <arpa/inet.h>
	7	#include <unistd.h>
	8	#include <stdlib.h>
	9	#include <string.h>
	10	#include <errno.h>
	11	#include <sched.h>
	12	#include <linux/compiler.h>
	13	#include <bpf/libbpf.h>
	14
	15	#include "network_helpers.h"
	16	#include "test_progs.h"
	17	#include "test_btf_skc_cls_ingress.skel.h"
	18
	19	static struct test_btf_skc_cls_ingress *skel;
	20	struct sockaddr_in6 srv_sa6;
	21	static __u32 duration;
	22
	23	#define PROG_PIN_FILE "/sys/fs/bpf/btf_skc_cls_ingress"
	24
	25	static int write_sysctl(const char sysctl, const char value)
	26	{
	27	int fd, err, len;
	28
	29	fd = open(sysctl, O_WRONLY);
	30	if (CHECK(fd == -1, "open sysctl", "open(%s): %s (%d)\n",
	31	sysctl, strerror(errno), errno))
	32	return -1;
	33
	34	len = strlen(value);
	35	err = write(fd, value, len);
	36	close(fd);
	37	if (CHECK(err != len, "write sysctl",
	38	"write(%s, %s, %d): err:%d %s (%d)\n",
	39	sysctl, value, len, err, strerror(errno), errno))
	40	return -1;
	41
	42	return 0;
	43	}
	44
	45	static int prepare_netns(void)
	46	{
	47	if (CHECK(unshare(CLONE_NEWNET), "create netns",
	48	"unshare(CLONE_NEWNET): %s (%d)",
	49	strerror(errno), errno))
	50	return -1;
	51
	52	if (CHECK(system("ip link set dev lo up"),
	53	"ip link set dev lo up", "failed\n"))
	54	return -1;
	55
	56	if (CHECK(system("tc qdisc add dev lo clsact"),
	57	"tc qdisc add dev lo clsact", "failed\n"))
	58	return -1;
	59
	60	if (CHECK(system("tc filter add dev lo ingress bpf direct-action object-pinned " PROG_PIN_FILE),
	61	"install tc cls-prog at ingress", "failed\n"))
	62	return -1;
	63
	64	/* Ensure 20 bytes options (i.e. in total 40 bytes tcp header) for the
	65	* bpf_tcp_gen_syncookie() helper.
	66	*/
	67	if (write_sysctl("/proc/sys/net/ipv4/tcp_window_scaling", "1") \|\|
	68	write_sysctl("/proc/sys/net/ipv4/tcp_timestamps", "1") \|\|
	69	write_sysctl("/proc/sys/net/ipv4/tcp_sack", "1"))
	70	return -1;
	71
	72	return 0;
	73	}
	74
	75	static void reset_test(void)
	76	{
	77	memset(&skel->bss->srv_sa6, 0, sizeof(skel->bss->srv_sa6));
	78	skel->bss->listen_tp_sport = 0;
	79	skel->bss->req_sk_sport = 0;
	80	skel->bss->recv_cookie = 0;
	81	skel->bss->gen_cookie = 0;
	82	skel->bss->linum = 0;
	83	}
	84
	85	static void print_err_line(void)
	86	{
	87	if (skel->bss->linum)
	88	printf("bpf prog error at line %u\n", skel->bss->linum);
	89	}
	90
	91	static void test_conn(void)
	92	{
	93	int listen_fd = -1, cli_fd = -1, err;
	94	socklen_t addrlen = sizeof(srv_sa6);
	95	int srv_port;
	96
	97	if (write_sysctl("/proc/sys/net/ipv4/tcp_syncookies", "1"))
	98	return;
	99
	100	listen_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
	101	if (CHECK_FAIL(listen_fd == -1))
	102	return;
	103
	104	err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen);
	105	if (CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d\n", err,
	106	errno))
	107	goto done;
	108	memcpy(&skel->bss->srv_sa6, &srv_sa6, sizeof(srv_sa6));
	109	srv_port = ntohs(srv_sa6.sin6_port);
	110
	111	cli_fd = connect_to_fd(listen_fd, 0);
	112	if (CHECK_FAIL(cli_fd == -1))
	113	goto done;
	114
	115	if (CHECK(skel->bss->listen_tp_sport != srv_port \|\|
	116	skel->bss->req_sk_sport != srv_port,
	117	"Unexpected sk src port",
	118	"listen_tp_sport:%u req_sk_sport:%u expected:%u\n",
	119	skel->bss->listen_tp_sport, skel->bss->req_sk_sport,
	120	srv_port))
	121	goto done;
	122
	123	if (CHECK(skel->bss->gen_cookie \|\| skel->bss->recv_cookie,
	124	"Unexpected syncookie states",
	125	"gen_cookie:%u recv_cookie:%u\n",
	126	skel->bss->gen_cookie, skel->bss->recv_cookie))
	127	goto done;
	128
	129	CHECK(skel->bss->linum, "bpf prog detected error", "at line %u\n",
	130	skel->bss->linum);
	131
	132	done:
	133	if (listen_fd != -1)
	134	close(listen_fd);
	135	if (cli_fd != -1)
	136	close(cli_fd);
	137	}
	138
	139	static void test_syncookie(void)
	140	{
	141	int listen_fd = -1, cli_fd = -1, err;
	142	socklen_t addrlen = sizeof(srv_sa6);
	143	int srv_port;
	144
	145	/* Enforce syncookie mode */
	146	if (write_sysctl("/proc/sys/net/ipv4/tcp_syncookies", "2"))
	147	return;
	148
	149	listen_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
	150	if (CHECK_FAIL(listen_fd == -1))
	151	return;
	152
	153	err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen);
	154	if (CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d\n", err,
	155	errno))
	156	goto done;
	157	memcpy(&skel->bss->srv_sa6, &srv_sa6, sizeof(srv_sa6));
	158	srv_port = ntohs(srv_sa6.sin6_port);
	159
	160	cli_fd = connect_to_fd(listen_fd, 0);
	161	if (CHECK_FAIL(cli_fd == -1))
	162	goto done;
	163
	164	if (CHECK(skel->bss->listen_tp_sport != srv_port,
	165	"Unexpected tp src port",
	166	"listen_tp_sport:%u expected:%u\n",
	167	skel->bss->listen_tp_sport, srv_port))
	168	goto done;
	169
	170	if (CHECK(skel->bss->req_sk_sport,
	171	"Unexpected req_sk src port",
	172	"req_sk_sport:%u expected:0\n",
	173	skel->bss->req_sk_sport))
	174	goto done;
	175
	176	if (CHECK(!skel->bss->gen_cookie \|\|
	177	skel->bss->gen_cookie != skel->bss->recv_cookie,
	178	"Unexpected syncookie states",
	179	"gen_cookie:%u recv_cookie:%u\n",
	180	skel->bss->gen_cookie, skel->bss->recv_cookie))
	181	goto done;
	182
	183	CHECK(skel->bss->linum, "bpf prog detected error", "at line %u\n",
	184	skel->bss->linum);
	185
	186	done:
	187	if (listen_fd != -1)
	188	close(listen_fd);
	189	if (cli_fd != -1)
	190	close(cli_fd);
	191	}
	192
	193	struct test {
	194	const char *desc;
	195	void (*run)(void);
	196	};
	197
	198	#define DEF_TEST(name) { #name, test_##name }
	199	static struct test tests[] = {
	200	DEF_TEST(conn),
	201	DEF_TEST(syncookie),
	202	};
	203
	204	void test_btf_skc_cls_ingress(void)
	205	{
	206	int i, err;
	207
	208	skel = test_btf_skc_cls_ingress__open_and_load();
	209	if (CHECK(!skel, "test_btf_skc_cls_ingress__open_and_load", "failed\n"))
	210	return;
	211
	212	err = bpf_program__pin(skel->progs.cls_ingress, PROG_PIN_FILE);
	213	if (CHECK(err, "bpf_program__pin",
	214	"cannot pin bpf prog to %s. err:%d\n", PROG_PIN_FILE, err)) {
	215	test_btf_skc_cls_ingress__destroy(skel);
	216	return;
	217	}
	218
	219	for (i = 0; i < ARRAY_SIZE(tests); i++) {
	220	if (!test__start_subtest(tests[i].desc))
	221	continue;
	222
	223	if (prepare_netns())
	224	break;
	225
	226	tests[i].run();
	227
	228	print_err_line();
	229	reset_test();
	230	}
	231
	232	bpf_program__unpin(skel->progs.cls_ingress, PROG_PIN_FILE);
	233	test_btf_skc_cls_ingress__destroy(skel);
	234	}

tools/testing/selftests/bpf/prog_tests/btf_write.c+244−0

@@ -0,0 +1,244 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3	#include <test_progs.h>
	4	#include <bpf/btf.h>
	5
	6	static int duration = 0;
	7
	8	void test_btf_write() {
	9	const struct btf_var_secinfo *vi;
	10	const struct btf_type *t;
	11	const struct btf_member *m;
	12	const struct btf_enum *v;
	13	const struct btf_param *p;
	14	struct btf *btf;
	15	int id, err, str_off;
	16
	17	btf = btf__new_empty();
	18	if (CHECK(IS_ERR(btf), "new_empty", "failed: %ld\n", PTR_ERR(btf)))
	19	return;
	20
	21	str_off = btf__find_str(btf, "int");
	22	ASSERT_EQ(str_off, -ENOENT, "int_str_missing_off");
	23
	24	str_off = btf__add_str(btf, "int");
	25	ASSERT_EQ(str_off, 1, "int_str_off");
	26
	27	str_off = btf__find_str(btf, "int");
	28	ASSERT_EQ(str_off, 1, "int_str_found_off");
	29
	30	/* BTF_KIND_INT */
	31	id = btf__add_int(btf, "int", 4, BTF_INT_SIGNED);
	32	ASSERT_EQ(id, 1, "int_id");
	33
	34	t = btf__type_by_id(btf, 1);
	35	/* should re-use previously added "int" string */
	36	ASSERT_EQ(t->name_off, str_off, "int_name_off");
	37	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "int", "int_name");
	38	ASSERT_EQ(btf_kind(t), BTF_KIND_INT, "int_kind");
	39	ASSERT_EQ(t->size, 4, "int_sz");
	40	ASSERT_EQ(btf_int_encoding(t), BTF_INT_SIGNED, "int_enc");
	41	ASSERT_EQ(btf_int_bits(t), 32, "int_bits");
	42
	43	/* invalid int size */
	44	id = btf__add_int(btf, "bad sz int", 7, 0);
	45	ASSERT_ERR(id, "int_bad_sz");
	46	/* invalid encoding */
	47	id = btf__add_int(btf, "bad enc int", 4, 123);
	48	ASSERT_ERR(id, "int_bad_enc");
	49	/* NULL name */
	50	id = btf__add_int(btf, NULL, 4, 0);
	51	ASSERT_ERR(id, "int_bad_null_name");
	52	/* empty name */
	53	id = btf__add_int(btf, "", 4, 0);
	54	ASSERT_ERR(id, "int_bad_empty_name");
	55
	56	/* PTR/CONST/VOLATILE/RESTRICT */
	57	id = btf__add_ptr(btf, 1);
	58	ASSERT_EQ(id, 2, "ptr_id");
	59	t = btf__type_by_id(btf, 2);
	60	ASSERT_EQ(btf_kind(t), BTF_KIND_PTR, "ptr_kind");
	61	ASSERT_EQ(t->type, 1, "ptr_type");
	62
	63	id = btf__add_const(btf, 5); /* points forward to restrict */
	64	ASSERT_EQ(id, 3, "const_id");
	65	t = btf__type_by_id(btf, 3);
	66	ASSERT_EQ(btf_kind(t), BTF_KIND_CONST, "const_kind");
	67	ASSERT_EQ(t->type, 5, "const_type");
	68
	69	id = btf__add_volatile(btf, 3);
	70	ASSERT_EQ(id, 4, "volatile_id");
	71	t = btf__type_by_id(btf, 4);
	72	ASSERT_EQ(btf_kind(t), BTF_KIND_VOLATILE, "volatile_kind");
	73	ASSERT_EQ(t->type, 3, "volatile_type");
	74
	75	id = btf__add_restrict(btf, 4);
	76	ASSERT_EQ(id, 5, "restrict_id");
	77	t = btf__type_by_id(btf, 5);
	78	ASSERT_EQ(btf_kind(t), BTF_KIND_RESTRICT, "restrict_kind");
	79	ASSERT_EQ(t->type, 4, "restrict_type");
	80
	81	/* ARRAY */
	82	id = btf__add_array(btf, 1, 2, 10); /* int [10] /
	83	ASSERT_EQ(id, 6, "array_id");
	84	t = btf__type_by_id(btf, 6);
	85	ASSERT_EQ(btf_kind(t), BTF_KIND_ARRAY, "array_kind");
	86	ASSERT_EQ(btf_array(t)->index_type, 1, "array_index_type");
	87	ASSERT_EQ(btf_array(t)->type, 2, "array_elem_type");
	88	ASSERT_EQ(btf_array(t)->nelems, 10, "array_nelems");
	89
	90	/* STRUCT */
	91	err = btf__add_field(btf, "field", 1, 0, 0);
	92	ASSERT_ERR(err, "no_struct_field");
	93	id = btf__add_struct(btf, "s1", 8);
	94	ASSERT_EQ(id, 7, "struct_id");
	95	err = btf__add_field(btf, "f1", 1, 0, 0);
	96	ASSERT_OK(err, "f1_res");
	97	err = btf__add_field(btf, "f2", 1, 32, 16);
	98	ASSERT_OK(err, "f2_res");
	99
	100	t = btf__type_by_id(btf, 7);
	101	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "s1", "struct_name");
	102	ASSERT_EQ(btf_kind(t), BTF_KIND_STRUCT, "struct_kind");
	103	ASSERT_EQ(btf_vlen(t), 2, "struct_vlen");
	104	ASSERT_EQ(btf_kflag(t), true, "struct_kflag");
	105	ASSERT_EQ(t->size, 8, "struct_sz");
	106	m = btf_members(t) + 0;
	107	ASSERT_STREQ(btf__str_by_offset(btf, m->name_off), "f1", "f1_name");
	108	ASSERT_EQ(m->type, 1, "f1_type");
	109	ASSERT_EQ(btf_member_bit_offset(t, 0), 0, "f1_bit_off");
	110	ASSERT_EQ(btf_member_bitfield_size(t, 0), 0, "f1_bit_sz");
	111	m = btf_members(t) + 1;
	112	ASSERT_STREQ(btf__str_by_offset(btf, m->name_off), "f2", "f2_name");
	113	ASSERT_EQ(m->type, 1, "f2_type");
	114	ASSERT_EQ(btf_member_bit_offset(t, 1), 32, "f2_bit_off");
	115	ASSERT_EQ(btf_member_bitfield_size(t, 1), 16, "f2_bit_sz");
	116
	117	/* UNION */
	118	id = btf__add_union(btf, "u1", 8);
	119	ASSERT_EQ(id, 8, "union_id");
	120
	121	/* invalid, non-zero offset */
	122	err = btf__add_field(btf, "field", 1, 1, 0);
	123	ASSERT_ERR(err, "no_struct_field");
	124
	125	err = btf__add_field(btf, "f1", 1, 0, 16);
	126	ASSERT_OK(err, "f1_res");
	127
	128	t = btf__type_by_id(btf, 8);
	129	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "u1", "union_name");
	130	ASSERT_EQ(btf_kind(t), BTF_KIND_UNION, "union_kind");
	131	ASSERT_EQ(btf_vlen(t), 1, "union_vlen");
	132	ASSERT_EQ(btf_kflag(t), true, "union_kflag");
	133	ASSERT_EQ(t->size, 8, "union_sz");
	134	m = btf_members(t) + 0;
	135	ASSERT_STREQ(btf__str_by_offset(btf, m->name_off), "f1", "f1_name");
	136	ASSERT_EQ(m->type, 1, "f1_type");
	137	ASSERT_EQ(btf_member_bit_offset(t, 0), 0, "f1_bit_off");
	138	ASSERT_EQ(btf_member_bitfield_size(t, 0), 16, "f1_bit_sz");
	139
	140	/* ENUM */
	141	id = btf__add_enum(btf, "e1", 4);
	142	ASSERT_EQ(id, 9, "enum_id");
	143	err = btf__add_enum_value(btf, "v1", 1);
	144	ASSERT_OK(err, "v1_res");
	145	err = btf__add_enum_value(btf, "v2", 2);
	146	ASSERT_OK(err, "v2_res");
	147
	148	t = btf__type_by_id(btf, 9);
	149	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "e1", "enum_name");
	150	ASSERT_EQ(btf_kind(t), BTF_KIND_ENUM, "enum_kind");
	151	ASSERT_EQ(btf_vlen(t), 2, "enum_vlen");
	152	ASSERT_EQ(t->size, 4, "enum_sz");
	153	v = btf_enum(t) + 0;
	154	ASSERT_STREQ(btf__str_by_offset(btf, v->name_off), "v1", "v1_name");
	155	ASSERT_EQ(v->val, 1, "v1_val");
	156	v = btf_enum(t) + 1;
	157	ASSERT_STREQ(btf__str_by_offset(btf, v->name_off), "v2", "v2_name");
	158	ASSERT_EQ(v->val, 2, "v2_val");
	159
	160	/* FWDs */
	161	id = btf__add_fwd(btf, "struct_fwd", BTF_FWD_STRUCT);
	162	ASSERT_EQ(id, 10, "struct_fwd_id");
	163	t = btf__type_by_id(btf, 10);
	164	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "struct_fwd", "fwd_name");
	165	ASSERT_EQ(btf_kind(t), BTF_KIND_FWD, "fwd_kind");
	166	ASSERT_EQ(btf_kflag(t), 0, "fwd_kflag");
	167
	168	id = btf__add_fwd(btf, "union_fwd", BTF_FWD_UNION);
	169	ASSERT_EQ(id, 11, "union_fwd_id");
	170	t = btf__type_by_id(btf, 11);
	171	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "union_fwd", "fwd_name");
	172	ASSERT_EQ(btf_kind(t), BTF_KIND_FWD, "fwd_kind");
	173	ASSERT_EQ(btf_kflag(t), 1, "fwd_kflag");
	174
	175	id = btf__add_fwd(btf, "enum_fwd", BTF_FWD_ENUM);
	176	ASSERT_EQ(id, 12, "enum_fwd_id");
	177	t = btf__type_by_id(btf, 12);
	178	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "enum_fwd", "fwd_name");
	179	ASSERT_EQ(btf_kind(t), BTF_KIND_ENUM, "enum_fwd_kind");
	180	ASSERT_EQ(btf_vlen(t), 0, "enum_fwd_kind");
	181	ASSERT_EQ(t->size, 4, "enum_fwd_sz");
	182
	183	/* TYPEDEF */
	184	id = btf__add_typedef(btf, "typedef1", 1);
	185	ASSERT_EQ(id, 13, "typedef_fwd_id");
	186	t = btf__type_by_id(btf, 13);
	187	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "typedef1", "typedef_name");
	188	ASSERT_EQ(btf_kind(t), BTF_KIND_TYPEDEF, "typedef_kind");
	189	ASSERT_EQ(t->type, 1, "typedef_type");
	190
	191	/* FUNC & FUNC_PROTO */
	192	id = btf__add_func(btf, "func1", BTF_FUNC_GLOBAL, 15);
	193	ASSERT_EQ(id, 14, "func_id");
	194	t = btf__type_by_id(btf, 14);
	195	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "func1", "func_name");
	196	ASSERT_EQ(t->type, 15, "func_type");
	197	ASSERT_EQ(btf_kind(t), BTF_KIND_FUNC, "func_kind");
	198	ASSERT_EQ(btf_vlen(t), BTF_FUNC_GLOBAL, "func_vlen");
	199
	200	id = btf__add_func_proto(btf, 1);
	201	ASSERT_EQ(id, 15, "func_proto_id");
	202	err = btf__add_func_param(btf, "p1", 1);
	203	ASSERT_OK(err, "p1_res");
	204	err = btf__add_func_param(btf, "p2", 2);
	205	ASSERT_OK(err, "p2_res");
	206
	207	t = btf__type_by_id(btf, 15);
	208	ASSERT_EQ(btf_kind(t), BTF_KIND_FUNC_PROTO, "func_proto_kind");
	209	ASSERT_EQ(btf_vlen(t), 2, "func_proto_vlen");
	210	ASSERT_EQ(t->type, 1, "func_proto_ret_type");
	211	p = btf_params(t) + 0;
	212	ASSERT_STREQ(btf__str_by_offset(btf, p->name_off), "p1", "p1_name");
	213	ASSERT_EQ(p->type, 1, "p1_type");
	214	p = btf_params(t) + 1;
	215	ASSERT_STREQ(btf__str_by_offset(btf, p->name_off), "p2", "p2_name");
	216	ASSERT_EQ(p->type, 2, "p2_type");
	217
	218	/* VAR */
	219	id = btf__add_var(btf, "var1", BTF_VAR_GLOBAL_ALLOCATED, 1);
	220	ASSERT_EQ(id, 16, "var_id");
	221	t = btf__type_by_id(btf, 16);
	222	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "var1", "var_name");
	223	ASSERT_EQ(btf_kind(t), BTF_KIND_VAR, "var_kind");
	224	ASSERT_EQ(t->type, 1, "var_type");
	225	ASSERT_EQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_ALLOCATED, "var_type");
	226
	227	/* DATASECT */
	228	id = btf__add_datasec(btf, "datasec1", 12);
	229	ASSERT_EQ(id, 17, "datasec_id");
	230	err = btf__add_datasec_var_info(btf, 1, 4, 8);
	231	ASSERT_OK(err, "v1_res");
	232
	233	t = btf__type_by_id(btf, 17);
	234	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "datasec1", "datasec_name");
	235	ASSERT_EQ(t->size, 12, "datasec_sz");
	236	ASSERT_EQ(btf_kind(t), BTF_KIND_DATASEC, "datasec_kind");
	237	ASSERT_EQ(btf_vlen(t), 1, "datasec_vlen");
	238	vi = btf_var_secinfos(t) + 0;
	239	ASSERT_EQ(vi->type, 1, "v1_type");
	240	ASSERT_EQ(vi->offset, 4, "v1_off");
	241	ASSERT_EQ(vi->size, 8, "v1_sz");
	242
	243	btf__free(btf);
	244	}

tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c+180−32

@@ -2,36 +2,79 @@ · 1 unmodified line
2	/* Copyright (c) 2019 Facebook */	2	/* Copyright (c) 2019 Facebook */
3	#include <test_progs.h>	3	#include <test_progs.h>
4	#include <network_helpers.h>	4	#include <network_helpers.h>
		5	#include <bpf/btf.h>
		6
		7	typedef int (test_cb)(struct bpf_object obj);
		8
		9	static int check_data_map(struct bpf_object *obj, int prog_cnt, bool reset)
		10	{
		11	struct bpf_map data_map = NULL, map;
		12	__u64 *result = NULL;
		13	const int zero = 0;
		14	__u32 duration = 0;
		15	int ret = -1, i;
		16
		17	result = malloc((prog_cnt + 32 /* spare /) sizeof(__u64));
		18	if (CHECK(!result, "alloc_memory", "failed to alloc memory"))
		19	return -ENOMEM;
		20
		21	bpf_object__for_each_map(map, obj)
		22	if (bpf_map__is_internal(map)) {
		23	data_map = map;
		24	break;
		25	}
		26	if (CHECK(!data_map, "find_data_map", "data map not found\n"))
		27	goto out;
		28
		29	ret = bpf_map_lookup_elem(bpf_map__fd(data_map), &zero, result);
		30	if (CHECK(ret, "get_result",
		31	"failed to get output data: %d\n", ret))
		32	goto out;
		33
		34	for (i = 0; i < prog_cnt; i++) {
		35	if (CHECK(result[i] != 1, "result",
		36	"fexit_bpf2bpf result[%d] failed err %llu\n",
		37	i, result[i]))
		38	goto out;
		39	result[i] = 0;
		40	}
		41	if (reset) {
		42	ret = bpf_map_update_elem(bpf_map__fd(data_map), &zero, result, 0);
		43	if (CHECK(ret, "reset_result", "failed to reset result\n"))
		44	goto out;
		45	}
		46
		47	ret = 0;
		48	out:
		49	free(result);
		50	return ret;
		51	}
5		52
6	static void test_fexit_bpf2bpf_common(const char *obj_file,	53	static void test_fexit_bpf2bpf_common(const char *obj_file,
7	const char *target_obj_file,	54	const char *target_obj_file,
8	int prog_cnt,	55	int prog_cnt,
9	const char **prog_name,	56	const char **prog_name,
10	bool run_prog)	57	bool run_prog,
		58	test_cb cb)
11	{	59	{
12	struct bpf_object obj = NULL, pkt_obj;	60	struct bpf_object obj = NULL, tgt_obj;
13	int err, pkt_fd, i;
14	struct bpf_link **link = NULL;
15	struct bpf_program **prog = NULL;	61	struct bpf_program **prog = NULL;
		62	struct bpf_link **link = NULL;
16	__u32 duration = 0, retval;	63	__u32 duration = 0, retval;
17	struct bpf_map *data_map;	64	int err, tgt_fd, i;
18	const int zero = 0;
19	__u64 *result = NULL;
20		65
21	err = bpf_prog_load(target_obj_file, BPF_PROG_TYPE_UNSPEC,	66	err = bpf_prog_load(target_obj_file, BPF_PROG_TYPE_UNSPEC,
22	&pkt_obj, &pkt_fd);	67	&tgt_obj, &tgt_fd);
23	if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n",	68	if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n",
24	target_obj_file, err, errno))	69	target_obj_file, err, errno))
25	return;	70	return;
26	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,	71	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts,
27	.attach_prog_fd = pkt_fd,	72	.attach_prog_fd = tgt_fd,
28	);	73	);
29		74
30	link = calloc(sizeof(struct bpf_link *), prog_cnt);	75	link = calloc(sizeof(struct bpf_link *), prog_cnt);
31	prog = calloc(sizeof(struct bpf_program *), prog_cnt);	76	prog = calloc(sizeof(struct bpf_program *), prog_cnt);
32	result = malloc((prog_cnt + 32 /* spare /) sizeof(__u64));	77	if (CHECK(!link \|\| !prog, "alloc_memory", "failed to alloc memory"))
33	if (CHECK(!link \|\| !prog \|\| !result, "alloc_memory",
34	"failed to alloc memory"))
35	goto close_prog;	78	goto close_prog;
36		79
37	obj = bpf_object__open_file(obj_file, &opts);	80	obj = bpf_object__open_file(obj_file, &opts);
@@ -53,39 +96,33 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, · 15 unmodified lines
53	goto close_prog;	96	goto close_prog;
54	}	97	}
55		98
56	if (!run_prog)	99	if (cb) {
57	goto close_prog;	100	err = cb(obj);
		101	if (err)
		102	goto close_prog;
		103	}
58		104
59	data_map = bpf_object__find_map_by_name(obj, "fexit_bp.bss");	105	if (!run_prog)
60	if (CHECK(!data_map, "find_data_map", "data map not found\n"))
61	goto close_prog;	106	goto close_prog;
62		107
63	err = bpf_prog_test_run(pkt_fd, 1, &pkt_v6, sizeof(pkt_v6),	108	err = bpf_prog_test_run(tgt_fd, 1, &pkt_v6, sizeof(pkt_v6),
64	NULL, NULL, &retval, &duration);	109	NULL, NULL, &retval, &duration);
65	CHECK(err \|\| retval, "ipv6",	110	CHECK(err \|\| retval, "ipv6",
66	"err %d errno %d retval %d duration %d\n",	111	"err %d errno %d retval %d duration %d\n",
67	err, errno, retval, duration);	112	err, errno, retval, duration);
68		113
69	err = bpf_map_lookup_elem(bpf_map__fd(data_map), &zero, result);	114	if (check_data_map(obj, prog_cnt, false))
70	if (CHECK(err, "get_result",
71	"failed to get output data: %d\n", err))
72	goto close_prog;	115	goto close_prog;
73		116
74	for (i = 0; i < prog_cnt; i++)
75	if (CHECK(result[i] != 1, "result", "fexit_bpf2bpf failed err %llu\n",
76	result[i]))
77	goto close_prog;
78
79	close_prog:	117	close_prog:
80	for (i = 0; i < prog_cnt; i++)	118	for (i = 0; i < prog_cnt; i++)
81	if (!IS_ERR_OR_NULL(link[i]))	119	if (!IS_ERR_OR_NULL(link[i]))
82	bpf_link__destroy(link[i]);	120	bpf_link__destroy(link[i]);
83	if (!IS_ERR_OR_NULL(obj))	121	if (!IS_ERR_OR_NULL(obj))
84	bpf_object__close(obj);	122	bpf_object__close(obj);
85	bpf_object__close(pkt_obj);	123	bpf_object__close(tgt_obj);
86	free(link);	124	free(link);
87	free(prog);	125	free(prog);
88	free(result);
89	}	126	}
90		127
91	static void test_target_no_callees(void)	128	static void test_target_no_callees(void)
@@ -96,7 +133,7 @@ static void test_target_no_callees(void) · 4 unmodified lines
96	test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.o",	133	test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.o",
97	"./test_pkt_md_access.o",	134	"./test_pkt_md_access.o",
98	ARRAY_SIZE(prog_name),	135	ARRAY_SIZE(prog_name),
99	prog_name, true);	136	prog_name, true, NULL);
100	}	137	}
101		138
102	static void test_target_yes_callees(void)	139	static void test_target_yes_callees(void)
@@ -110,7 +147,7 @@ static void test_target_yes_callees(void) · 7 unmodified lines
110	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",	147	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",
111	"./test_pkt_access.o",	148	"./test_pkt_access.o",
112	ARRAY_SIZE(prog_name),	149	ARRAY_SIZE(prog_name),
113	prog_name, true);	150	prog_name, true, NULL);
114	}	151	}
115		152
116	static void test_func_replace(void)	153	static void test_func_replace(void)
@@ -128,7 +165,7 @@ static void test_func_replace(void) · 11 unmodified lines
128	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",	165	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",
129	"./test_pkt_access.o",	166	"./test_pkt_access.o",
130	ARRAY_SIZE(prog_name),	167	ARRAY_SIZE(prog_name),
131	prog_name, true);	168	prog_name, true, NULL);
132	}	169	}
133		170
134	static void test_func_replace_verify(void)	171	static void test_func_replace_verify(void)
@@ -139,9 +176,116 @@ static void test_func_replace_verify(void) · 4 unmodified lines
139	test_fexit_bpf2bpf_common("./freplace_connect4.o",	176	test_fexit_bpf2bpf_common("./freplace_connect4.o",
140	"./connect4_prog.o",	177	"./connect4_prog.o",
141	ARRAY_SIZE(prog_name),	178	ARRAY_SIZE(prog_name),
142	prog_name, false);	179	prog_name, false, NULL);
143	}	180	}
144		181
		182	static int test_second_attach(struct bpf_object *obj)
		183	{
		184	const char *prog_name = "freplace/get_constant";
		185	const char tgt_name = prog_name + 9; / cut off freplace/ */
		186	const char *tgt_obj_file = "./test_pkt_access.o";
		187	struct bpf_program *prog = NULL;
		188	struct bpf_object *tgt_obj;
		189	__u32 duration = 0, retval;
		190	struct bpf_link *link;
		191	int err = 0, tgt_fd;
		192
		193	prog = bpf_object__find_program_by_title(obj, prog_name);
		194	if (CHECK(!prog, "find_prog", "prog %s not found\n", prog_name))
		195	return -ENOENT;
		196
		197	err = bpf_prog_load(tgt_obj_file, BPF_PROG_TYPE_UNSPEC,
		198	&tgt_obj, &tgt_fd);
		199	if (CHECK(err, "second_prog_load", "file %s err %d errno %d\n",
		200	tgt_obj_file, err, errno))
		201	return err;
		202
		203	link = bpf_program__attach_freplace(prog, tgt_fd, tgt_name);
		204	if (CHECK(IS_ERR(link), "second_link", "failed to attach second link prog_fd %d tgt_fd %d\n", bpf_program__fd(prog), tgt_fd))
		205	goto out;
		206
		207	err = bpf_prog_test_run(tgt_fd, 1, &pkt_v6, sizeof(pkt_v6),
		208	NULL, NULL, &retval, &duration);
		209	if (CHECK(err \|\| retval, "ipv6",
		210	"err %d errno %d retval %d duration %d\n",
		211	err, errno, retval, duration))
		212	goto out;
		213
		214	err = check_data_map(obj, 1, true);
		215	if (err)
		216	goto out;
		217
		218	out:
		219	bpf_link__destroy(link);
		220	bpf_object__close(tgt_obj);
		221	return err;
		222	}
		223
		224	static void test_func_replace_multi(void)
		225	{
		226	const char *prog_name[] = {
		227	"freplace/get_constant",
		228	};
		229	test_fexit_bpf2bpf_common("./freplace_get_constant.o",
		230	"./test_pkt_access.o",
		231	ARRAY_SIZE(prog_name),
		232	prog_name, true, test_second_attach);
		233	}
		234
		235	static void test_fmod_ret_freplace(void)
		236	{
		237	struct bpf_object freplace_obj = NULL, pkt_obj, *fmod_obj = NULL;
		238	const char *freplace_name = "./freplace_get_constant.o";
		239	const char *fmod_ret_name = "./fmod_ret_freplace.o";
		240	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts);
		241	const char *tgt_name = "./test_pkt_access.o";
		242	struct bpf_link *freplace_link = NULL;
		243	struct bpf_program *prog;
		244	__u32 duration = 0;
		245	int err, pkt_fd;
		246
		247	err = bpf_prog_load(tgt_name, BPF_PROG_TYPE_UNSPEC,
		248	&pkt_obj, &pkt_fd);
		249	/* the target prog should load fine */
		250	if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n",
		251	tgt_name, err, errno))
		252	return;
		253	opts.attach_prog_fd = pkt_fd;
		254
		255	freplace_obj = bpf_object__open_file(freplace_name, &opts);
		256	if (CHECK(IS_ERR_OR_NULL(freplace_obj), "freplace_obj_open",
		257	"failed to open %s: %ld\n", freplace_name,
		258	PTR_ERR(freplace_obj)))
		259	goto out;
		260
		261	err = bpf_object__load(freplace_obj);
		262	if (CHECK(err, "freplace_obj_load", "err %d\n", err))
		263	goto out;
		264
		265	prog = bpf_program__next(NULL, freplace_obj);
		266	freplace_link = bpf_program__attach_trace(prog);
		267	if (CHECK(IS_ERR(freplace_link), "freplace_attach_trace", "failed to link\n"))
		268	goto out;
		269
		270	opts.attach_prog_fd = bpf_program__fd(prog);
		271	fmod_obj = bpf_object__open_file(fmod_ret_name, &opts);
		272	if (CHECK(IS_ERR_OR_NULL(fmod_obj), "fmod_obj_open",
		273	"failed to open %s: %ld\n", fmod_ret_name,
		274	PTR_ERR(fmod_obj)))
		275	goto out;
		276
		277	err = bpf_object__load(fmod_obj);
		278	if (CHECK(!err, "fmod_obj_load", "loading fmod_ret should fail\n"))
		279	goto out;
		280
		281	out:
		282	bpf_link__destroy(freplace_link);
		283	bpf_object__close(freplace_obj);
		284	bpf_object__close(fmod_obj);
		285	bpf_object__close(pkt_obj);
		286	}
		287
		288
145	static void test_func_sockmap_update(void)	289	static void test_func_sockmap_update(void)
146	{	290	{
147	const char *prog_name[] = {	291	const char *prog_name[] = {
@@ -150,7 +294,7 @@ static void test_func_sockmap_update(void) · 2 unmodified lines
150	test_fexit_bpf2bpf_common("./freplace_cls_redirect.o",	294	test_fexit_bpf2bpf_common("./freplace_cls_redirect.o",
151	"./test_cls_redirect.o",	295	"./test_cls_redirect.o",
152	ARRAY_SIZE(prog_name),	296	ARRAY_SIZE(prog_name),
153	prog_name, false);	297	prog_name, false, NULL);
154	}	298	}
155		299
156	static void test_obj_load_failure_common(const char *obj_file,	300	static void test_obj_load_failure_common(const char *obj_file,
@@ -222,4 +366,8 @@ void test_fexit_bpf2bpf(void) · 65 unmodified lines
222	test_func_replace_return_code();	366	test_func_replace_return_code();
223	if (test__start_subtest("func_map_prog_compatibility"))	367	if (test__start_subtest("func_map_prog_compatibility"))
224	test_func_map_prog_compatibility();	368	test_func_map_prog_compatibility();
		369	if (test__start_subtest("func_replace_multi"))
		370	test_func_replace_multi();
		371	if (test__start_subtest("fmod_ret_freplace"))
		372	test_fmod_ret_freplace();
225	}	373	}

tools/testing/selftests/bpf/prog_tests/pe_preserve_elems.c+66−0

@@ -0,0 +1,66 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2019 Facebook */
	3	#include <test_progs.h>
	4	#include <linux/bpf.h>
	5	#include "test_pe_preserve_elems.skel.h"
	6
	7	static int duration;
	8
	9	static void test_one_map(struct bpf_map map, struct bpf_program prog,
	10	bool has_share_pe)
	11	{
	12	int err, key = 0, pfd = -1, mfd = bpf_map__fd(map);
	13	DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts);
	14	struct perf_event_attr attr = {
	15	.size = sizeof(struct perf_event_attr),
	16	.type = PERF_TYPE_SOFTWARE,
	17	.config = PERF_COUNT_SW_CPU_CLOCK,
	18	};
	19
	20	pfd = syscall(__NR_perf_event_open, &attr, 0 /* pid */,
	21	-1 /* cpu 0 /, -1 / group id /, 0 / flags */);
	22	if (CHECK(pfd < 0, "perf_event_open", "failed\n"))
	23	return;
	24
	25	err = bpf_map_update_elem(mfd, &key, &pfd, BPF_ANY);
	26	close(pfd);
	27	if (CHECK(err < 0, "bpf_map_update_elem", "failed\n"))
	28	return;
	29
	30	err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
	31	if (CHECK(err < 0, "bpf_prog_test_run_opts", "failed\n"))
	32	return;
	33	if (CHECK(opts.retval != 0, "bpf_perf_event_read_value",
	34	"failed with %d\n", opts.retval))
	35	return;
	36
	37	/* closing mfd, prog still holds a reference on map */
	38	close(mfd);
	39
	40	err = bpf_prog_test_run_opts(bpf_program__fd(prog), &opts);
	41	if (CHECK(err < 0, "bpf_prog_test_run_opts", "failed\n"))
	42	return;
	43
	44	if (has_share_pe) {
	45	CHECK(opts.retval != 0, "bpf_perf_event_read_value",
	46	"failed with %d\n", opts.retval);
	47	} else {
	48	CHECK(opts.retval != -ENOENT, "bpf_perf_event_read_value",
	49	"should have failed with %d, but got %d\n", -ENOENT,
	50	opts.retval);
	51	}
	52	}
	53
	54	void test_pe_preserve_elems(void)
	55	{
	56	struct test_pe_preserve_elems *skel;
	57
	58	skel = test_pe_preserve_elems__open_and_load();
	59	if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
	60	return;
	61
	62	test_one_map(skel->maps.array_1, skel->progs.read_array_1, false);
	63	test_one_map(skel->maps.array_2, skel->progs.read_array_2, true);
	64
	65	test_pe_preserve_elems__destroy(skel);
	66	}

tools/testing/selftests/bpf/prog_tests/raw_tp_test_run.c+96−0

@@ -0,0 +1,96 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2019 Facebook */
	3	#include <test_progs.h>
	4	#include <linux/bpf.h>
	5	#include "bpf/libbpf_internal.h"
	6	#include "test_raw_tp_test_run.skel.h"
	7
	8	static int duration;
	9
	10	void test_raw_tp_test_run(void)
	11	{
	12	struct bpf_prog_test_run_attr test_attr = {};
	13	int comm_fd = -1, err, nr_online, i, prog_fd;
	14	__u64 args[2] = {0x1234ULL, 0x5678ULL};
	15	int expected_retval = 0x1234 + 0x5678;
	16	struct test_raw_tp_test_run *skel;
	17	char buf[] = "new_name";
	18	bool *online = NULL;
	19	DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts,
	20	.ctx_in = args,
	21	.ctx_size_in = sizeof(args),
	22	.flags = BPF_F_TEST_RUN_ON_CPU,
	23	);
	24
	25	err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online,
	26	&nr_online);
	27	if (CHECK(err, "parse_cpu_mask_file", "err %d\n", err))
	28	return;
	29
	30	skel = test_raw_tp_test_run__open_and_load();
	31	if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
	32	goto cleanup;
	33
	34	err = test_raw_tp_test_run__attach(skel);
	35	if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
	36	goto cleanup;
	37
	38	comm_fd = open("/proc/self/comm", O_WRONLY\|O_TRUNC);
	39	if (CHECK(comm_fd < 0, "open /proc/self/comm", "err %d\n", errno))
	40	goto cleanup;
	41
	42	err = write(comm_fd, buf, sizeof(buf));
	43	CHECK(err < 0, "task rename", "err %d", errno);
	44
	45	CHECK(skel->bss->count == 0, "check_count", "didn't increase\n");
	46	CHECK(skel->data->on_cpu != 0xffffffff, "check_on_cpu", "got wrong value\n");
	47
	48	prog_fd = bpf_program__fd(skel->progs.rename);
	49	test_attr.prog_fd = prog_fd;
	50	test_attr.ctx_in = args;
	51	test_attr.ctx_size_in = sizeof(__u64);
	52
	53	err = bpf_prog_test_run_xattr(&test_attr);
	54	CHECK(err == 0, "test_run", "should fail for too small ctx\n");
	55
	56	test_attr.ctx_size_in = sizeof(args);
	57	err = bpf_prog_test_run_xattr(&test_attr);
	58	CHECK(err < 0, "test_run", "err %d\n", errno);
	59	CHECK(test_attr.retval != expected_retval, "check_retval",
	60	"expect 0x%x, got 0x%x\n", expected_retval, test_attr.retval);
	61
	62	for (i = 0; i < nr_online; i++) {
	63	if (!online[i])
	64	continue;
	65
	66	opts.cpu = i;
	67	opts.retval = 0;
	68	err = bpf_prog_test_run_opts(prog_fd, &opts);
	69	CHECK(err < 0, "test_run_opts", "err %d\n", errno);
	70	CHECK(skel->data->on_cpu != i, "check_on_cpu",
	71	"expect %d got %d\n", i, skel->data->on_cpu);
	72	CHECK(opts.retval != expected_retval,
	73	"check_retval", "expect 0x%x, got 0x%x\n",
	74	expected_retval, opts.retval);
	75	}
	76
	77	/* invalid cpu ID should fail with ENXIO */
	78	opts.cpu = 0xffffffff;
	79	err = bpf_prog_test_run_opts(prog_fd, &opts);
	80	CHECK(err != -1 \|\| errno != ENXIO,
	81	"test_run_opts_fail",
	82	"should failed with ENXIO\n");
	83
	84	/* non-zero cpu w/o BPF_F_TEST_RUN_ON_CPU should fail with EINVAL */
	85	opts.cpu = 1;
	86	opts.flags = 0;
	87	err = bpf_prog_test_run_opts(prog_fd, &opts);
	88	CHECK(err != -1 \|\| errno != EINVAL,
	89	"test_run_opts_fail",
	90	"should failed with EINVAL\n");
	91
	92	cleanup:
	93	close(comm_fd);
	94	test_raw_tp_test_run__destroy(skel);
	95	free(online);
	96	}

tools/testing/selftests/bpf/prog_tests/resolve_btfids.c+6−0

 	{ "func",    BTF_KIND_FUNC,    -1 },
 };
+/* Align the .BTF_ids section to 4 bytes */
+asm (
+".pushsection " BTF_IDS_SECTION " ,\"a\"; \n"
+".balign 4, 0;                            \n"
+".popsection;                             \n");
 BTF_ID_LIST(test_list_local)
 BTF_ID_UNUSED
 BTF_ID(typedef, S)

tools/testing/selftests/bpf/prog_tests/snprintf_btf.c+62−0

@@ -0,0 +1,62 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <test_progs.h>
	3	#include <linux/btf.h>
	4	#include "netif_receive_skb.skel.h"
	5
	6	/* Demonstrate that bpf_snprintf_btf succeeds and that various data types
	7	* are formatted correctly.
	8	*/
	9	void test_snprintf_btf(void)
	10	{
	11	struct netif_receive_skb *skel;
	12	struct netif_receive_skb__bss *bss;
	13	int err, duration = 0;
	14
	15	skel = netif_receive_skb__open();
	16	if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
	17	return;
	18
	19	err = netif_receive_skb__load(skel);
	20	if (CHECK(err, "skel_load", "failed to load skeleton: %d\n", err))
	21	goto cleanup;
	22
	23	bss = skel->bss;
	24
	25	err = netif_receive_skb__attach(skel);
	26	if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
	27	goto cleanup;
	28
	29	/* generate receive event */
	30	err = system("ping -c 1 127.0.0.1 > /dev/null");
	31	if (CHECK(err, "system", "ping failed: %d\n", err))
	32	goto cleanup;
	33
	34	if (bss->skip) {
	35	printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);
	36	test__skip();
	37	goto cleanup;
	38	}
	39
	40	/*
	41	* Make sure netif_receive_skb program was triggered
	42	* and it set expected return values from bpf_trace_printk()s
	43	* and all tests ran.
	44	*/
	45	if (CHECK(bss->ret <= 0,
	46	"bpf_snprintf_btf: got return value",
	47	"ret <= 0 %ld test %d\n", bss->ret, bss->ran_subtests))
	48	goto cleanup;
	49
	50	if (CHECK(bss->ran_subtests == 0, "check if subtests ran",
	51	"no subtests ran, did BPF program run?"))
	52	goto cleanup;
	53
	54	if (CHECK(bss->num_subtests != bss->ran_subtests,
	55	"check all subtests ran",
	56	"only ran %d of %d tests\n", bss->num_subtests,
	57	bss->ran_subtests))
	58	goto cleanup;
	59
	60	cleanup:
	61	netif_receive_skb__destroy(skel);
	62	}

tools/testing/selftests/bpf/prog_tests/sock_fields.c+382−0

@@ -0,0 +1,382 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2019 Facebook */
	3
	4	#include <netinet/in.h>
	5	#include <arpa/inet.h>
	6	#include <unistd.h>
	7	#include <stdlib.h>
	8	#include <string.h>
	9	#include <errno.h>
	10
	11	#include <bpf/bpf.h>
	12	#include <bpf/libbpf.h>
	13	#include <linux/compiler.h>
	14
	15	#include "network_helpers.h"
	16	#include "cgroup_helpers.h"
	17	#include "test_progs.h"
	18	#include "bpf_rlimit.h"
	19	#include "test_sock_fields.skel.h"
	20
	21	enum bpf_linum_array_idx {
	22	EGRESS_LINUM_IDX,
	23	INGRESS_LINUM_IDX,
	24	__NR_BPF_LINUM_ARRAY_IDX,
	25	};
	26
	27	struct bpf_spinlock_cnt {
	28	struct bpf_spin_lock lock;
	29	__u32 cnt;
	30	};
	31
	32	#define PARENT_CGROUP "/test-bpf-sock-fields"
	33	#define CHILD_CGROUP "/test-bpf-sock-fields/child"
	34	#define DATA "Hello BPF!"
	35	#define DATA_LEN sizeof(DATA)
	36
	37	static struct sockaddr_in6 srv_sa6, cli_sa6;
	38	static int sk_pkt_out_cnt10_fd;
	39	static struct test_sock_fields *skel;
	40	static int sk_pkt_out_cnt_fd;
	41	static __u64 parent_cg_id;
	42	static __u64 child_cg_id;
	43	static int linum_map_fd;
	44	static __u32 duration;
	45
	46	static __u32 egress_linum_idx = EGRESS_LINUM_IDX;
	47	static __u32 ingress_linum_idx = INGRESS_LINUM_IDX;
	48
	49	static void print_sk(const struct bpf_sock sk, const char prefix)
	50	{
	51	char src_ip4[24], dst_ip4[24];
	52	char src_ip6[64], dst_ip6[64];
	53
	54	inet_ntop(AF_INET, &sk->src_ip4, src_ip4, sizeof(src_ip4));
	55	inet_ntop(AF_INET6, &sk->src_ip6, src_ip6, sizeof(src_ip6));
	56	inet_ntop(AF_INET, &sk->dst_ip4, dst_ip4, sizeof(dst_ip4));
	57	inet_ntop(AF_INET6, &sk->dst_ip6, dst_ip6, sizeof(dst_ip6));
	58
	59	printf("%s: state:%u bound_dev_if:%u family:%u type:%u protocol:%u mark:%u priority:%u "
	60	"src_ip4:%x(%s) src_ip6:%x:%x:%x:%x(%s) src_port:%u "
	61	"dst_ip4:%x(%s) dst_ip6:%x:%x:%x:%x(%s) dst_port:%u\n",
	62	prefix,
	63	sk->state, sk->bound_dev_if, sk->family, sk->type, sk->protocol,
	64	sk->mark, sk->priority,
	65	sk->src_ip4, src_ip4,
	66	sk->src_ip6[0], sk->src_ip6[1], sk->src_ip6[2], sk->src_ip6[3],
	67	src_ip6, sk->src_port,
	68	sk->dst_ip4, dst_ip4,
	69	sk->dst_ip6[0], sk->dst_ip6[1], sk->dst_ip6[2], sk->dst_ip6[3],
	70	dst_ip6, ntohs(sk->dst_port));
	71	}
	72
	73	static void print_tp(const struct bpf_tcp_sock tp, const char prefix)
	74	{
	75	printf("%s: snd_cwnd:%u srtt_us:%u rtt_min:%u snd_ssthresh:%u rcv_nxt:%u "
	76	"snd_nxt:%u snd:una:%u mss_cache:%u ecn_flags:%u "
	77	"rate_delivered:%u rate_interval_us:%u packets_out:%u "
	78	"retrans_out:%u total_retrans:%u segs_in:%u data_segs_in:%u "
	79	"segs_out:%u data_segs_out:%u lost_out:%u sacked_out:%u "
	80	"bytes_received:%llu bytes_acked:%llu\n",
	81	prefix,
	82	tp->snd_cwnd, tp->srtt_us, tp->rtt_min, tp->snd_ssthresh,
	83	tp->rcv_nxt, tp->snd_nxt, tp->snd_una, tp->mss_cache,
	84	tp->ecn_flags, tp->rate_delivered, tp->rate_interval_us,
	85	tp->packets_out, tp->retrans_out, tp->total_retrans,
	86	tp->segs_in, tp->data_segs_in, tp->segs_out,
	87	tp->data_segs_out, tp->lost_out, tp->sacked_out,
	88	tp->bytes_received, tp->bytes_acked);
	89	}
	90
	91	static void check_result(void)
	92	{
	93	struct bpf_tcp_sock srv_tp, cli_tp, listen_tp;
	94	struct bpf_sock srv_sk, cli_sk, listen_sk;
	95	__u32 ingress_linum, egress_linum;
	96	int err;
	97
	98	err = bpf_map_lookup_elem(linum_map_fd, &egress_linum_idx,
	99	&egress_linum);
	100	CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)",
	101	"err:%d errno:%d\n", err, errno);
	102
	103	err = bpf_map_lookup_elem(linum_map_fd, &ingress_linum_idx,
	104	&ingress_linum);
	105	CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)",
	106	"err:%d errno:%d\n", err, errno);
	107
	108	memcpy(&srv_sk, &skel->bss->srv_sk, sizeof(srv_sk));
	109	memcpy(&srv_tp, &skel->bss->srv_tp, sizeof(srv_tp));
	110	memcpy(&cli_sk, &skel->bss->cli_sk, sizeof(cli_sk));
	111	memcpy(&cli_tp, &skel->bss->cli_tp, sizeof(cli_tp));
	112	memcpy(&listen_sk, &skel->bss->listen_sk, sizeof(listen_sk));
	113	memcpy(&listen_tp, &skel->bss->listen_tp, sizeof(listen_tp));
	114
	115	print_sk(&listen_sk, "listen_sk");
	116	print_sk(&srv_sk, "srv_sk");
	117	print_sk(&cli_sk, "cli_sk");
	118	print_tp(&listen_tp, "listen_tp");
	119	print_tp(&srv_tp, "srv_tp");
	120	print_tp(&cli_tp, "cli_tp");
	121
	122	CHECK(listen_sk.state != 10 \|\|
	123	listen_sk.family != AF_INET6 \|\|
	124	listen_sk.protocol != IPPROTO_TCP \|\|
	125	memcmp(listen_sk.src_ip6, &in6addr_loopback,
	126	sizeof(listen_sk.src_ip6)) \|\|
	127	listen_sk.dst_ip6[0] \|\| listen_sk.dst_ip6[1] \|\|
	128	listen_sk.dst_ip6[2] \|\| listen_sk.dst_ip6[3] \|\|
	129	listen_sk.src_port != ntohs(srv_sa6.sin6_port) \|\|
	130	listen_sk.dst_port,
	131	"listen_sk",
	132	"Unexpected. Check listen_sk output. ingress_linum:%u\n",
	133	ingress_linum);
	134
	135	CHECK(srv_sk.state == 10 \|\|
	136	!srv_sk.state \|\|
	137	srv_sk.family != AF_INET6 \|\|
	138	srv_sk.protocol != IPPROTO_TCP \|\|
	139	memcmp(srv_sk.src_ip6, &in6addr_loopback,
	140	sizeof(srv_sk.src_ip6)) \|\|
	141	memcmp(srv_sk.dst_ip6, &in6addr_loopback,
	142	sizeof(srv_sk.dst_ip6)) \|\|
	143	srv_sk.src_port != ntohs(srv_sa6.sin6_port) \|\|
	144	srv_sk.dst_port != cli_sa6.sin6_port,
	145	"srv_sk", "Unexpected. Check srv_sk output. egress_linum:%u\n",
	146	egress_linum);
	147
	148	CHECK(!skel->bss->lsndtime, "srv_tp", "Unexpected lsndtime:0\n");
	149
	150	CHECK(cli_sk.state == 10 \|\|
	151	!cli_sk.state \|\|
	152	cli_sk.family != AF_INET6 \|\|
	153	cli_sk.protocol != IPPROTO_TCP \|\|
	154	memcmp(cli_sk.src_ip6, &in6addr_loopback,
	155	sizeof(cli_sk.src_ip6)) \|\|
	156	memcmp(cli_sk.dst_ip6, &in6addr_loopback,
	157	sizeof(cli_sk.dst_ip6)) \|\|
	158	cli_sk.src_port != ntohs(cli_sa6.sin6_port) \|\|
	159	cli_sk.dst_port != srv_sa6.sin6_port,
	160	"cli_sk", "Unexpected. Check cli_sk output. egress_linum:%u\n",
	161	egress_linum);
	162
	163	CHECK(listen_tp.data_segs_out \|\|
	164	listen_tp.data_segs_in \|\|
	165	listen_tp.total_retrans \|\|
	166	listen_tp.bytes_acked,
	167	"listen_tp",
	168	"Unexpected. Check listen_tp output. ingress_linum:%u\n",
	169	ingress_linum);
	170
	171	CHECK(srv_tp.data_segs_out != 2 \|\|
	172	srv_tp.data_segs_in \|\|
	173	srv_tp.snd_cwnd != 10 \|\|
	174	srv_tp.total_retrans \|\|
	175	srv_tp.bytes_acked < 2 * DATA_LEN,
	176	"srv_tp", "Unexpected. Check srv_tp output. egress_linum:%u\n",
	177	egress_linum);
	178
	179	CHECK(cli_tp.data_segs_out \|\|
	180	cli_tp.data_segs_in != 2 \|\|
	181	cli_tp.snd_cwnd != 10 \|\|
	182	cli_tp.total_retrans \|\|
	183	cli_tp.bytes_received < 2 * DATA_LEN,
	184	"cli_tp", "Unexpected. Check cli_tp output. egress_linum:%u\n",
	185	egress_linum);
	186
	187	CHECK(skel->bss->parent_cg_id != parent_cg_id,
	188	"parent_cg_id", "%zu != %zu\n",
	189	(size_t)skel->bss->parent_cg_id, (size_t)parent_cg_id);
	190
	191	CHECK(skel->bss->child_cg_id != child_cg_id,
	192	"child_cg_id", "%zu != %zu\n",
	193	(size_t)skel->bss->child_cg_id, (size_t)child_cg_id);
	194	}
	195
	196	static void check_sk_pkt_out_cnt(int accept_fd, int cli_fd)
	197	{
	198	struct bpf_spinlock_cnt pkt_out_cnt = {}, pkt_out_cnt10 = {};
	199	int err;
	200
	201	pkt_out_cnt.cnt = ~0;
	202	pkt_out_cnt10.cnt = ~0;
	203	err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &accept_fd, &pkt_out_cnt);
	204	if (!err)
	205	err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &accept_fd,
	206	&pkt_out_cnt10);
	207
	208	/* The bpf prog only counts for fullsock and
	209	* passive connection did not become fullsock until 3WHS
	210	* had been finished, so the bpf prog only counted two data
	211	* packet out.
	212	*/
	213	CHECK(err \|\| pkt_out_cnt.cnt < 0xeB9F + 2 \|\|
	214	pkt_out_cnt10.cnt < 0xeB9F + 20,
	215	"bpf_map_lookup_elem(sk_pkt_out_cnt, &accept_fd)",
	216	"err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u\n",
	217	err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt);
	218
	219	pkt_out_cnt.cnt = ~0;
	220	pkt_out_cnt10.cnt = ~0;
	221	err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &cli_fd, &pkt_out_cnt);
	222	if (!err)
	223	err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &cli_fd,
	224	&pkt_out_cnt10);
	225	/* Active connection is fullsock from the beginning.
	226	* 1 SYN and 1 ACK during 3WHS
	227	* 2 Acks on data packet.
	228	*
	229	* The bpf_prog initialized it to 0xeB9F.
	230	*/
	231	CHECK(err \|\| pkt_out_cnt.cnt < 0xeB9F + 4 \|\|
	232	pkt_out_cnt10.cnt < 0xeB9F + 40,
	233	"bpf_map_lookup_elem(sk_pkt_out_cnt, &cli_fd)",
	234	"err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u\n",
	235	err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt);
	236	}
	237
	238	static int init_sk_storage(int sk_fd, __u32 pkt_out_cnt)
	239	{
	240	struct bpf_spinlock_cnt scnt = {};
	241	int err;
	242
	243	scnt.cnt = pkt_out_cnt;
	244	err = bpf_map_update_elem(sk_pkt_out_cnt_fd, &sk_fd, &scnt,
	245	BPF_NOEXIST);
	246	if (CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt_fd)",
	247	"err:%d errno:%d\n", err, errno))
	248	return err;
	249
	250	err = bpf_map_update_elem(sk_pkt_out_cnt10_fd, &sk_fd, &scnt,
	251	BPF_NOEXIST);
	252	if (CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt10_fd)",
	253	"err:%d errno:%d\n", err, errno))
	254	return err;
	255
	256	return 0;
	257	}
	258
	259	static void test(void)
	260	{
	261	int listen_fd = -1, cli_fd = -1, accept_fd = -1, err, i;
	262	socklen_t addrlen = sizeof(struct sockaddr_in6);
	263	char buf[DATA_LEN];
	264
	265	/* Prepare listen_fd */
	266	listen_fd = start_server(AF_INET6, SOCK_STREAM, "::1", 0, 0);
	267	/* start_server() has logged the error details */
	268	if (CHECK_FAIL(listen_fd == -1))
	269	goto done;
	270
	271	err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen);
	272	if (CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d\n", err,
	273	errno))
	274	goto done;
	275	memcpy(&skel->bss->srv_sa6, &srv_sa6, sizeof(srv_sa6));
	276
	277	cli_fd = connect_to_fd(listen_fd, 0);
	278	if (CHECK_FAIL(cli_fd == -1))
	279	goto done;
	280
	281	err = getsockname(cli_fd, (struct sockaddr *)&cli_sa6, &addrlen);
	282	if (CHECK(err, "getsockname(cli_fd)", "err:%d errno:%d\n",
	283	err, errno))
	284	goto done;
	285
	286	accept_fd = accept(listen_fd, NULL, NULL);
	287	if (CHECK(accept_fd == -1, "accept(listen_fd)",
	288	"accept_fd:%d errno:%d\n",
	289	accept_fd, errno))
	290	goto done;
	291
	292	if (init_sk_storage(accept_fd, 0xeB9F))
	293	goto done;
	294
	295	for (i = 0; i < 2; i++) {
	296	/* Send some data from accept_fd to cli_fd.
	297	* MSG_EOR to stop kernel from coalescing two pkts.
	298	*/
	299	err = send(accept_fd, DATA, DATA_LEN, MSG_EOR);
	300	if (CHECK(err != DATA_LEN, "send(accept_fd)",
	301	"err:%d errno:%d\n", err, errno))
	302	goto done;
	303
	304	err = recv(cli_fd, buf, DATA_LEN, 0);
	305	if (CHECK(err != DATA_LEN, "recv(cli_fd)", "err:%d errno:%d\n",
	306	err, errno))
	307	goto done;
	308	}
	309
	310	shutdown(cli_fd, SHUT_WR);
	311	err = recv(accept_fd, buf, 1, 0);
	312	if (CHECK(err, "recv(accept_fd) for fin", "err:%d errno:%d\n",
	313	err, errno))
	314	goto done;
	315	shutdown(accept_fd, SHUT_WR);
	316	err = recv(cli_fd, buf, 1, 0);
	317	if (CHECK(err, "recv(cli_fd) for fin", "err:%d errno:%d\n",
	318	err, errno))
	319	goto done;
	320	check_sk_pkt_out_cnt(accept_fd, cli_fd);
	321	check_result();
	322
	323	done:
	324	if (accept_fd != -1)
	325	close(accept_fd);
	326	if (cli_fd != -1)
	327	close(cli_fd);
	328	if (listen_fd != -1)
	329	close(listen_fd);
	330	}
	331
	332	void test_sock_fields(void)
	333	{
	334	struct bpf_link egress_link = NULL, ingress_link = NULL;
	335	int parent_cg_fd = -1, child_cg_fd = -1;
	336
	337	/* Create a cgroup, get fd, and join it */
	338	parent_cg_fd = test__join_cgroup(PARENT_CGROUP);
	339	if (CHECK_FAIL(parent_cg_fd < 0))
	340	return;
	341	parent_cg_id = get_cgroup_id(PARENT_CGROUP);
	342	if (CHECK_FAIL(!parent_cg_id))
	343	goto done;
	344
	345	child_cg_fd = test__join_cgroup(CHILD_CGROUP);
	346	if (CHECK_FAIL(child_cg_fd < 0))
	347	goto done;
	348	child_cg_id = get_cgroup_id(CHILD_CGROUP);
	349	if (CHECK_FAIL(!child_cg_id))
	350	goto done;
	351
	352	skel = test_sock_fields__open_and_load();
	353	if (CHECK(!skel, "test_sock_fields__open_and_load", "failed\n"))
	354	goto done;
	355
	356	egress_link = bpf_program__attach_cgroup(skel->progs.egress_read_sock_fields,
	357	child_cg_fd);
	358	if (CHECK(IS_ERR(egress_link), "attach_cgroup(egress)", "err:%ld\n",
	359	PTR_ERR(egress_link)))
	360	goto done;
	361
	362	ingress_link = bpf_program__attach_cgroup(skel->progs.ingress_read_sock_fields,
	363	child_cg_fd);
	364	if (CHECK(IS_ERR(ingress_link), "attach_cgroup(ingress)", "err:%ld\n",
	365	PTR_ERR(ingress_link)))
	366	goto done;
	367
	368	linum_map_fd = bpf_map__fd(skel->maps.linum_map);
	369	sk_pkt_out_cnt_fd = bpf_map__fd(skel->maps.sk_pkt_out_cnt);
	370	sk_pkt_out_cnt10_fd = bpf_map__fd(skel->maps.sk_pkt_out_cnt10);
	371
	372	test();
	373
	374	done:
	375	bpf_link__destroy(egress_link);
	376	bpf_link__destroy(ingress_link);
	377	test_sock_fields__destroy(skel);
	378	if (child_cg_fd != -1)
	379	close(child_cg_fd);
	380	if (parent_cg_fd != -1)
	381	close(parent_cg_fd);
	382	}

tools/testing/selftests/bpf/prog_tests/sockmap_basic.c+62−38

 #include "test_sockmap_invalid_update.skel.h"
 #include "bpf_iter_sockmap.skel.h"
-#include "progs/bpf_iter_sockmap.h"
 #define TCP_REPAIR		19	/* TCP sock is under repair right now */
 #define TCP_REPAIR_ON		1
 	return -1;
 }
+static void compare_cookies(struct bpf_map *src, struct bpf_map *dst)
+{
+	__u32 i, max_entries = bpf_map__max_entries(src);
+	int err, duration = 0, src_fd, dst_fd;
+	src_fd = bpf_map__fd(src);
+	dst_fd = bpf_map__fd(dst);
+	for (i = 0; i < max_entries; i++) {
+		__u64 src_cookie, dst_cookie;
+		err = bpf_map_lookup_elem(src_fd, &i, &src_cookie);
+		if (err && errno == ENOENT) {
+			err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
+			CHECK(!err, "map_lookup_elem(dst)", "element %u not deleted\n", i);
+			CHECK(err && errno != ENOENT, "map_lookup_elem(dst)", "%s\n",
+			      strerror(errno));
+			continue;
+		}
+		if (CHECK(err, "lookup_elem(src)", "%s\n", strerror(errno)))
+			continue;
+		err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
+		if (CHECK(err, "lookup_elem(dst)", "%s\n", strerror(errno)))
+			continue;
+		CHECK(dst_cookie != src_cookie, "cookie mismatch",
+		      "%llu != %llu (pos %u)\n", dst_cookie, src_cookie, i);
+	}
+}
 /* Create a map, populate it with one socket, and free the map. */
 static void test_sockmap_create_update_free(enum bpf_map_type map_type)
 {
 static void test_sockmap_update(enum bpf_map_type map_type)
 {
 	struct bpf_prog_test_run_attr tattr;
-	int err, prog, src, dst, duration = 0;
+	int err, prog, src, duration = 0;
 	struct test_sockmap_update *skel;
-	__u64 src_cookie, dst_cookie;
+	struct bpf_map *dst_map;
 	const __u32 zero = 0;
 	char dummy[14] = {0};
 	__s64 sk;
 	prog = bpf_program__fd(skel->progs.copy_sock_map);
 	src = bpf_map__fd(skel->maps.src);
 	if (map_type == BPF_MAP_TYPE_SOCKMAP)
-		dst = bpf_map__fd(skel->maps.dst_sock_map);
+		dst_map = skel->maps.dst_sock_map;
 	else
-		dst = bpf_map__fd(skel->maps.dst_sock_hash);
+		dst_map = skel->maps.dst_sock_hash;
 	err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST);
 	if (CHECK(err, "update_elem(src)", "errno=%u\n", errno))
 		goto out;
-	err = bpf_map_lookup_elem(src, &zero, &src_cookie);
-	if (CHECK(err, "lookup_elem(src, cookie)", "errno=%u\n", errno))
-		goto out;
 	tattr = (struct bpf_prog_test_run_attr){
 		.prog_fd = prog,
 		.repeat = 1,
 		       "errno=%u retval=%u\n", errno, tattr.retval))
 		goto out;
-	err = bpf_map_lookup_elem(dst, &zero, &dst_cookie);
+	compare_cookies(skel->maps.src, dst_map);
-	if (CHECK(err, "lookup_elem(dst, cookie)", "errno=%u\n", errno))
-		goto out;
-	CHECK(dst_cookie != src_cookie, "cookie mismatch", "%llu != %llu\n",
-	      dst_cookie, src_cookie);
 out:
 	test_sockmap_update__destroy(skel);
 		test_sockmap_invalid_update__destroy(skel);
 }
-static void test_sockmap_iter(enum bpf_map_type map_type)
+static void test_sockmap_copy(enum bpf_map_type map_type)
 {
 	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 	int err, len, src_fd, iter_fd, duration = 0;
 	union bpf_iter_link_info linfo = {0};
-	__s64 sock_fd[SOCKMAP_MAX_ENTRIES];
+	__u32 i, num_sockets, num_elems;
-	__u32 i, num_sockets, max_elems;
 	struct bpf_iter_sockmap *skel;
+	__s64 *sock_fd = NULL;
 	struct bpf_link *link;
 	struct bpf_map *src;
 	char buf[64];
 	if (CHECK(!skel, "bpf_iter_sockmap__open_and_load", "skeleton open_and_load failed\n"))
 		return;
-	for (i = 0; i < ARRAY_SIZE(sock_fd); i++)
-		sock_fd[i] = -1;
-	/* Make sure we have at least one "empty" entry to test iteration of
-	 * an empty slot.
-	 */
-	num_sockets = ARRAY_SIZE(sock_fd) - 1;
 	if (map_type == BPF_MAP_TYPE_SOCKMAP) {
 		src = skel->maps.sockmap;
-		max_elems = bpf_map__max_entries(src);
+		num_elems = bpf_map__max_entries(src);
+		num_sockets = num_elems - 1;
 	} else {
 		src = skel->maps.sockhash;
-		max_elems = num_sockets;
+		num_elems = bpf_map__max_entries(src) - 1;
+		num_sockets = num_elems;
 	}
+	sock_fd = calloc(num_sockets, sizeof(*sock_fd));
+	if (CHECK(!sock_fd, "calloc(sock_fd)", "failed to allocate\n"))
+		goto out;
+	for (i = 0; i < num_sockets; i++)
+		sock_fd[i] = -1;
 	src_fd = bpf_map__fd(src);
 	for (i = 0; i < num_sockets; i++) {
 	linfo.map.map_fd = src_fd;
 	opts.link_info = &linfo;
 	opts.link_info_len = sizeof(linfo);
-	link = bpf_program__attach_iter(skel->progs.count_elems, &opts);
+	link = bpf_program__attach_iter(skel->progs.copy, &opts);
 	if (CHECK(IS_ERR(link), "attach_iter", "attach_iter failed\n"))
 		goto out;
 		goto close_iter;
 	/* test results */
-	if (CHECK(skel->bss->elems != max_elems, "elems", "got %u expected %u\n",
+	if (CHECK(skel->bss->elems != num_elems, "elems", "got %u expected %u\n",
-		  skel->bss->elems, max_elems))
+		  skel->bss->elems, num_elems))
 		goto close_iter;
 	if (CHECK(skel->bss->socks != num_sockets, "socks", "got %u expected %u\n",
 		  skel->bss->socks, num_sockets))
 		goto close_iter;
+	compare_cookies(src, skel->maps.dst);
 close_iter:
 	close(iter_fd);
 free_link:
 	bpf_link__destroy(link);
 out:
-	for (i = 0; i < num_sockets; i++) {
+	for (i = 0; sock_fd && i < num_sockets; i++)
 		if (sock_fd[i] >= 0)
 			close(sock_fd[i]);
-	}
+	if (sock_fd)
+		free(sock_fd);
 	bpf_iter_sockmap__destroy(skel);
 }
 		test_sockmap_update(BPF_MAP_TYPE_SOCKHASH);
 	if (test__start_subtest("sockmap update in unsafe context"))
 		test_sockmap_invalid_update();
-	if (test__start_subtest("sockmap iter"))
+	if (test__start_subtest("sockmap copy"))
-		test_sockmap_iter(BPF_MAP_TYPE_SOCKMAP);
+		test_sockmap_copy(BPF_MAP_TYPE_SOCKMAP);
-	if (test__start_subtest("sockhash iter"))
+	if (test__start_subtest("sockhash copy"))
-		test_sockmap_iter(BPF_MAP_TYPE_SOCKHASH);
+		test_sockmap_copy(BPF_MAP_TYPE_SOCKHASH);
 }

tools/testing/selftests/bpf/prog_tests/test_overhead.c+1−13

 	const char *raw_tp_name = "raw_tp/task_rename";
 	const char *fentry_name = "fentry/__set_task_comm";
 	const char *fexit_name = "fexit/__set_task_comm";
-	const char *fmodret_name = "fmod_ret/__set_task_comm";
 	const char *kprobe_func = "__set_task_comm";
 	struct bpf_program *kprobe_prog, *kretprobe_prog, *raw_tp_prog;
-	struct bpf_program *fentry_prog, *fexit_prog, *fmodret_prog;
+	struct bpf_program *fentry_prog, *fexit_prog;
 	struct bpf_object *obj;
 	struct bpf_link *link;
 	int err, duration = 0;
 	if (CHECK(!fexit_prog, "find_probe",
 		  "prog '%s' not found\n", fexit_name))
 		goto cleanup;
-	fmodret_prog = bpf_object__find_program_by_title(obj, fmodret_name);
-	if (CHECK(!fmodret_prog, "find_probe",
-		  "prog '%s' not found\n", fmodret_name))
-		goto cleanup;
 	err = bpf_object__load(obj);
 	if (CHECK(err, "obj_load", "err %d\n", err))
 		goto cleanup;
 	test_run("fexit");
 	bpf_link__destroy(link);
-	/* attach fmod_ret */
-	link = bpf_program__attach_trace(fmodret_prog);
-	if (CHECK(IS_ERR(link), "attach fmod_ret", "err %ld\n", PTR_ERR(link)))
-		goto cleanup;
-	test_run("fmod_ret");
-	bpf_link__destroy(link);
 cleanup:
 	prctl(PR_SET_NAME, comm, 0L, 0L, 0L);
 	bpf_object__close(obj);

tools/testing/selftests/bpf/prog_tests/trace_ext.c+111−0

@@ -0,0 +1,111 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#define _GNU_SOURCE
	4	#include <test_progs.h>
	5	#include <network_helpers.h>
	6	#include <sys/stat.h>
	7	#include <linux/sched.h>
	8	#include <sys/syscall.h>
	9
	10	#include "test_pkt_md_access.skel.h"
	11	#include "test_trace_ext.skel.h"
	12	#include "test_trace_ext_tracing.skel.h"
	13
	14	static __u32 duration;
	15
	16	void test_trace_ext(void)
	17	{
	18	struct test_pkt_md_access *skel_pkt = NULL;
	19	struct test_trace_ext_tracing *skel_trace = NULL;
	20	struct test_trace_ext_tracing__bss *bss_trace;
	21	struct test_trace_ext *skel_ext = NULL;
	22	struct test_trace_ext__bss *bss_ext;
	23	int err, pkt_fd, ext_fd;
	24	struct bpf_program *prog;
	25	char buf[100];
	26	__u32 retval;
	27	__u64 len;
	28
	29	/* open/load/attach test_pkt_md_access */
	30	skel_pkt = test_pkt_md_access__open_and_load();
	31	if (CHECK(!skel_pkt, "setup", "classifier/test_pkt_md_access open failed\n"))
	32	goto cleanup;
	33
	34	err = test_pkt_md_access__attach(skel_pkt);
	35	if (CHECK(err, "setup", "classifier/test_pkt_md_access attach failed: %d\n", err))
	36	goto cleanup;
	37
	38	prog = skel_pkt->progs.test_pkt_md_access;
	39	pkt_fd = bpf_program__fd(prog);
	40
	41	/* open extension */
	42	skel_ext = test_trace_ext__open();
	43	if (CHECK(!skel_ext, "setup", "freplace/test_pkt_md_access open failed\n"))
	44	goto cleanup;
	45
	46	/* set extension's attach target - test_pkt_md_access */
	47	prog = skel_ext->progs.test_pkt_md_access_new;
	48	bpf_program__set_attach_target(prog, pkt_fd, "test_pkt_md_access");
	49
	50	/* load/attach extension */
	51	err = test_trace_ext__load(skel_ext);
	52	if (CHECK(err, "setup", "freplace/test_pkt_md_access load failed\n")) {
	53	libbpf_strerror(err, buf, sizeof(buf));
	54	fprintf(stderr, "%s\n", buf);
	55	goto cleanup;
	56	}
	57
	58	err = test_trace_ext__attach(skel_ext);
	59	if (CHECK(err, "setup", "freplace/test_pkt_md_access attach failed: %d\n", err))
	60	goto cleanup;
	61
	62	prog = skel_ext->progs.test_pkt_md_access_new;
	63	ext_fd = bpf_program__fd(prog);
	64
	65	/* open tracing */
	66	skel_trace = test_trace_ext_tracing__open();
	67	if (CHECK(!skel_trace, "setup", "tracing/test_pkt_md_access_new open failed\n"))
	68	goto cleanup;
	69
	70	/* set tracing's attach target - fentry */
	71	prog = skel_trace->progs.fentry;
	72	bpf_program__set_attach_target(prog, ext_fd, "test_pkt_md_access_new");
	73
	74	/* set tracing's attach target - fexit */
	75	prog = skel_trace->progs.fexit;
	76	bpf_program__set_attach_target(prog, ext_fd, "test_pkt_md_access_new");
	77
	78	/* load/attach tracing */
	79	err = test_trace_ext_tracing__load(skel_trace);
	80	if (CHECK(err, "setup", "tracing/test_pkt_md_access_new load failed\n")) {
	81	libbpf_strerror(err, buf, sizeof(buf));
	82	fprintf(stderr, "%s\n", buf);
	83	goto cleanup;
	84	}
	85
	86	err = test_trace_ext_tracing__attach(skel_trace);
	87	if (CHECK(err, "setup", "tracing/test_pkt_md_access_new attach failed: %d\n", err))
	88	goto cleanup;
	89
	90	/* trigger the test */
	91	err = bpf_prog_test_run(pkt_fd, 1, &pkt_v4, sizeof(pkt_v4),
	92	NULL, NULL, &retval, &duration);
	93	CHECK(err \|\| retval, "run", "err %d errno %d retval %d\n", err, errno, retval);
	94
	95	bss_ext = skel_ext->bss;
	96	bss_trace = skel_trace->bss;
	97
	98	len = bss_ext->ext_called;
	99
	100	CHECK(bss_ext->ext_called == 0,
	101	"check", "failed to trigger freplace/test_pkt_md_access\n");
	102	CHECK(bss_trace->fentry_called != len,
	103	"check", "failed to trigger fentry/test_pkt_md_access_new\n");
	104	CHECK(bss_trace->fexit_called != len,
	105	"check", "failed to trigger fexit/test_pkt_md_access_new\n");
	106
	107	cleanup:
	108	test_trace_ext_tracing__destroy(skel_trace);
	109	test_trace_ext__destroy(skel_ext);
	110	test_pkt_md_access__destroy(skel_pkt);
	111	}

tools/testing/selftests/bpf/progs/bpf_cubic.c+2−0

@@ -15,6 +15,8 @@ · 14 unmodified lines
15	*/	15	*/
16		16
17	#include <linux/bpf.h>	17	#include <linux/bpf.h>
		18	#include <linux/stddef.h>
		19	#include <linux/tcp.h>
18	#include "bpf_tcp_helpers.h"	20	#include "bpf_tcp_helpers.h"
19		21
20	char _license[] SEC("license") = "GPL";	22	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/bpf_dctcp.c+2−0

 #include <stddef.h>
 #include <linux/bpf.h>
 #include <linux/types.h>
+#include <linux/stddef.h>
+#include <linux/tcp.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 #include "bpf_tcp_helpers.h"

tools/testing/selftests/bpf/progs/bpf_flow.c+6−6

@@ -118,18 +118,18 @@ static __always_inline int parse_eth_proto(struct __sk_buff *skb, __be16 proto) · 117 unmodified lines
118		118
119	switch (proto) {	119	switch (proto) {
120	case bpf_htons(ETH_P_IP):	120	case bpf_htons(ETH_P_IP):
121	bpf_tail_call(skb, &jmp_table, IP);	121	bpf_tail_call_static(skb, &jmp_table, IP);
122	break;	122	break;
123	case bpf_htons(ETH_P_IPV6):	123	case bpf_htons(ETH_P_IPV6):
124	bpf_tail_call(skb, &jmp_table, IPV6);	124	bpf_tail_call_static(skb, &jmp_table, IPV6);
125	break;	125	break;
126	case bpf_htons(ETH_P_MPLS_MC):	126	case bpf_htons(ETH_P_MPLS_MC):
127	case bpf_htons(ETH_P_MPLS_UC):	127	case bpf_htons(ETH_P_MPLS_UC):
128	bpf_tail_call(skb, &jmp_table, MPLS);	128	bpf_tail_call_static(skb, &jmp_table, MPLS);
129	break;	129	break;
130	case bpf_htons(ETH_P_8021Q):	130	case bpf_htons(ETH_P_8021Q):
131	case bpf_htons(ETH_P_8021AD):	131	case bpf_htons(ETH_P_8021AD):
132	bpf_tail_call(skb, &jmp_table, VLAN);	132	bpf_tail_call_static(skb, &jmp_table, VLAN);
133	break;	133	break;
134	default:	134	default:
135	/* Protocol not supported */	135	/* Protocol not supported */
@@ -246,10 +246,10 @@ static __always_inline int parse_ipv6_proto(struct __sk_buff *skb, __u8 nexthdr) · 110 unmodified lines
246	switch (nexthdr) {	246	switch (nexthdr) {
247	case IPPROTO_HOPOPTS:	247	case IPPROTO_HOPOPTS:
248	case IPPROTO_DSTOPTS:	248	case IPPROTO_DSTOPTS:
249	bpf_tail_call(skb, &jmp_table, IPV6OP);	249	bpf_tail_call_static(skb, &jmp_table, IPV6OP);
250	break;	250	break;
251	case IPPROTO_FRAGMENT:	251	case IPPROTO_FRAGMENT:
252	bpf_tail_call(skb, &jmp_table, IPV6FR);	252	bpf_tail_call_static(skb, &jmp_table, IPV6FR);
253	break;	253	break;
254	default:	254	default:
255	return parse_ip_proto(skb, nexthdr);	255	return parse_ip_proto(skb, nexthdr);

tools/testing/selftests/bpf/progs/bpf_iter.h+23−0

@@ -14,6 +14,11 @@ · 13 unmodified lines
14	#define bpf_iter__bpf_map_elem bpf_iter__bpf_map_elem___not_used	14	#define bpf_iter__bpf_map_elem bpf_iter__bpf_map_elem___not_used
15	#define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used	15	#define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used
16	#define bpf_iter__sockmap bpf_iter__sockmap___not_used	16	#define bpf_iter__sockmap bpf_iter__sockmap___not_used
		17	#define btf_ptr btf_ptr___not_used
		18	#define BTF_F_COMPACT BTF_F_COMPACT___not_used
		19	#define BTF_F_NONAME BTF_F_NONAME___not_used
		20	#define BTF_F_PTR_RAW BTF_F_PTR_RAW___not_used
		21	#define BTF_F_ZERO BTF_F_ZERO___not_used
17	#include "vmlinux.h"	22	#include "vmlinux.h"
18	#undef bpf_iter_meta	23	#undef bpf_iter_meta
19	#undef bpf_iter__bpf_map	24	#undef bpf_iter__bpf_map
@@ -28,6 +33,11 @@ · 8 unmodified lines
28	#undef bpf_iter__bpf_map_elem	33	#undef bpf_iter__bpf_map_elem
29	#undef bpf_iter__bpf_sk_storage_map	34	#undef bpf_iter__bpf_sk_storage_map
30	#undef bpf_iter__sockmap	35	#undef bpf_iter__sockmap
		36	#undef btf_ptr
		37	#undef BTF_F_COMPACT
		38	#undef BTF_F_NONAME
		39	#undef BTF_F_PTR_RAW
		40	#undef BTF_F_ZERO
31		41
32	struct bpf_iter_meta {	42	struct bpf_iter_meta {
33	struct seq_file *seq;	43	struct seq_file *seq;
@@ -105,3 +115,16 @@ struct bpf_iter__sockmap { · 71 unmodified lines
105	void *key;	115	void *key;
106	struct sock *sk;	116	struct sock *sk;
107	};	117	};
		118
		119	struct btf_ptr {
		120	void *ptr;
		121	__u32 type_id;
		122	__u32 flags;
		123	};
		124
		125	enum {
		126	BTF_F_COMPACT = (1ULL << 0),
		127	BTF_F_NONAME = (1ULL << 1),
		128	BTF_F_PTR_RAW = (1ULL << 2),
		129	BTF_F_ZERO = (1ULL << 3),
		130	};

tools/testing/selftests/bpf/progs/bpf_iter_sockmap.c+24−8

 /* Copyright (c) 2020 Cloudflare */
 #include "bpf_iter.h"
 #include "bpf_tracing_net.h"
-#include "bpf_iter_sockmap.h"
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 #include <errno.h>
 struct {
 	__uint(type, BPF_MAP_TYPE_SOCKMAP);
-	__uint(max_entries, SOCKMAP_MAX_ENTRIES);
+	__uint(max_entries, 64);
 	__type(key, __u32);
 	__type(value, __u64);
 } sockmap SEC(".maps");
 struct {
 	__uint(type, BPF_MAP_TYPE_SOCKHASH);
-	__uint(max_entries, SOCKMAP_MAX_ENTRIES);
+	__uint(max_entries, 64);
 	__type(key, __u32);
 	__type(value, __u64);
 } sockhash SEC(".maps");
+struct {
+	__uint(type, BPF_MAP_TYPE_SOCKHASH);
+	__uint(max_entries, 64);
+	__type(key, __u32);
+	__type(value, __u64);
+} dst SEC(".maps");
 __u32 elems = 0;
 __u32 socks = 0;
 SEC("iter/sockmap")
-int count_elems(struct bpf_iter__sockmap *ctx)
+int copy(struct bpf_iter__sockmap *ctx)
 {
 	struct sock *sk = ctx->sk;
 	__u32 tmp, *key = ctx->key;
 	int ret;
-	if (key)
+	if (!key)
-		elems++;
+		return 0;
+	elems++;
+	/* We need a temporary buffer on the stack, since the verifier doesn't
+	 * let us use the pointer from the context as an argument to the helper.
+	 */
+	tmp = *key;
-	if (sk)
+	if (sk) {
 		socks++;
+		return bpf_map_update_elem(&dst, &tmp, sk, 0) != 0;
+	}
-	return 0;
+	ret = bpf_map_delete_elem(&dst, &tmp);
+	return ret && ret != -ENOENT;
 }

tools/testing/selftests/bpf/progs/bpf_iter_sockmap.h+0−3

@@ -1,3 +0,0 @@
1	/* SPDX-License-Identifier: GPL-2.0 */
2
3	#define SOCKMAP_MAX_ENTRIES (64)

tools/testing/selftests/bpf/progs/bpf_iter_task_btf.c+50−0

@@ -0,0 +1,50 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020, Oracle and/or its affiliates. */
	3	#include "bpf_iter.h"
	4	#include <bpf/bpf_helpers.h>
	5	#include <bpf/bpf_tracing.h>
	6	#include <bpf/bpf_core_read.h>
	7
	8	#include <errno.h>
	9
	10	char _license[] SEC("license") = "GPL";
	11
	12	long tasks = 0;
	13	long seq_err = 0;
	14	bool skip = false;
	15
	16	SEC("iter/task")
	17	int dump_task_struct(struct bpf_iter__task *ctx)
	18	{
	19	struct seq_file *seq = ctx->meta->seq;
	20	struct task_struct *task = ctx->task;
	21	static struct btf_ptr ptr = { };
	22	long ret;
	23
	24	#if __has_builtin(__builtin_btf_type_id)
	25	ptr.type_id = bpf_core_type_id_kernel(struct task_struct);
	26	ptr.ptr = task;
	27
	28	if (ctx->meta->seq_num == 0)
	29	BPF_SEQ_PRINTF(seq, "Raw BTF task\n");
	30
	31	ret = bpf_seq_printf_btf(seq, &ptr, sizeof(ptr), 0);
	32	switch (ret) {
	33	case 0:
	34	tasks++;
	35	break;
	36	case -ERANGE:
	37	/* NULL task or task->fs, don't count it as an error. */
	38	break;
	39	case -E2BIG:
	40	return 1;
	41	default:
	42	seq_err = ret;
	43	break;
	44	}
	45	#else
	46	skip = true;
	47	#endif
	48
	49	return 0;
	50	}

tools/testing/selftests/bpf/progs/btf_ptr.h+27−0

@@ -0,0 +1,27 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/* Copyright (c) 2020, Oracle and/or its affiliates. */
	3	/* "undefine" structs in vmlinux.h, because we "override" them below */
	4	#define btf_ptr btf_ptr___not_used
	5	#define BTF_F_COMPACT BTF_F_COMPACT___not_used
	6	#define BTF_F_NONAME BTF_F_NONAME___not_used
	7	#define BTF_F_PTR_RAW BTF_F_PTR_RAW___not_used
	8	#define BTF_F_ZERO BTF_F_ZERO___not_used
	9	#include "vmlinux.h"
	10	#undef btf_ptr
	11	#undef BTF_F_COMPACT
	12	#undef BTF_F_NONAME
	13	#undef BTF_F_PTR_RAW
	14	#undef BTF_F_ZERO
	15
	16	struct btf_ptr {
	17	void *ptr;
	18	__u32 type_id;
	19	__u32 flags;
	20	};
	21
	22	enum {
	23	BTF_F_COMPACT = (1ULL << 0),
	24	BTF_F_NONAME = (1ULL << 1),
	25	BTF_F_PTR_RAW = (1ULL << 2),
	26	BTF_F_ZERO = (1ULL << 3),
	27	};

tools/testing/selftests/bpf/progs/fmod_ret_freplace.c+14−0

@@ -0,0 +1,14 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/bpf.h>
	3	#include <bpf/bpf_helpers.h>
	4	#include <bpf/bpf_tracing.h>
	5
	6	volatile __u64 test_fmod_ret = 0;
	7	SEC("fmod_ret/security_new_get_constant")
	8	int BPF_PROG(fmod_ret_test, long val, int ret)
	9	{
	10	test_fmod_ret = 1;
	11	return 120;
	12	}
	13
	14	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/freplace_get_constant.c+15−0

@@ -0,0 +1,15 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <linux/bpf.h>
	3	#include <bpf/bpf_helpers.h>
	4	#include <bpf/bpf_endian.h>
	5
	6	volatile __u64 test_get_constant = 0;
	7	SEC("freplace/get_constant")
	8	int security_new_get_constant(long val)
	9	{
	10	if (val != 123)
	11	return 0;
	12	test_get_constant = 1;
	13	return test_get_constant; /* original get_constant() returns val - 122 */
	14	}
	15	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/netif_receive_skb.c+249−0

@@ -0,0 +1,249 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020, Oracle and/or its affiliates. */
	3
	4	#include "btf_ptr.h"
	5	#include <bpf/bpf_helpers.h>
	6	#include <bpf/bpf_tracing.h>
	7	#include <bpf/bpf_core_read.h>
	8
	9	#include <errno.h>
	10
	11	long ret = 0;
	12	int num_subtests = 0;
	13	int ran_subtests = 0;
	14	bool skip = false;
	15
	16	#define STRSIZE 2048
	17	#define EXPECTED_STRSIZE 256
	18
	19	#ifndef ARRAY_SIZE
	20	#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
	21	#endif
	22
	23	struct {
	24	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
	25	__uint(max_entries, 1);
	26	__type(key, __u32);
	27	__type(value, char[STRSIZE]);
	28	} strdata SEC(".maps");
	29
	30	static int __strncmp(const void m1, const void m2, size_t len)
	31	{
	32	const unsigned char *s1 = m1;
	33	const unsigned char *s2 = m2;
	34	int i, delta = 0;
	35
	36	for (i = 0; i < len; i++) {
	37	delta = s1[i] - s2[i];
	38	if (delta \|\| s1[i] == 0 \|\| s2[i] == 0)
	39	break;
	40	}
	41	return delta;
	42	}
	43
	44	#if __has_builtin(__builtin_btf_type_id)
	45	#define TEST_BTF(_str, _type, _flags, _expected, ...) \
	46	do { \
	47	static const char _expectedval[EXPECTED_STRSIZE] = \
	48	_expected; \
	49	static const char _ptrtype[64] = #_type; \
	50	__u64 _hflags = _flags \| BTF_F_COMPACT; \
	51	static _type _ptrdata = __VA_ARGS__; \
	52	static struct btf_ptr _ptr = { }; \
	53	int _cmp; \
	54	\
	55	++num_subtests; \
	56	if (ret < 0) \
	57	break; \
	58	++ran_subtests; \
	59	_ptr.ptr = &_ptrdata; \
	60	_ptr.type_id = bpf_core_type_id_kernel(_type); \
	61	if (_ptr.type_id <= 0) { \
	62	ret = -EINVAL; \
	63	break; \
	64	} \
	65	ret = bpf_snprintf_btf(_str, STRSIZE, \
	66	&_ptr, sizeof(_ptr), _hflags); \
	67	if (ret) \
	68	break; \
	69	_cmp = __strncmp(_str, _expectedval, EXPECTED_STRSIZE); \
	70	if (_cmp != 0) { \
	71	bpf_printk("(%d) got %s", _cmp, _str); \
	72	bpf_printk("(%d) expected %s", _cmp, \
	73	_expectedval); \
	74	ret = -EBADMSG; \
	75	break; \
	76	} \
	77	} while (0)
	78	#endif
	79
	80	/* Use where expected data string matches its stringified declaration */
	81	#define TEST_BTF_C(_str, _type, _flags, ...) \
	82	TEST_BTF(_str, _type, _flags, "(" #_type ")" #__VA_ARGS__, \
	83	__VA_ARGS__)
	84
	85	/* TRACE_EVENT(netif_receive_skb,
	86	* TP_PROTO(struct sk_buff *skb),
	87	*/
	88	SEC("tp_btf/netif_receive_skb")
	89	int BPF_PROG(trace_netif_receive_skb, struct sk_buff *skb)
	90	{
	91	static __u64 flags[] = { 0, BTF_F_COMPACT, BTF_F_ZERO, BTF_F_PTR_RAW,
	92	BTF_F_NONAME, BTF_F_COMPACT \| BTF_F_ZERO \|
	93	BTF_F_PTR_RAW \| BTF_F_NONAME };
	94	static struct btf_ptr p = { };
	95	__u32 key = 0;
	96	int i, __ret;
	97	char *str;
	98
	99	#if __has_builtin(__builtin_btf_type_id)
	100	str = bpf_map_lookup_elem(&strdata, &key);
	101	if (!str)
	102	return 0;
	103
	104	/* Ensure we can write skb string representation */
	105	p.type_id = bpf_core_type_id_kernel(struct sk_buff);
	106	p.ptr = skb;
	107	for (i = 0; i < ARRAY_SIZE(flags); i++) {
	108	++num_subtests;
	109	ret = bpf_snprintf_btf(str, STRSIZE, &p, sizeof(p), 0);
	110	if (ret < 0)
	111	bpf_printk("returned %d when writing skb", ret);
	112	++ran_subtests;
	113	}
	114
	115	/* Check invalid ptr value */
	116	p.ptr = 0;
	117	__ret = bpf_snprintf_btf(str, STRSIZE, &p, sizeof(p), 0);
	118	if (__ret >= 0) {
	119	bpf_printk("printing NULL should generate error, got (%d)",
	120	__ret);
	121	ret = -ERANGE;
	122	}
	123
	124	/* Verify type display for various types. */
	125
	126	/* simple int */
	127	TEST_BTF_C(str, int, 0, 1234);
	128	TEST_BTF(str, int, BTF_F_NONAME, "1234", 1234);
	129	/* zero value should be printed at toplevel */
	130	TEST_BTF(str, int, 0, "(int)0", 0);
	131	TEST_BTF(str, int, BTF_F_NONAME, "0", 0);
	132	TEST_BTF(str, int, BTF_F_ZERO, "(int)0", 0);
	133	TEST_BTF(str, int, BTF_F_NONAME \| BTF_F_ZERO, "0", 0);
	134	TEST_BTF_C(str, int, 0, -4567);
	135	TEST_BTF(str, int, BTF_F_NONAME, "-4567", -4567);
	136
	137	/* simple char */
	138	TEST_BTF_C(str, char, 0, 100);
	139	TEST_BTF(str, char, BTF_F_NONAME, "100", 100);
	140	/* zero value should be printed at toplevel */
	141	TEST_BTF(str, char, 0, "(char)0", 0);
	142	TEST_BTF(str, char, BTF_F_NONAME, "0", 0);
	143	TEST_BTF(str, char, BTF_F_ZERO, "(char)0", 0);
	144	TEST_BTF(str, char, BTF_F_NONAME \| BTF_F_ZERO, "0", 0);
	145
	146	/* simple typedef */
	147	TEST_BTF_C(str, uint64_t, 0, 100);
	148	TEST_BTF(str, u64, BTF_F_NONAME, "1", 1);
	149	/* zero value should be printed at toplevel */
	150	TEST_BTF(str, u64, 0, "(u64)0", 0);
	151	TEST_BTF(str, u64, BTF_F_NONAME, "0", 0);
	152	TEST_BTF(str, u64, BTF_F_ZERO, "(u64)0", 0);
	153	TEST_BTF(str, u64, BTF_F_NONAME\|BTF_F_ZERO, "0", 0);
	154
	155	/* typedef struct */
	156	TEST_BTF_C(str, atomic_t, 0, {.counter = (int)1,});
	157	TEST_BTF(str, atomic_t, BTF_F_NONAME, "{1,}", {.counter = 1,});
	158	/* typedef with 0 value should be printed at toplevel */
	159	TEST_BTF(str, atomic_t, 0, "(atomic_t){}", {.counter = 0,});
	160	TEST_BTF(str, atomic_t, BTF_F_NONAME, "{}", {.counter = 0,});
	161	TEST_BTF(str, atomic_t, BTF_F_ZERO, "(atomic_t){.counter = (int)0,}",
	162	{.counter = 0,});
	163	TEST_BTF(str, atomic_t, BTF_F_NONAME\|BTF_F_ZERO,
	164	"{0,}", {.counter = 0,});
	165
	166	/* enum where enum value does (and does not) exist */
	167	TEST_BTF_C(str, enum bpf_cmd, 0, BPF_MAP_CREATE);
	168	TEST_BTF(str, enum bpf_cmd, 0, "(enum bpf_cmd)BPF_MAP_CREATE", 0);
	169	TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME, "BPF_MAP_CREATE",
	170	BPF_MAP_CREATE);
	171	TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME\|BTF_F_ZERO,
	172	"BPF_MAP_CREATE", 0);
	173
	174	TEST_BTF(str, enum bpf_cmd, BTF_F_ZERO, "(enum bpf_cmd)BPF_MAP_CREATE",
	175	BPF_MAP_CREATE);
	176	TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME\|BTF_F_ZERO,
	177	"BPF_MAP_CREATE", BPF_MAP_CREATE);
	178	TEST_BTF_C(str, enum bpf_cmd, 0, 2000);
	179	TEST_BTF(str, enum bpf_cmd, BTF_F_NONAME, "2000", 2000);
	180
	181	/* simple struct */
	182	TEST_BTF_C(str, struct btf_enum, 0,
	183	{.name_off = (__u32)3,.val = (__s32)-1,});
	184	TEST_BTF(str, struct btf_enum, BTF_F_NONAME, "{3,-1,}",
	185	{ .name_off = 3, .val = -1,});
	186	TEST_BTF(str, struct btf_enum, BTF_F_NONAME, "{-1,}",
	187	{ .name_off = 0, .val = -1,});
	188	TEST_BTF(str, struct btf_enum, BTF_F_NONAME\|BTF_F_ZERO, "{0,-1,}",
	189	{ .name_off = 0, .val = -1,});
	190	/* empty struct should be printed */
	191	TEST_BTF(str, struct btf_enum, 0, "(struct btf_enum){}",
	192	{ .name_off = 0, .val = 0,});
	193	TEST_BTF(str, struct btf_enum, BTF_F_NONAME, "{}",
	194	{ .name_off = 0, .val = 0,});
	195	TEST_BTF(str, struct btf_enum, BTF_F_ZERO,
	196	"(struct btf_enum){.name_off = (__u32)0,.val = (__s32)0,}",
	197	{ .name_off = 0, .val = 0,});
	198
	199	/* struct with pointers */
	200	TEST_BTF(str, struct list_head, BTF_F_PTR_RAW,
	201	"(struct list_head){.next = (struct list_head *)0x0000000000000001,}",
	202	{ .next = (struct list_head *)1 });
	203	/* NULL pointer should not be displayed */
	204	TEST_BTF(str, struct list_head, BTF_F_PTR_RAW,
	205	"(struct list_head){}",
	206	{ .next = (struct list_head *)0 });
	207
	208	/* struct with char array */
	209	TEST_BTF(str, struct bpf_prog_info, 0,
	210	"(struct bpf_prog_info){.name = (char[])['f','o','o',],}",
	211	{ .name = "foo",});
	212	TEST_BTF(str, struct bpf_prog_info, BTF_F_NONAME,
	213	"{['f','o','o',],}",
	214	{.name = "foo",});
	215	/* leading null char means do not display string */
	216	TEST_BTF(str, struct bpf_prog_info, 0,
	217	"(struct bpf_prog_info){}",
	218	{.name = {'\0', 'f', 'o', 'o'}});
	219	/* handle non-printable characters */
	220	TEST_BTF(str, struct bpf_prog_info, 0,
	221	"(struct bpf_prog_info){.name = (char[])[1,2,3,],}",
	222	{ .name = {1, 2, 3, 0}});
	223
	224	/* struct with non-char array */
	225	TEST_BTF(str, struct __sk_buff, 0,
	226	"(struct __sk_buff){.cb = (__u32[])[1,2,3,4,5,],}",
	227	{ .cb = {1, 2, 3, 4, 5,},});
	228	TEST_BTF(str, struct __sk_buff, BTF_F_NONAME,
	229	"{[1,2,3,4,5,],}",
	230	{ .cb = { 1, 2, 3, 4, 5},});
	231	/* For non-char, arrays, show non-zero values only */
	232	TEST_BTF(str, struct __sk_buff, 0,
	233	"(struct __sk_buff){.cb = (__u32[])[1,],}",
	234	{ .cb = { 0, 0, 1, 0, 0},});
	235
	236	/* struct with bitfields */
	237	TEST_BTF_C(str, struct bpf_insn, 0,
	238	{.code = (__u8)1,.dst_reg = (__u8)0x2,.src_reg = (__u8)0x3,.off = (__s16)4,.imm = (__s32)5,});
	239	TEST_BTF(str, struct bpf_insn, BTF_F_NONAME, "{1,0x2,0x3,4,5,}",
	240	{.code = 1, .dst_reg = 0x2, .src_reg = 0x3, .off = 4,
	241	.imm = 5,});
	242	#else
	243	skip = true;
	244	#endif
	245
	246	return 0;
	247	}
	248
	249	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/tailcall1.c+14−14

@@ -26,20 +26,20 @@ int entry(struct __sk_buff *skb) · 25 unmodified lines
26	/* Multiple locations to make sure we patch	26	/* Multiple locations to make sure we patch
27	* all of them.	27	* all of them.
28	*/	28	*/
29	bpf_tail_call(skb, &jmp_table, 0);	29	bpf_tail_call_static(skb, &jmp_table, 0);
30	bpf_tail_call(skb, &jmp_table, 0);	30	bpf_tail_call_static(skb, &jmp_table, 0);
31	bpf_tail_call(skb, &jmp_table, 0);	31	bpf_tail_call_static(skb, &jmp_table, 0);
32	bpf_tail_call(skb, &jmp_table, 0);	32	bpf_tail_call_static(skb, &jmp_table, 0);
33		33
34	bpf_tail_call(skb, &jmp_table, 1);	34	bpf_tail_call_static(skb, &jmp_table, 1);
35	bpf_tail_call(skb, &jmp_table, 1);	35	bpf_tail_call_static(skb, &jmp_table, 1);
36	bpf_tail_call(skb, &jmp_table, 1);	36	bpf_tail_call_static(skb, &jmp_table, 1);
37	bpf_tail_call(skb, &jmp_table, 1);	37	bpf_tail_call_static(skb, &jmp_table, 1);
38		38
39	bpf_tail_call(skb, &jmp_table, 2);	39	bpf_tail_call_static(skb, &jmp_table, 2);
40	bpf_tail_call(skb, &jmp_table, 2);	40	bpf_tail_call_static(skb, &jmp_table, 2);
41	bpf_tail_call(skb, &jmp_table, 2);	41	bpf_tail_call_static(skb, &jmp_table, 2);
42	bpf_tail_call(skb, &jmp_table, 2);	42	bpf_tail_call_static(skb, &jmp_table, 2);
43		43
44	return 3;	44	return 3;
45	}	45	}

tools/testing/selftests/bpf/progs/tailcall2.c+7−7

@@ -13,14 +13,14 @@ struct { · 12 unmodified lines
13	SEC("classifier/0")	13	SEC("classifier/0")
14	int bpf_func_0(struct __sk_buff *skb)	14	int bpf_func_0(struct __sk_buff *skb)
15	{	15	{
16	bpf_tail_call(skb, &jmp_table, 1);	16	bpf_tail_call_static(skb, &jmp_table, 1);
17	return 0;	17	return 0;
18	}	18	}
19		19
20	SEC("classifier/1")	20	SEC("classifier/1")
21	int bpf_func_1(struct __sk_buff *skb)	21	int bpf_func_1(struct __sk_buff *skb)
22	{	22	{
23	bpf_tail_call(skb, &jmp_table, 2);	23	bpf_tail_call_static(skb, &jmp_table, 2);
24	return 1;	24	return 1;
25	}	25	}
26		26
@@ -33,25 +33,25 @@ int bpf_func_2(struct __sk_buff *skb) · 6 unmodified lines
33	SEC("classifier/3")	33	SEC("classifier/3")
34	int bpf_func_3(struct __sk_buff *skb)	34	int bpf_func_3(struct __sk_buff *skb)
35	{	35	{
36	bpf_tail_call(skb, &jmp_table, 4);	36	bpf_tail_call_static(skb, &jmp_table, 4);
37	return 3;	37	return 3;
38	}	38	}
39		39
40	SEC("classifier/4")	40	SEC("classifier/4")
41	int bpf_func_4(struct __sk_buff *skb)	41	int bpf_func_4(struct __sk_buff *skb)
42	{	42	{
43	bpf_tail_call(skb, &jmp_table, 3);	43	bpf_tail_call_static(skb, &jmp_table, 3);
44	return 4;	44	return 4;
45	}	45	}
46		46
47	SEC("classifier")	47	SEC("classifier")
48	int entry(struct __sk_buff *skb)	48	int entry(struct __sk_buff *skb)
49	{	49	{
50	bpf_tail_call(skb, &jmp_table, 0);	50	bpf_tail_call_static(skb, &jmp_table, 0);
51	/* Check multi-prog update. */	51	/* Check multi-prog update. */
52	bpf_tail_call(skb, &jmp_table, 2);	52	bpf_tail_call_static(skb, &jmp_table, 2);
53	/* Check tail call limit. */	53	/* Check tail call limit. */
54	bpf_tail_call(skb, &jmp_table, 3);	54	bpf_tail_call_static(skb, &jmp_table, 3);
55	return 3;	55	return 3;
56	}	56	}
57		57

tools/testing/selftests/bpf/progs/tailcall3.c+2−2

@@ -16,14 +16,14 @@ SEC("classifier/0") · 15 unmodified lines
16	int bpf_func_0(struct __sk_buff *skb)	16	int bpf_func_0(struct __sk_buff *skb)
17	{	17	{
18	count++;	18	count++;
19	bpf_tail_call(skb, &jmp_table, 0);	19	bpf_tail_call_static(skb, &jmp_table, 0);
20	return 1;	20	return 1;
21	}	21	}
22		22
23	SEC("classifier")	23	SEC("classifier")
24	int entry(struct __sk_buff *skb)	24	int entry(struct __sk_buff *skb)
25	{	25	{
26	bpf_tail_call(skb, &jmp_table, 0);	26	bpf_tail_call_static(skb, &jmp_table, 0);
27	return 0;	27	return 0;
28	}	28	}
29		29

tools/testing/selftests/bpf/progs/tailcall_bpf2bpf1.c+2−2

@@ -21,7 +21,7 @@ TAIL_FUNC(1) · 20 unmodified lines
21	static __noinline	21	static __noinline
22	int subprog_tail(struct __sk_buff *skb)	22	int subprog_tail(struct __sk_buff *skb)
23	{	23	{
24	bpf_tail_call(skb, &jmp_table, 0);	24	bpf_tail_call_static(skb, &jmp_table, 0);
25		25
26	return skb->len * 2;	26	return skb->len * 2;
27	}	27	}
@@ -29,7 +29,7 @@ int subprog_tail(struct __sk_buff *skb) · 1 unmodified line
29	SEC("classifier")	29	SEC("classifier")
30	int entry(struct __sk_buff *skb)	30	int entry(struct __sk_buff *skb)
31	{	31	{
32	bpf_tail_call(skb, &jmp_table, 1);	32	bpf_tail_call_static(skb, &jmp_table, 1);
33		33
34	return subprog_tail(skb);	34	return subprog_tail(skb);
35	}	35	}

tools/testing/selftests/bpf/progs/tailcall_bpf2bpf2.c+3−3

@@ -14,9 +14,9 @@ static __noinline · 13 unmodified lines
14	int subprog_tail(struct __sk_buff *skb)	14	int subprog_tail(struct __sk_buff *skb)
15	{	15	{
16	if (load_byte(skb, 0))	16	if (load_byte(skb, 0))
17	bpf_tail_call(skb, &jmp_table, 1);	17	bpf_tail_call_static(skb, &jmp_table, 1);
18	else	18	else
19	bpf_tail_call(skb, &jmp_table, 0);	19	bpf_tail_call_static(skb, &jmp_table, 0);
20	return 1;	20	return 1;
21	}	21	}
22		22
@@ -32,7 +32,7 @@ int bpf_func_0(struct __sk_buff *skb) · 9 unmodified lines
32	SEC("classifier")	32	SEC("classifier")
33	int entry(struct __sk_buff *skb)	33	int entry(struct __sk_buff *skb)
34	{	34	{
35	bpf_tail_call(skb, &jmp_table, 0);	35	bpf_tail_call_static(skb, &jmp_table, 0);
36		36
37	return 0;	37	return 0;
38	}	38	}

tools/testing/selftests/bpf/progs/tailcall_bpf2bpf3.c+3−3

@@ -16,9 +16,9 @@ int subprog_tail2(struct __sk_buff *skb) · 15 unmodified lines
16	volatile char arr[64] = {};	16	volatile char arr[64] = {};
17		17
18	if (load_word(skb, 0) \|\| load_half(skb, 0))	18	if (load_word(skb, 0) \|\| load_half(skb, 0))
19	bpf_tail_call(skb, &jmp_table, 10);	19	bpf_tail_call_static(skb, &jmp_table, 10);
20	else	20	else
21	bpf_tail_call(skb, &jmp_table, 1);	21	bpf_tail_call_static(skb, &jmp_table, 1);
22		22
23	return skb->len;	23	return skb->len;
24	}	24	}
@@ -28,7 +28,7 @@ int subprog_tail(struct __sk_buff *skb) · 3 unmodified lines
28	{	28	{
29	volatile char arr[64] = {};	29	volatile char arr[64] = {};
30		30
31	bpf_tail_call(skb, &jmp_table, 0);	31	bpf_tail_call_static(skb, &jmp_table, 0);
32		32
33	return skb->len * 2;	33	return skb->len * 2;
34	}	34	}

tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c+3−3

@@ -14,21 +14,21 @@ static volatile int count; · 13 unmodified lines
14	__noinline	14	__noinline
15	int subprog_tail_2(struct __sk_buff *skb)	15	int subprog_tail_2(struct __sk_buff *skb)
16	{	16	{
17	bpf_tail_call(skb, &jmp_table, 2);	17	bpf_tail_call_static(skb, &jmp_table, 2);
18	return skb->len * 3;	18	return skb->len * 3;
19	}	19	}
20		20
21	__noinline	21	__noinline
22	int subprog_tail_1(struct __sk_buff *skb)	22	int subprog_tail_1(struct __sk_buff *skb)
23	{	23	{
24	bpf_tail_call(skb, &jmp_table, 1);	24	bpf_tail_call_static(skb, &jmp_table, 1);
25	return skb->len * 2;	25	return skb->len * 2;
26	}	26	}
27		27
28	__noinline	28	__noinline
29	int subprog_tail(struct __sk_buff *skb)	29	int subprog_tail(struct __sk_buff *skb)
30	{	30	{
31	bpf_tail_call(skb, &jmp_table, 0);	31	bpf_tail_call_static(skb, &jmp_table, 0);
32	return skb->len;	32	return skb->len;
33	}	33	}
34		34

tools/testing/selftests/bpf/progs/test_btf_skc_cls_ingress.c+174−0

@@ -0,0 +1,174 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3
	4	#include <string.h>
	5	#include <errno.h>
	6	#include <netinet/in.h>
	7	#include <linux/stddef.h>
	8	#include <linux/bpf.h>
	9	#include <linux/ipv6.h>
	10	#include <linux/tcp.h>
	11	#include <linux/if_ether.h>
	12	#include <linux/pkt_cls.h>
	13
	14	#include <bpf/bpf_helpers.h>
	15	#include <bpf/bpf_endian.h>
	16	#include "bpf_tcp_helpers.h"
	17
	18	struct sockaddr_in6 srv_sa6 = {};
	19	__u16 listen_tp_sport = 0;
	20	__u16 req_sk_sport = 0;
	21	__u32 recv_cookie = 0;
	22	__u32 gen_cookie = 0;
	23	__u32 linum = 0;
	24
	25	#define LOG() ({ if (!linum) linum = __LINE__; })
	26
	27	static void test_syncookie_helper(struct ipv6hdr ip6h, struct tcphdr th,
	28	struct tcp_sock *tp,
	29	struct __sk_buff *skb)
	30	{
	31	if (th->syn) {
	32	__s64 mss_cookie;
	33	void *data_end;
	34
	35	data_end = (void *)(long)(skb->data_end);
	36
	37	if (th->doff * 4 != 40) {
	38	LOG();
	39	return;
	40	}
	41
	42	if ((void *)th + 40 > data_end) {
	43	LOG();
	44	return;
	45	}
	46
	47	mss_cookie = bpf_tcp_gen_syncookie(tp, ip6h, sizeof(*ip6h),
	48	th, 40);
	49	if (mss_cookie < 0) {
	50	if (mss_cookie != -ENOENT)
	51	LOG();
	52	} else {
	53	gen_cookie = (__u32)mss_cookie;
	54	}
	55	} else if (gen_cookie) {
	56	/* It was in cookie mode */
	57	int ret = bpf_tcp_check_syncookie(tp, ip6h, sizeof(*ip6h),
	58	th, sizeof(*th));
	59
	60	if (ret < 0) {
	61	if (ret != -ENOENT)
	62	LOG();
	63	} else {
	64	recv_cookie = bpf_ntohl(th->ack_seq) - 1;
	65	}
	66	}
	67	}
	68
	69	static int handle_ip6_tcp(struct ipv6hdr ip6h, struct __sk_buff skb)
	70	{
	71	struct bpf_sock_tuple *tuple;
	72	struct bpf_sock *bpf_skc;
	73	unsigned int tuple_len;
	74	struct tcphdr *th;
	75	void *data_end;
	76
	77	data_end = (void *)(long)(skb->data_end);
	78
	79	th = (struct tcphdr *)(ip6h + 1);
	80	if (th + 1 > data_end)
	81	return TC_ACT_OK;
	82
	83	/* Is it the testing traffic? */
	84	if (th->dest != srv_sa6.sin6_port)
	85	return TC_ACT_OK;
	86
	87	tuple_len = sizeof(tuple->ipv6);
	88	tuple = (struct bpf_sock_tuple *)&ip6h->saddr;
	89	if ((void *)tuple + tuple_len > data_end) {
	90	LOG();
	91	return TC_ACT_OK;
	92	}
	93
	94	bpf_skc = bpf_skc_lookup_tcp(skb, tuple, tuple_len,
	95	BPF_F_CURRENT_NETNS, 0);
	96	if (!bpf_skc) {
	97	LOG();
	98	return TC_ACT_OK;
	99	}
	100
	101	if (bpf_skc->state == BPF_TCP_NEW_SYN_RECV) {
	102	struct request_sock *req_sk;
	103
	104	req_sk = (struct request_sock *)bpf_skc_to_tcp_request_sock(bpf_skc);
	105	if (!req_sk) {
	106	LOG();
	107	goto release;
	108	}
	109
	110	if (bpf_sk_assign(skb, req_sk, 0)) {
	111	LOG();
	112	goto release;
	113	}
	114
	115	req_sk_sport = req_sk->__req_common.skc_num;
	116
	117	bpf_sk_release(req_sk);
	118	return TC_ACT_OK;
	119	} else if (bpf_skc->state == BPF_TCP_LISTEN) {
	120	struct tcp_sock *tp;
	121
	122	tp = bpf_skc_to_tcp_sock(bpf_skc);
	123	if (!tp) {
	124	LOG();
	125	goto release;
	126	}
	127
	128	if (bpf_sk_assign(skb, tp, 0)) {
	129	LOG();
	130	goto release;
	131	}
	132
	133	listen_tp_sport = tp->inet_conn.icsk_inet.sk.__sk_common.skc_num;
	134
	135	test_syncookie_helper(ip6h, th, tp, skb);
	136	bpf_sk_release(tp);
	137	return TC_ACT_OK;
	138	}
	139
	140	if (bpf_sk_assign(skb, bpf_skc, 0))
	141	LOG();
	142
	143	release:
	144	bpf_sk_release(bpf_skc);
	145	return TC_ACT_OK;
	146	}
	147
	148	SEC("classifier/ingress")
	149	int cls_ingress(struct __sk_buff *skb)
	150	{
	151	struct ipv6hdr *ip6h;
	152	struct ethhdr *eth;
	153	void *data_end;
	154
	155	data_end = (void *)(long)(skb->data_end);
	156
	157	eth = (struct ethhdr *)(long)(skb->data);
	158	if (eth + 1 > data_end)
	159	return TC_ACT_OK;
	160
	161	if (eth->h_proto != bpf_htons(ETH_P_IPV6))
	162	return TC_ACT_OK;
	163
	164	ip6h = (struct ipv6hdr *)(eth + 1);
	165	if (ip6h + 1 > data_end)
	166	return TC_ACT_OK;
	167
	168	if (ip6h->nexthdr == IPPROTO_TCP)
	169	return handle_ip6_tcp(ip6h, skb);
	170
	171	return TC_ACT_OK;
	172	}
	173
	174	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_overhead.c+0−6

@@ -39,10 +39,4 @@ int BPF_PROG(prog5, struct task_struct tsk, const char buf, bool exec) · 38 unmodified lines
39	return 0;	39	return 0;
40	}	40	}
41		41
42	SEC("fmod_ret/__set_task_comm")
43	int BPF_PROG(prog6, struct task_struct tsk, const char buf, bool exec)
44	{
45	return !tsk;
46	}
47
48	char _license[] SEC("license") = "GPL";	42	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_pe_preserve_elems.c+38−0

@@ -0,0 +1,38 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	// Copyright (c) 2020 Facebook
	3	#include <linux/bpf.h>
	4	#include <bpf/bpf_helpers.h>
	5	#include <bpf/bpf_tracing.h>
	6
	7	struct {
	8	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
	9	__uint(max_entries, 1);
	10	__uint(key_size, sizeof(int));
	11	__uint(value_size, sizeof(int));
	12	} array_1 SEC(".maps");
	13
	14	struct {
	15	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
	16	__uint(max_entries, 1);
	17	__uint(key_size, sizeof(int));
	18	__uint(value_size, sizeof(int));
	19	__uint(map_flags, BPF_F_PRESERVE_ELEMS);
	20	} array_2 SEC(".maps");
	21
	22	SEC("raw_tp/sched_switch")
	23	int BPF_PROG(read_array_1)
	24	{
	25	struct bpf_perf_event_value val;
	26
	27	return bpf_perf_event_read_value(&array_1, 0, &val, sizeof(val));
	28	}
	29
	30	SEC("raw_tp/task_rename")
	31	int BPF_PROG(read_array_2)
	32	{
	33	struct bpf_perf_event_value val;
	34
	35	return bpf_perf_event_read_value(&array_2, 0, &val, sizeof(val));
	36	}
	37
	38	char LICENSE[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_raw_tp_test_run.c+24−0

@@ -0,0 +1,24 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2020 Facebook */
	3
	4	#include "vmlinux.h"
	5	#include <bpf/bpf_helpers.h>
	6	#include <bpf/bpf_tracing.h>
	7
	8	__u32 count = 0;
	9	__u32 on_cpu = 0xffffffff;
	10
	11	SEC("raw_tp/task_rename")
	12	int BPF_PROG(rename, struct task_struct task, char comm)
	13	{
	14
	15	count++;
	16	if ((__u64) task == 0x1234ULL && (__u64) comm == 0x5678ULL) {
	17	on_cpu = bpf_get_smp_processor_id();
	18	return (long)task + (long)comm;
	19	}
	20
	21	return 0;
	22	}
	23
	24	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_sk_lookup.c+101−115

@@ -19,6 +19,17 @@ · 18 unmodified lines
19	#define IP6(aaaa, bbbb, cccc, dddd) \	19	#define IP6(aaaa, bbbb, cccc, dddd) \
20	{ bpf_htonl(aaaa), bpf_htonl(bbbb), bpf_htonl(cccc), bpf_htonl(dddd) }	20	{ bpf_htonl(aaaa), bpf_htonl(bbbb), bpf_htonl(cccc), bpf_htonl(dddd) }
21		21
		22	/* Macros for least-significant byte and word accesses. */
		23	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
		24	#define LSE_INDEX(index, size) (index)
		25	#else
		26	#define LSE_INDEX(index, size) ((size) - (index) - 1)
		27	#endif
		28	#define LSB(value, index) \
		29	(((__u8 *)&(value))[LSE_INDEX((index), sizeof(value))])
		30	#define LSW(value, index) \
		31	(((__u16 *)&(value))[LSE_INDEX((index), sizeof(value) / 2)])
		32
22	#define MAX_SOCKS 32	33	#define MAX_SOCKS 32
23		34
24	struct {	35	struct {
@@ -369,171 +380,146 @@ int ctx_narrow_access(struct bpf_sk_lookup *ctx) · 344 unmodified lines
369	{	380	{
370	struct bpf_sock *sk;	381	struct bpf_sock *sk;
371	int err, family;	382	int err, family;
372	__u16 *half;
373	__u8 *byte;
374	bool v4;	383	bool v4;
375		384
376	v4 = (ctx->family == AF_INET);	385	v4 = (ctx->family == AF_INET);
377		386
378	/* Narrow loads from family field */	387	/* Narrow loads from family field */
379	byte = (__u8 *)&ctx->family;	388	if (LSB(ctx->family, 0) != (v4 ? AF_INET : AF_INET6) \|\|
380	half = (__u16 *)&ctx->family;	389	LSB(ctx->family, 1) != 0 \|\| LSB(ctx->family, 2) != 0 \|\| LSB(ctx->family, 3) != 0)
381	if (byte[0] != (v4 ? AF_INET : AF_INET6) \|\|
382	byte[1] != 0 \|\| byte[2] != 0 \|\| byte[3] != 0)
383	return SK_DROP;	390	return SK_DROP;
384	if (half[0] != (v4 ? AF_INET : AF_INET6))	391	if (LSW(ctx->family, 0) != (v4 ? AF_INET : AF_INET6))
385	return SK_DROP;	392	return SK_DROP;
386		393
387	byte = (__u8 *)&ctx->protocol;	394	/* Narrow loads from protocol field */
388	if (byte[0] != IPPROTO_TCP \|\|	395	if (LSB(ctx->protocol, 0) != IPPROTO_TCP \|\|
389	byte[1] != 0 \|\| byte[2] != 0 \|\| byte[3] != 0)	396	LSB(ctx->protocol, 1) != 0 \|\| LSB(ctx->protocol, 2) != 0 \|\| LSB(ctx->protocol, 3) != 0)
390	return SK_DROP;	397	return SK_DROP;
391	half = (__u16 *)&ctx->protocol;	398	if (LSW(ctx->protocol, 0) != IPPROTO_TCP)
392	if (half[0] != IPPROTO_TCP)
393	return SK_DROP;	399	return SK_DROP;
394		400
395	/* Narrow loads from remote_port field. Expect non-0 value. */	401	/* Narrow loads from remote_port field. Expect non-0 value. */
396	byte = (__u8 *)&ctx->remote_port;	402	if (LSB(ctx->remote_port, 0) == 0 && LSB(ctx->remote_port, 1) == 0 &&
397	if (byte[0] == 0 && byte[1] == 0 && byte[2] == 0 && byte[3] == 0)	403	LSB(ctx->remote_port, 2) == 0 && LSB(ctx->remote_port, 3) == 0)
398	return SK_DROP;	404	return SK_DROP;
399	half = (__u16 *)&ctx->remote_port;	405	if (LSW(ctx->remote_port, 0) == 0)
400	if (half[0] == 0)
401	return SK_DROP;	406	return SK_DROP;
402		407
403	/* Narrow loads from local_port field. Expect DST_PORT. */	408	/* Narrow loads from local_port field. Expect DST_PORT. */
404	byte = (__u8 *)&ctx->local_port;	409	if (LSB(ctx->local_port, 0) != ((DST_PORT >> 0) & 0xff) \|\|
405	if (byte[0] != ((DST_PORT >> 0) & 0xff) \|\|	410	LSB(ctx->local_port, 1) != ((DST_PORT >> 8) & 0xff) \|\|
406	byte[1] != ((DST_PORT >> 8) & 0xff) \|\|	411	LSB(ctx->local_port, 2) != 0 \|\| LSB(ctx->local_port, 3) != 0)
407	byte[2] != 0 \|\| byte[3] != 0)
408	return SK_DROP;	412	return SK_DROP;
409	half = (__u16 *)&ctx->local_port;	413	if (LSW(ctx->local_port, 0) != DST_PORT)
410	if (half[0] != DST_PORT)
411	return SK_DROP;	414	return SK_DROP;
412		415
413	/* Narrow loads from IPv4 fields */	416	/* Narrow loads from IPv4 fields */
414	if (v4) {	417	if (v4) {
415	/* Expect non-0.0.0.0 in remote_ip4 */	418	/* Expect non-0.0.0.0 in remote_ip4 */
416	byte = (__u8 *)&ctx->remote_ip4;	419	if (LSB(ctx->remote_ip4, 0) == 0 && LSB(ctx->remote_ip4, 1) == 0 &&
417	if (byte[0] == 0 && byte[1] == 0 &&	420	LSB(ctx->remote_ip4, 2) == 0 && LSB(ctx->remote_ip4, 3) == 0)
418	byte[2] == 0 && byte[3] == 0)
419	return SK_DROP;	421	return SK_DROP;
420	half = (__u16 *)&ctx->remote_ip4;	422	if (LSW(ctx->remote_ip4, 0) == 0 && LSW(ctx->remote_ip4, 1) == 0)
421	if (half[0] == 0 && half[1] == 0)
422	return SK_DROP;	423	return SK_DROP;
423		424
424	/* Expect DST_IP4 in local_ip4 */	425	/* Expect DST_IP4 in local_ip4 */
425	byte = (__u8 *)&ctx->local_ip4;	426	if (LSB(ctx->local_ip4, 0) != ((DST_IP4 >> 0) & 0xff) \|\|
426	if (byte[0] != ((DST_IP4 >> 0) & 0xff) \|\|	427	LSB(ctx->local_ip4, 1) != ((DST_IP4 >> 8) & 0xff) \|\|
427	byte[1] != ((DST_IP4 >> 8) & 0xff) \|\|	428	LSB(ctx->local_ip4, 2) != ((DST_IP4 >> 16) & 0xff) \|\|
428	byte[2] != ((DST_IP4 >> 16) & 0xff) \|\|	429	LSB(ctx->local_ip4, 3) != ((DST_IP4 >> 24) & 0xff))
429	byte[3] != ((DST_IP4 >> 24) & 0xff))
430	return SK_DROP;	430	return SK_DROP;
431	half = (__u16 *)&ctx->local_ip4;	431	if (LSW(ctx->local_ip4, 0) != ((DST_IP4 >> 0) & 0xffff) \|\|
432	if (half[0] != ((DST_IP4 >> 0) & 0xffff) \|\|	432	LSW(ctx->local_ip4, 1) != ((DST_IP4 >> 16) & 0xffff))
433	half[1] != ((DST_IP4 >> 16) & 0xffff))
434	return SK_DROP;	433	return SK_DROP;
435	} else {	434	} else {
436	/* Expect 0.0.0.0 IPs when family != AF_INET */	435	/* Expect 0.0.0.0 IPs when family != AF_INET */
437	byte = (__u8 *)&ctx->remote_ip4;	436	if (LSB(ctx->remote_ip4, 0) != 0 \|\| LSB(ctx->remote_ip4, 1) != 0 \|\|
438	if (byte[0] != 0 \|\| byte[1] != 0 &&	437	LSB(ctx->remote_ip4, 2) != 0 \|\| LSB(ctx->remote_ip4, 3) != 0)
439	byte[2] != 0 \|\| byte[3] != 0)
440	return SK_DROP;	438	return SK_DROP;
441	half = (__u16 *)&ctx->remote_ip4;	439	if (LSW(ctx->remote_ip4, 0) != 0 \|\| LSW(ctx->remote_ip4, 1) != 0)
442	if (half[0] != 0 \|\| half[1] != 0)
443	return SK_DROP;	440	return SK_DROP;
444		441
445	byte = (__u8 *)&ctx->local_ip4;	442	if (LSB(ctx->local_ip4, 0) != 0 \|\| LSB(ctx->local_ip4, 1) != 0 \|\|
446	if (byte[0] != 0 \|\| byte[1] != 0 &&	443	LSB(ctx->local_ip4, 2) != 0 \|\| LSB(ctx->local_ip4, 3) != 0)
447	byte[2] != 0 \|\| byte[3] != 0)
448	return SK_DROP;	444	return SK_DROP;
449	half = (__u16 *)&ctx->local_ip4;	445	if (LSW(ctx->local_ip4, 0) != 0 \|\| LSW(ctx->local_ip4, 1) != 0)
450	if (half[0] != 0 \|\| half[1] != 0)
451	return SK_DROP;	446	return SK_DROP;
452	}	447	}
453		448
454	/* Narrow loads from IPv6 fields */	449	/* Narrow loads from IPv6 fields */
455	if (!v4) {	450	if (!v4) {
456	/* Expenct non-:: IP in remote_ip6 */	451	/* Expect non-:: IP in remote_ip6 */
457	byte = (__u8 *)&ctx->remote_ip6;	452	if (LSB(ctx->remote_ip6[0], 0) == 0 && LSB(ctx->remote_ip6[0], 1) == 0 &&
458	if (byte[0] == 0 && byte[1] == 0 &&	453	LSB(ctx->remote_ip6[0], 2) == 0 && LSB(ctx->remote_ip6[0], 3) == 0 &&
459	byte[2] == 0 && byte[3] == 0 &&	454	LSB(ctx->remote_ip6[1], 0) == 0 && LSB(ctx->remote_ip6[1], 1) == 0 &&
460	byte[4] == 0 && byte[5] == 0 &&	455	LSB(ctx->remote_ip6[1], 2) == 0 && LSB(ctx->remote_ip6[1], 3) == 0 &&
461	byte[6] == 0 && byte[7] == 0 &&	456	LSB(ctx->remote_ip6[2], 0) == 0 && LSB(ctx->remote_ip6[2], 1) == 0 &&
462	byte[8] == 0 && byte[9] == 0 &&	457	LSB(ctx->remote_ip6[2], 2) == 0 && LSB(ctx->remote_ip6[2], 3) == 0 &&
463	byte[10] == 0 && byte[11] == 0 &&	458	LSB(ctx->remote_ip6[3], 0) == 0 && LSB(ctx->remote_ip6[3], 1) == 0 &&
464	byte[12] == 0 && byte[13] == 0 &&	459	LSB(ctx->remote_ip6[3], 2) == 0 && LSB(ctx->remote_ip6[3], 3) == 0)
465	byte[14] == 0 && byte[15] == 0)
466	return SK_DROP;	460	return SK_DROP;
467	half = (__u16 *)&ctx->remote_ip6;	461	if (LSW(ctx->remote_ip6[0], 0) == 0 && LSW(ctx->remote_ip6[0], 1) == 0 &&
468	if (half[0] == 0 && half[1] == 0 &&	462	LSW(ctx->remote_ip6[1], 0) == 0 && LSW(ctx->remote_ip6[1], 1) == 0 &&
469	half[2] == 0 && half[3] == 0 &&	463	LSW(ctx->remote_ip6[2], 0) == 0 && LSW(ctx->remote_ip6[2], 1) == 0 &&
470	half[4] == 0 && half[5] == 0 &&	464	LSW(ctx->remote_ip6[3], 0) == 0 && LSW(ctx->remote_ip6[3], 1) == 0)
471	half[6] == 0 && half[7] == 0)
472	return SK_DROP;	465	return SK_DROP;
473
474	/* Expect DST_IP6 in local_ip6 */	466	/* Expect DST_IP6 in local_ip6 */
475	byte = (__u8 *)&ctx->local_ip6;	467	if (LSB(ctx->local_ip6[0], 0) != ((DST_IP6[0] >> 0) & 0xff) \|\|
476	if (byte[0] != ((DST_IP6[0] >> 0) & 0xff) \|\|	468	LSB(ctx->local_ip6[0], 1) != ((DST_IP6[0] >> 8) & 0xff) \|\|
477	byte[1] != ((DST_IP6[0] >> 8) & 0xff) \|\|	469	LSB(ctx->local_ip6[0], 2) != ((DST_IP6[0] >> 16) & 0xff) \|\|
478	byte[2] != ((DST_IP6[0] >> 16) & 0xff) \|\|	470	LSB(ctx->local_ip6[0], 3) != ((DST_IP6[0] >> 24) & 0xff) \|\|
479	byte[3] != ((DST_IP6[0] >> 24) & 0xff) \|\|	471	LSB(ctx->local_ip6[1], 0) != ((DST_IP6[1] >> 0) & 0xff) \|\|
480	byte[4] != ((DST_IP6[1] >> 0) & 0xff) \|\|	472	LSB(ctx->local_ip6[1], 1) != ((DST_IP6[1] >> 8) & 0xff) \|\|
481	byte[5] != ((DST_IP6[1] >> 8) & 0xff) \|\|	473	LSB(ctx->local_ip6[1], 2) != ((DST_IP6[1] >> 16) & 0xff) \|\|
482	byte[6] != ((DST_IP6[1] >> 16) & 0xff) \|\|	474	LSB(ctx->local_ip6[1], 3) != ((DST_IP6[1] >> 24) & 0xff) \|\|
483	byte[7] != ((DST_IP6[1] >> 24) & 0xff) \|\|	475	LSB(ctx->local_ip6[2], 0) != ((DST_IP6[2] >> 0) & 0xff) \|\|
484	byte[8] != ((DST_IP6[2] >> 0) & 0xff) \|\|	476	LSB(ctx->local_ip6[2], 1) != ((DST_IP6[2] >> 8) & 0xff) \|\|
485	byte[9] != ((DST_IP6[2] >> 8) & 0xff) \|\|	477	LSB(ctx->local_ip6[2], 2) != ((DST_IP6[2] >> 16) & 0xff) \|\|
486	byte[10] != ((DST_IP6[2] >> 16) & 0xff) \|\|	478	LSB(ctx->local_ip6[2], 3) != ((DST_IP6[2] >> 24) & 0xff) \|\|
487	byte[11] != ((DST_IP6[2] >> 24) & 0xff) \|\|	479	LSB(ctx->local_ip6[3], 0) != ((DST_IP6[3] >> 0) & 0xff) \|\|
488	byte[12] != ((DST_IP6[3] >> 0) & 0xff) \|\|	480	LSB(ctx->local_ip6[3], 1) != ((DST_IP6[3] >> 8) & 0xff) \|\|
489	byte[13] != ((DST_IP6[3] >> 8) & 0xff) \|\|	481	LSB(ctx->local_ip6[3], 2) != ((DST_IP6[3] >> 16) & 0xff) \|\|
490	byte[14] != ((DST_IP6[3] >> 16) & 0xff) \|\|	482	LSB(ctx->local_ip6[3], 3) != ((DST_IP6[3] >> 24) & 0xff))
491	byte[15] != ((DST_IP6[3] >> 24) & 0xff))
492	return SK_DROP;	483	return SK_DROP;
493	half = (__u16 *)&ctx->local_ip6;	484	if (LSW(ctx->local_ip6[0], 0) != ((DST_IP6[0] >> 0) & 0xffff) \|\|
494	if (half[0] != ((DST_IP6[0] >> 0) & 0xffff) \|\|	485	LSW(ctx->local_ip6[0], 1) != ((DST_IP6[0] >> 16) & 0xffff) \|\|
495	half[1] != ((DST_IP6[0] >> 16) & 0xffff) \|\|	486	LSW(ctx->local_ip6[1], 0) != ((DST_IP6[1] >> 0) & 0xffff) \|\|
496	half[2] != ((DST_IP6[1] >> 0) & 0xffff) \|\|	487	LSW(ctx->local_ip6[1], 1) != ((DST_IP6[1] >> 16) & 0xffff) \|\|
497	half[3] != ((DST_IP6[1] >> 16) & 0xffff) \|\|	488	LSW(ctx->local_ip6[2], 0) != ((DST_IP6[2] >> 0) & 0xffff) \|\|
498	half[4] != ((DST_IP6[2] >> 0) & 0xffff) \|\|	489	LSW(ctx->local_ip6[2], 1) != ((DST_IP6[2] >> 16) & 0xffff) \|\|
499	half[5] != ((DST_IP6[2] >> 16) & 0xffff) \|\|	490	LSW(ctx->local_ip6[3], 0) != ((DST_IP6[3] >> 0) & 0xffff) \|\|
500	half[6] != ((DST_IP6[3] >> 0) & 0xffff) \|\|	491	LSW(ctx->local_ip6[3], 1) != ((DST_IP6[3] >> 16) & 0xffff))
501	half[7] != ((DST_IP6[3] >> 16) & 0xffff))
502	return SK_DROP;	492	return SK_DROP;
503	} else {	493	} else {
504	/* Expect :: IPs when family != AF_INET6 */	494	/* Expect :: IPs when family != AF_INET6 */
505	byte = (__u8 *)&ctx->remote_ip6;	495	if (LSB(ctx->remote_ip6[0], 0) != 0 \|\| LSB(ctx->remote_ip6[0], 1) != 0 \|\|
506	if (byte[0] != 0 \|\| byte[1] != 0 \|\|	496	LSB(ctx->remote_ip6[0], 2) != 0 \|\| LSB(ctx->remote_ip6[0], 3) != 0 \|\|
507	byte[2] != 0 \|\| byte[3] != 0 \|\|	497	LSB(ctx->remote_ip6[1], 0) != 0 \|\| LSB(ctx->remote_ip6[1], 1) != 0 \|\|
508	byte[4] != 0 \|\| byte[5] != 0 \|\|	498	LSB(ctx->remote_ip6[1], 2) != 0 \|\| LSB(ctx->remote_ip6[1], 3) != 0 \|\|
509	byte[6] != 0 \|\| byte[7] != 0 \|\|	499	LSB(ctx->remote_ip6[2], 0) != 0 \|\| LSB(ctx->remote_ip6[2], 1) != 0 \|\|
510	byte[8] != 0 \|\| byte[9] != 0 \|\|	500	LSB(ctx->remote_ip6[2], 2) != 0 \|\| LSB(ctx->remote_ip6[2], 3) != 0 \|\|
511	byte[10] != 0 \|\| byte[11] != 0 \|\|	501	LSB(ctx->remote_ip6[3], 0) != 0 \|\| LSB(ctx->remote_ip6[3], 1) != 0 \|\|
512	byte[12] != 0 \|\| byte[13] != 0 \|\|	502	LSB(ctx->remote_ip6[3], 2) != 0 \|\| LSB(ctx->remote_ip6[3], 3) != 0)
513	byte[14] != 0 \|\| byte[15] != 0)
514	return SK_DROP;	503	return SK_DROP;
515	half = (__u16 *)&ctx->remote_ip6;	504	if (LSW(ctx->remote_ip6[0], 0) != 0 \|\| LSW(ctx->remote_ip6[0], 1) != 0 \|\|
516	if (half[0] != 0 \|\| half[1] != 0 \|\|	505	LSW(ctx->remote_ip6[1], 0) != 0 \|\| LSW(ctx->remote_ip6[1], 1) != 0 \|\|
517	half[2] != 0 \|\| half[3] != 0 \|\|	506	LSW(ctx->remote_ip6[2], 0) != 0 \|\| LSW(ctx->remote_ip6[2], 1) != 0 \|\|
518	half[4] != 0 \|\| half[5] != 0 \|\|	507	LSW(ctx->remote_ip6[3], 0) != 0 \|\| LSW(ctx->remote_ip6[3], 1) != 0)
519	half[6] != 0 \|\| half[7] != 0)
520	return SK_DROP;	508	return SK_DROP;
521		509
522	byte = (__u8 *)&ctx->local_ip6;	510	if (LSB(ctx->local_ip6[0], 0) != 0 \|\| LSB(ctx->local_ip6[0], 1) != 0 \|\|
523	if (byte[0] != 0 \|\| byte[1] != 0 \|\|	511	LSB(ctx->local_ip6[0], 2) != 0 \|\| LSB(ctx->local_ip6[0], 3) != 0 \|\|
524	byte[2] != 0 \|\| byte[3] != 0 \|\|	512	LSB(ctx->local_ip6[1], 0) != 0 \|\| LSB(ctx->local_ip6[1], 1) != 0 \|\|
525	byte[4] != 0 \|\| byte[5] != 0 \|\|	513	LSB(ctx->local_ip6[1], 2) != 0 \|\| LSB(ctx->local_ip6[1], 3) != 0 \|\|
526	byte[6] != 0 \|\| byte[7] != 0 \|\|	514	LSB(ctx->local_ip6[2], 0) != 0 \|\| LSB(ctx->local_ip6[2], 1) != 0 \|\|
527	byte[8] != 0 \|\| byte[9] != 0 \|\|	515	LSB(ctx->local_ip6[2], 2) != 0 \|\| LSB(ctx->local_ip6[2], 3) != 0 \|\|
528	byte[10] != 0 \|\| byte[11] != 0 \|\|	516	LSB(ctx->local_ip6[3], 0) != 0 \|\| LSB(ctx->local_ip6[3], 1) != 0 \|\|
529	byte[12] != 0 \|\| byte[13] != 0 \|\|	517	LSB(ctx->local_ip6[3], 2) != 0 \|\| LSB(ctx->local_ip6[3], 3) != 0)
530	byte[14] != 0 \|\| byte[15] != 0)
531	return SK_DROP;	518	return SK_DROP;
532	half = (__u16 *)&ctx->local_ip6;	519	if (LSW(ctx->remote_ip6[0], 0) != 0 \|\| LSW(ctx->remote_ip6[0], 1) != 0 \|\|
533	if (half[0] != 0 \|\| half[1] != 0 \|\|	520	LSW(ctx->remote_ip6[1], 0) != 0 \|\| LSW(ctx->remote_ip6[1], 1) != 0 \|\|
534	half[2] != 0 \|\| half[3] != 0 \|\|	521	LSW(ctx->remote_ip6[2], 0) != 0 \|\| LSW(ctx->remote_ip6[2], 1) != 0 \|\|
535	half[4] != 0 \|\| half[5] != 0 \|\|	522	LSW(ctx->remote_ip6[3], 0) != 0 \|\| LSW(ctx->remote_ip6[3], 1) != 0)
536	half[6] != 0 \|\| half[7] != 0)
537	return SK_DROP;	523	return SK_DROP;
538	}	524	}
539		525

tools/testing/selftests/bpf/progs/test_sock_fields.crenamed from tools/testing/selftests/bpf/progs/test_sock_fields_kern.c+87−89

@@ -7,19 +7,7 @@ · 6 unmodified lines
7		7
8	#include <bpf/bpf_helpers.h>	8	#include <bpf/bpf_helpers.h>
9	#include <bpf/bpf_endian.h>	9	#include <bpf/bpf_endian.h>
10		10	#include "bpf_tcp_helpers.h"
11	enum bpf_addr_array_idx {
12	ADDR_SRV_IDX,
13	ADDR_CLI_IDX,
14	__NR_BPF_ADDR_ARRAY_IDX,
15	};
16
17	enum bpf_result_array_idx {
18	EGRESS_SRV_IDX,
19	EGRESS_CLI_IDX,
20	INGRESS_LISTEN_IDX,
21	__NR_BPF_RESULT_ARRAY_IDX,
22	};
23		11
24	enum bpf_linum_array_idx {	12	enum bpf_linum_array_idx {
25	EGRESS_LINUM_IDX,	13	EGRESS_LINUM_IDX,
@@ -27,27 +15,6 @@ enum bpf_linum_array_idx { · 1 unmodified line
27	__NR_BPF_LINUM_ARRAY_IDX,	15	__NR_BPF_LINUM_ARRAY_IDX,
28	};	16	};
29		17
30	struct {
31	__uint(type, BPF_MAP_TYPE_ARRAY);
32	__uint(max_entries, __NR_BPF_ADDR_ARRAY_IDX);
33	__type(key, __u32);
34	__type(value, struct sockaddr_in6);
35	} addr_map SEC(".maps");
36
37	struct {
38	__uint(type, BPF_MAP_TYPE_ARRAY);
39	__uint(max_entries, __NR_BPF_RESULT_ARRAY_IDX);
40	__type(key, __u32);
41	__type(value, struct bpf_sock);
42	} sock_result_map SEC(".maps");
43
44	struct {
45	__uint(type, BPF_MAP_TYPE_ARRAY);
46	__uint(max_entries, __NR_BPF_RESULT_ARRAY_IDX);
47	__type(key, __u32);
48	__type(value, struct bpf_tcp_sock);
49	} tcp_sock_result_map SEC(".maps");
50
51	struct {	18	struct {
52	__uint(type, BPF_MAP_TYPE_ARRAY);	19	__uint(type, BPF_MAP_TYPE_ARRAY);
53	__uint(max_entries, __NR_BPF_LINUM_ARRAY_IDX);	20	__uint(max_entries, __NR_BPF_LINUM_ARRAY_IDX);
@@ -74,6 +41,17 @@ struct { · 20 unmodified lines
74	__type(value, struct bpf_spinlock_cnt);	41	__type(value, struct bpf_spinlock_cnt);
75	} sk_pkt_out_cnt10 SEC(".maps");	42	} sk_pkt_out_cnt10 SEC(".maps");
76		43
		44	struct bpf_tcp_sock listen_tp = {};
		45	struct sockaddr_in6 srv_sa6 = {};
		46	struct bpf_tcp_sock cli_tp = {};
		47	struct bpf_tcp_sock srv_tp = {};
		48	struct bpf_sock listen_sk = {};
		49	struct bpf_sock srv_sk = {};
		50	struct bpf_sock cli_sk = {};
		51	__u64 parent_cg_id = 0;
		52	__u64 child_cg_id = 0;
		53	__u64 lsndtime = 0;
		54
77	static bool is_loopback6(__u32 *a6)	55	static bool is_loopback6(__u32 *a6)
78	{	56	{
79	return !a6[0] && !a6[1] && !a6[2] && a6[3] == bpf_htonl(1);	57	return !a6[0] && !a6[1] && !a6[2] && a6[3] == bpf_htonl(1);
@@ -130,62 +108,86 @@ static void tpcpy(struct bpf_tcp_sock *dst, · 50 unmodified lines
130	dst->bytes_acked = src->bytes_acked;	108	dst->bytes_acked = src->bytes_acked;
131	}	109	}
132		110
133	#define RETURN { \	111	/* Always return CG_OK so that no pkt will be filtered out */
		112	#define CG_OK 1
		113
		114	#define RET_LOG() ({ \
134	linum = __LINE__; \	115	linum = __LINE__; \
135	bpf_map_update_elem(&linum_map, &linum_idx, &linum, 0); \	116	bpf_map_update_elem(&linum_map, &linum_idx, &linum, BPF_NOEXIST); \
136	return 1; \	117	return CG_OK; \
137	}	118	})
138		119
139	SEC("cgroup_skb/egress")	120	SEC("cgroup_skb/egress")
140	int egress_read_sock_fields(struct __sk_buff *skb)	121	int egress_read_sock_fields(struct __sk_buff *skb)
141	{	122	{
142	struct bpf_spinlock_cnt cli_cnt_init = { .lock = 0, .cnt = 0xeB9F };	123	struct bpf_spinlock_cnt cli_cnt_init = { .lock = 0, .cnt = 0xeB9F };
143	__u32 srv_idx = ADDR_SRV_IDX, cli_idx = ADDR_CLI_IDX, result_idx;
144	struct bpf_spinlock_cnt pkt_out_cnt, pkt_out_cnt10;	124	struct bpf_spinlock_cnt pkt_out_cnt, pkt_out_cnt10;
145	struct sockaddr_in6 srv_sa6, cli_sa6;
146	struct bpf_tcp_sock tp, tp_ret;	125	struct bpf_tcp_sock tp, tp_ret;
147	struct bpf_sock sk, sk_ret;	126	struct bpf_sock sk, sk_ret;
148	__u32 linum, linum_idx;	127	__u32 linum, linum_idx;
		128	struct tcp_sock *ktp;
149		129
150	linum_idx = EGRESS_LINUM_IDX;	130	linum_idx = EGRESS_LINUM_IDX;
151		131
152	sk = skb->sk;	132	sk = skb->sk;
153	if (!sk \|\| sk->state == 10)	133	if (!sk)
154	RETURN;	134	RET_LOG();
155		135
		136	/* Not the testing egress traffic or
		137	* TCP_LISTEN (10) socket will be copied at the ingress side.
		138	*/
		139	if (sk->family != AF_INET6 \|\| !is_loopback6(sk->src_ip6) \|\|
		140	sk->state == 10)
		141	return CG_OK;
		142
		143	if (sk->src_port == bpf_ntohs(srv_sa6.sin6_port)) {
		144	/* Server socket */
		145	sk_ret = &srv_sk;
		146	tp_ret = &srv_tp;
		147	} else if (sk->dst_port == srv_sa6.sin6_port) {
		148	/* Client socket */
		149	sk_ret = &cli_sk;
		150	tp_ret = &cli_tp;
		151	} else {
		152	/* Not the testing egress traffic */
		153	return CG_OK;
		154	}
		155
		156	/* It must be a fullsock for cgroup_skb/egress prog */
156	sk = bpf_sk_fullsock(sk);	157	sk = bpf_sk_fullsock(sk);
157	if (!sk \|\| sk->family != AF_INET6 \|\| sk->protocol != IPPROTO_TCP \|\|	158	if (!sk)
158	!is_loopback6(sk->src_ip6))	159	RET_LOG();
159	RETURN;	160
		161	/* Not the testing egress traffic */
		162	if (sk->protocol != IPPROTO_TCP)
		163	return CG_OK;
160		164
161	tp = bpf_tcp_sock(sk);	165	tp = bpf_tcp_sock(sk);
162	if (!tp)	166	if (!tp)
163	RETURN;	167	RET_LOG();
164		168
165	srv_sa6 = bpf_map_lookup_elem(&addr_map, &srv_idx);	169	skcpy(sk_ret, sk);
166	cli_sa6 = bpf_map_lookup_elem(&addr_map, &cli_idx);	170	tpcpy(tp_ret, tp);
167	if (!srv_sa6 \|\| !cli_sa6)
168	RETURN;
169		171
170	if (sk->src_port == bpf_ntohs(srv_sa6->sin6_port))	172	if (sk_ret == &srv_sk) {
171	result_idx = EGRESS_SRV_IDX;	173	ktp = bpf_skc_to_tcp_sock(sk);
172	else if (sk->src_port == bpf_ntohs(cli_sa6->sin6_port))
173	result_idx = EGRESS_CLI_IDX;
174	else
175	RETURN;
176		174
177	sk_ret = bpf_map_lookup_elem(&sock_result_map, &result_idx);	175	if (!ktp)
178	tp_ret = bpf_map_lookup_elem(&tcp_sock_result_map, &result_idx);	176	RET_LOG();
179	if (!sk_ret \|\| !tp_ret)
180	RETURN;
181		177
182	skcpy(sk_ret, sk);	178	lsndtime = ktp->lsndtime;
183	tpcpy(tp_ret, tp);	179
		180	child_cg_id = bpf_sk_cgroup_id(ktp);
		181	if (!child_cg_id)
		182	RET_LOG();
		183
		184	parent_cg_id = bpf_sk_ancestor_cgroup_id(ktp, 2);
		185	if (!parent_cg_id)
		186	RET_LOG();
184		187
185	if (result_idx == EGRESS_SRV_IDX) {
186	/* The userspace has created it for srv sk */	188	/* The userspace has created it for srv sk */
187	pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, sk, 0, 0);	189	pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, ktp, 0, 0);
188	pkt_out_cnt10 = bpf_sk_storage_get(&sk_pkt_out_cnt10, sk,	190	pkt_out_cnt10 = bpf_sk_storage_get(&sk_pkt_out_cnt10, ktp,
189	0, 0);	191	0, 0);
190	} else {	192	} else {
191	pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, sk,	193	pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, sk,
@@ -197,7 +199,7 @@ int egress_read_sock_fields(struct __sk_buff *skb) · 5 unmodified lines
197	}	199	}
198		200
199	if (!pkt_out_cnt \|\| !pkt_out_cnt10)	201	if (!pkt_out_cnt \|\| !pkt_out_cnt10)
200	RETURN;	202	RET_LOG();
201		203
202	/* Even both cnt and cnt10 have lock defined in their BTF,	204	/* Even both cnt and cnt10 have lock defined in their BTF,
203	* intentionally one cnt takes lock while one does not	205	* intentionally one cnt takes lock while one does not
@@ -208,48 +210,44 @@ int egress_read_sock_fields(struct __sk_buff *skb) · 4 unmodified lines
208	pkt_out_cnt10->cnt += 10;	210	pkt_out_cnt10->cnt += 10;
209	bpf_spin_unlock(&pkt_out_cnt10->lock);	211	bpf_spin_unlock(&pkt_out_cnt10->lock);
210		212
211	RETURN;	213	return CG_OK;
212	}	214	}
213		215
214	SEC("cgroup_skb/ingress")	216	SEC("cgroup_skb/ingress")
215	int ingress_read_sock_fields(struct __sk_buff *skb)	217	int ingress_read_sock_fields(struct __sk_buff *skb)
216	{	218	{
217	__u32 srv_idx = ADDR_SRV_IDX, result_idx = INGRESS_LISTEN_IDX;	219	struct bpf_tcp_sock *tp;
218	struct bpf_tcp_sock tp, tp_ret;
219	struct bpf_sock sk, sk_ret;
220	struct sockaddr_in6 *srv_sa6;
221	__u32 linum, linum_idx;	220	__u32 linum, linum_idx;
		221	struct bpf_sock *sk;
222		222
223	linum_idx = INGRESS_LINUM_IDX;	223	linum_idx = INGRESS_LINUM_IDX;
224		224
225	sk = skb->sk;	225	sk = skb->sk;
226	if (!sk \|\| sk->family != AF_INET6 \|\| !is_loopback6(sk->src_ip6))	226	if (!sk)
227	RETURN;	227	RET_LOG();
228		228
229	srv_sa6 = bpf_map_lookup_elem(&addr_map, &srv_idx);	229	/* Not the testing ingress traffic to the server */
230	if (!srv_sa6 \|\| sk->src_port != bpf_ntohs(srv_sa6->sin6_port))	230	if (sk->family != AF_INET6 \|\| !is_loopback6(sk->src_ip6) \|\|
231	RETURN;	231	sk->src_port != bpf_ntohs(srv_sa6.sin6_port))
		232	return CG_OK;
232		233
233	if (sk->state != 10 && sk->state != 12)	234	/* Only interested in TCP_LISTEN */
234	RETURN;	235	if (sk->state != 10)
		236	return CG_OK;
235		237
236	sk = bpf_get_listener_sock(sk);	238	/* It must be a fullsock for cgroup_skb/ingress prog */
		239	sk = bpf_sk_fullsock(sk);
237	if (!sk)	240	if (!sk)
238	RETURN;	241	RET_LOG();
239		242
240	tp = bpf_tcp_sock(sk);	243	tp = bpf_tcp_sock(sk);
241	if (!tp)	244	if (!tp)
242	RETURN;	245	RET_LOG();
243
244	sk_ret = bpf_map_lookup_elem(&sock_result_map, &result_idx);
245	tp_ret = bpf_map_lookup_elem(&tcp_sock_result_map, &result_idx);
246	if (!sk_ret \|\| !tp_ret)
247	RETURN;
248		246
249	skcpy(sk_ret, sk);	247	skcpy(&listen_sk, sk);
250	tpcpy(tp_ret, tp);	248	tpcpy(&listen_tp, tp);
251		249
252	RETURN;	250	return CG_OK;
253	}	251	}
254		252
255	char _license[] SEC("license") = "GPL";	253	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_tc_neigh.c+144−0

@@ -0,0 +1,144 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	#include <stdint.h>
	3	#include <stdbool.h>
	4
	5	#include <linux/bpf.h>
	6	#include <linux/stddef.h>
	7	#include <linux/pkt_cls.h>
	8	#include <linux/if_ether.h>
	9	#include <linux/in.h>
	10	#include <linux/ip.h>
	11	#include <linux/ipv6.h>
	12
	13	#include <bpf/bpf_helpers.h>
	14	#include <bpf/bpf_endian.h>
	15
	16	#ifndef barrier_data
	17	# define barrier_data(ptr) asm volatile("": :"r"(ptr) :"memory")
	18	#endif
	19
	20	#ifndef ctx_ptr
	21	# define ctx_ptr(field) (void *)(long)(field)
	22	#endif
	23
	24	#define dst_to_src_tmp 0xeeddddeeU
	25	#define src_to_dst_tmp 0xeeffffeeU
	26
	27	#define ip4_src 0xac100164 /* 172.16.1.100 */
	28	#define ip4_dst 0xac100264 /* 172.16.2.100 */
	29
	30	#define ip6_src { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
	31	0x00, 0x01, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe }
	32	#define ip6_dst { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
	33	0x00, 0x02, 0xde, 0xad, 0xbe, 0xef, 0xca, 0xfe }
	34
	35	#ifndef v6_equal
	36	# define v6_equal(a, b) (a.s6_addr32[0] == b.s6_addr32[0] && \
	37	a.s6_addr32[1] == b.s6_addr32[1] && \
	38	a.s6_addr32[2] == b.s6_addr32[2] && \
	39	a.s6_addr32[3] == b.s6_addr32[3])
	40	#endif
	41
	42	static __always_inline bool is_remote_ep_v4(struct __sk_buff *skb,
	43	__be32 addr)
	44	{
	45	void *data_end = ctx_ptr(skb->data_end);
	46	void *data = ctx_ptr(skb->data);
	47	struct iphdr *ip4h;
	48
	49	if (data + sizeof(struct ethhdr) > data_end)
	50	return false;
	51
	52	ip4h = (struct iphdr *)(data + sizeof(struct ethhdr));
	53	if ((void *)(ip4h + 1) > data_end)
	54	return false;
	55
	56	return ip4h->daddr == addr;
	57	}
	58
	59	static __always_inline bool is_remote_ep_v6(struct __sk_buff *skb,
	60	struct in6_addr addr)
	61	{
	62	void *data_end = ctx_ptr(skb->data_end);
	63	void *data = ctx_ptr(skb->data);
	64	struct ipv6hdr *ip6h;
	65
	66	if (data + sizeof(struct ethhdr) > data_end)
	67	return false;
	68
	69	ip6h = (struct ipv6hdr *)(data + sizeof(struct ethhdr));
	70	if ((void *)(ip6h + 1) > data_end)
	71	return false;
	72
	73	return v6_equal(ip6h->daddr, addr);
	74	}
	75
	76	SEC("chk_neigh") int tc_chk(struct __sk_buff *skb)
	77	{
	78	void *data_end = ctx_ptr(skb->data_end);
	79	void *data = ctx_ptr(skb->data);
	80	__u32 *raw = data;
	81
	82	if (data + sizeof(struct ethhdr) > data_end)
	83	return TC_ACT_SHOT;
	84
	85	return !raw[0] && !raw[1] && !raw[2] ? TC_ACT_SHOT : TC_ACT_OK;
	86	}
	87
	88	SEC("dst_ingress") int tc_dst(struct __sk_buff *skb)
	89	{
	90	int idx = dst_to_src_tmp;
	91	__u8 zero[ETH_ALEN * 2];
	92	bool redirect = false;
	93
	94	switch (skb->protocol) {
	95	case __bpf_constant_htons(ETH_P_IP):
	96	redirect = is_remote_ep_v4(skb, __bpf_constant_htonl(ip4_src));
	97	break;
	98	case __bpf_constant_htons(ETH_P_IPV6):
	99	redirect = is_remote_ep_v6(skb, (struct in6_addr)ip6_src);
	100	break;
	101	}
	102
	103	if (!redirect)
	104	return TC_ACT_OK;
	105
	106	barrier_data(&idx);
	107	idx = bpf_ntohl(idx);
	108
	109	__builtin_memset(&zero, 0, sizeof(zero));
	110	if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0)
	111	return TC_ACT_SHOT;
	112
	113	return bpf_redirect_neigh(idx, 0);
	114	}
	115
	116	SEC("src_ingress") int tc_src(struct __sk_buff *skb)
	117	{
	118	int idx = src_to_dst_tmp;
	119	__u8 zero[ETH_ALEN * 2];
	120	bool redirect = false;
	121
	122	switch (skb->protocol) {
	123	case __bpf_constant_htons(ETH_P_IP):
	124	redirect = is_remote_ep_v4(skb, __bpf_constant_htonl(ip4_dst));
	125	break;
	126	case __bpf_constant_htons(ETH_P_IPV6):
	127	redirect = is_remote_ep_v6(skb, (struct in6_addr)ip6_dst);
	128	break;
	129	}
	130
	131	if (!redirect)
	132	return TC_ACT_OK;
	133
	134	barrier_data(&idx);
	135	idx = bpf_ntohl(idx);
	136
	137	__builtin_memset(&zero, 0, sizeof(zero));
	138	if (bpf_skb_store_bytes(skb, 0, &zero, sizeof(zero), 0) < 0)
	139	return TC_ACT_SHOT;
	140
	141	return bpf_redirect_neigh(idx, 0);
	142	}
	143
	144	char __license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_trace_ext.c+18−0

@@ -0,0 +1,18 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	// Copyright (c) 2019 Facebook
	3	#include <linux/bpf.h>
	4	#include <stdbool.h>
	5	#include <bpf/bpf_helpers.h>
	6	#include <bpf/bpf_endian.h>
	7	#include <bpf/bpf_tracing.h>
	8
	9	__u64 ext_called = 0;
	10
	11	SEC("freplace/test_pkt_md_access")
	12	int test_pkt_md_access_new(struct __sk_buff *skb)
	13	{
	14	ext_called = skb->len;
	15	return 0;
	16	}
	17
	18	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_trace_ext_tracing.c+25−0

@@ -0,0 +1,25 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2
	3	#include "vmlinux.h"
	4	#include <bpf/bpf_helpers.h>
	5	#include <bpf/bpf_tracing.h>
	6
	7	__u64 fentry_called = 0;
	8
	9	SEC("fentry/test_pkt_md_access_new")
	10	int BPF_PROG(fentry, struct sk_buff *skb)
	11	{
	12	fentry_called = skb->len;
	13	return 0;
	14	}
	15
	16	__u64 fexit_called = 0;
	17
	18	SEC("fexit/test_pkt_md_access_new")
	19	int BPF_PROG(fexit, struct sk_buff *skb)
	20	{
	21	fexit_called = skb->len;
	22	return 0;
	23	}
	24
	25	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/test_progs.h+63−0

@@ -130,6 +130,69 @@ extern int test__join_cgroup(const char *path); · 129 unmodified lines
130	#define CHECK_ATTR(condition, tag, format...) \	130	#define CHECK_ATTR(condition, tag, format...) \
131	_CHECK(condition, tag, tattr.duration, format)	131	_CHECK(condition, tag, tattr.duration, format)
132		132
		133	#define ASSERT_EQ(actual, expected, name) ({ \
		134	static int duration = 0; \
		135	typeof(actual) ___act = (actual); \
		136	typeof(expected) ___exp = (expected); \
		137	bool ___ok = ___act == ___exp; \
		138	CHECK(!___ok, (name), \
		139	"unexpected %s: actual %lld != expected %lld\n", \
		140	(name), (long long)(___act), (long long)(___exp)); \
		141	___ok; \
		142	})
		143
		144	#define ASSERT_STREQ(actual, expected, name) ({ \
		145	static int duration = 0; \
		146	const char *___act = actual; \
		147	const char *___exp = expected; \
		148	bool ___ok = strcmp(___act, ___exp) == 0; \
		149	CHECK(!___ok, (name), \
		150	"unexpected %s: actual '%s' != expected '%s'\n", \
		151	(name), ___act, ___exp); \
		152	___ok; \
		153	})
		154
		155	#define ASSERT_OK(res, name) ({ \
		156	static int duration = 0; \
		157	long long ___res = (res); \
		158	bool ___ok = ___res == 0; \
		159	CHECK(!___ok, (name), "unexpected error: %lld\n", ___res); \
		160	___ok; \
		161	})
		162
		163	#define ASSERT_ERR(res, name) ({ \
		164	static int duration = 0; \
		165	long long ___res = (res); \
		166	bool ___ok = ___res < 0; \
		167	CHECK(!___ok, (name), "unexpected success: %lld\n", ___res); \
		168	___ok; \
		169	})
		170
		171	#define ASSERT_NULL(ptr, name) ({ \
		172	static int duration = 0; \
		173	const void *___res = (ptr); \
		174	bool ___ok = !___res; \
		175	CHECK(!___ok, (name), "unexpected pointer: %p\n", ___res); \
		176	___ok; \
		177	})
		178
		179	#define ASSERT_OK_PTR(ptr, name) ({ \
		180	static int duration = 0; \
		181	const void *___res = (ptr); \
		182	bool ___ok = !IS_ERR_OR_NULL(___res); \
		183	CHECK(!___ok, (name), \
		184	"unexpected error: %ld\n", PTR_ERR(___res)); \
		185	___ok; \
		186	})
		187
		188	#define ASSERT_ERR_PTR(ptr, name) ({ \
		189	static int duration = 0; \
		190	const void *___res = (ptr); \
		191	bool ___ok = IS_ERR(___res) \
		192	CHECK(!___ok, (name), "unexpected pointer: %p\n", ___res); \
		193	___ok; \
		194	})
		195
133	static inline __u64 ptr_to_u64(const void *ptr)	196	static inline __u64 ptr_to_u64(const void *ptr)
134	{	197	{
135	return (__u64) (unsigned long) ptr;	198	return (__u64) (unsigned long) ptr;

tools/testing/selftests/bpf/test_sock_fields.c+0−482

@@ -1,482 +0,0 @@
1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (c) 2019 Facebook */
3
4	#include <sys/socket.h>
5	#include <sys/epoll.h>
6	#include <netinet/in.h>
7	#include <arpa/inet.h>
8	#include <unistd.h>
9	#include <stdlib.h>
10	#include <string.h>
11	#include <errno.h>
12
13	#include <bpf/bpf.h>
14	#include <bpf/libbpf.h>
15
16	#include "cgroup_helpers.h"
17	#include "bpf_rlimit.h"
18
19	enum bpf_addr_array_idx {
20	ADDR_SRV_IDX,
21	ADDR_CLI_IDX,
22	__NR_BPF_ADDR_ARRAY_IDX,
23	};
24
25	enum bpf_result_array_idx {
26	EGRESS_SRV_IDX,
27	EGRESS_CLI_IDX,
28	INGRESS_LISTEN_IDX,
29	__NR_BPF_RESULT_ARRAY_IDX,
30	};
31
32	enum bpf_linum_array_idx {
33	EGRESS_LINUM_IDX,
34	INGRESS_LINUM_IDX,
35	__NR_BPF_LINUM_ARRAY_IDX,
36	};
37
38	struct bpf_spinlock_cnt {
39	struct bpf_spin_lock lock;
40	__u32 cnt;
41	};
42
43	#define CHECK(condition, tag, format...) ({ \
44	int __ret = !!(condition); \
45	if (__ret) { \
46	printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag); \
47	printf(format); \
48	printf("\n"); \
49	exit(-1); \
50	} \
51	})
52
53	#define TEST_CGROUP "/test-bpf-sock-fields"
54	#define DATA "Hello BPF!"
55	#define DATA_LEN sizeof(DATA)
56
57	static struct sockaddr_in6 srv_sa6, cli_sa6;
58	static int sk_pkt_out_cnt10_fd;
59	static int sk_pkt_out_cnt_fd;
60	static int linum_map_fd;
61	static int addr_map_fd;
62	static int tp_map_fd;
63	static int sk_map_fd;
64
65	static __u32 addr_srv_idx = ADDR_SRV_IDX;
66	static __u32 addr_cli_idx = ADDR_CLI_IDX;
67
68	static __u32 egress_srv_idx = EGRESS_SRV_IDX;
69	static __u32 egress_cli_idx = EGRESS_CLI_IDX;
70	static __u32 ingress_listen_idx = INGRESS_LISTEN_IDX;
71
72	static __u32 egress_linum_idx = EGRESS_LINUM_IDX;
73	static __u32 ingress_linum_idx = INGRESS_LINUM_IDX;
74
75	static void init_loopback6(struct sockaddr_in6 *sa6)
76	{
77	memset(sa6, 0, sizeof(*sa6));
78	sa6->sin6_family = AF_INET6;
79	sa6->sin6_addr = in6addr_loopback;
80	}
81
82	static void print_sk(const struct bpf_sock *sk)
83	{
84	char src_ip4[24], dst_ip4[24];
85	char src_ip6[64], dst_ip6[64];
86
87	inet_ntop(AF_INET, &sk->src_ip4, src_ip4, sizeof(src_ip4));
88	inet_ntop(AF_INET6, &sk->src_ip6, src_ip6, sizeof(src_ip6));
89	inet_ntop(AF_INET, &sk->dst_ip4, dst_ip4, sizeof(dst_ip4));
90	inet_ntop(AF_INET6, &sk->dst_ip6, dst_ip6, sizeof(dst_ip6));
91
92	printf("state:%u bound_dev_if:%u family:%u type:%u protocol:%u mark:%u priority:%u "
93	"src_ip4:%x(%s) src_ip6:%x:%x:%x:%x(%s) src_port:%u "
94	"dst_ip4:%x(%s) dst_ip6:%x:%x:%x:%x(%s) dst_port:%u\n",
95	sk->state, sk->bound_dev_if, sk->family, sk->type, sk->protocol,
96	sk->mark, sk->priority,
97	sk->src_ip4, src_ip4,
98	sk->src_ip6[0], sk->src_ip6[1], sk->src_ip6[2], sk->src_ip6[3],
99	src_ip6, sk->src_port,
100	sk->dst_ip4, dst_ip4,
101	sk->dst_ip6[0], sk->dst_ip6[1], sk->dst_ip6[2], sk->dst_ip6[3],
102	dst_ip6, ntohs(sk->dst_port));
103	}
104
105	static void print_tp(const struct bpf_tcp_sock *tp)
106	{
107	printf("snd_cwnd:%u srtt_us:%u rtt_min:%u snd_ssthresh:%u rcv_nxt:%u "
108	"snd_nxt:%u snd:una:%u mss_cache:%u ecn_flags:%u "
109	"rate_delivered:%u rate_interval_us:%u packets_out:%u "
110	"retrans_out:%u total_retrans:%u segs_in:%u data_segs_in:%u "
111	"segs_out:%u data_segs_out:%u lost_out:%u sacked_out:%u "
112	"bytes_received:%llu bytes_acked:%llu\n",
113	tp->snd_cwnd, tp->srtt_us, tp->rtt_min, tp->snd_ssthresh,
114	tp->rcv_nxt, tp->snd_nxt, tp->snd_una, tp->mss_cache,
115	tp->ecn_flags, tp->rate_delivered, tp->rate_interval_us,
116	tp->packets_out, tp->retrans_out, tp->total_retrans,
117	tp->segs_in, tp->data_segs_in, tp->segs_out,
118	tp->data_segs_out, tp->lost_out, tp->sacked_out,
119	tp->bytes_received, tp->bytes_acked);
120	}
121
122	static void check_result(void)
123	{
124	struct bpf_tcp_sock srv_tp, cli_tp, listen_tp;
125	struct bpf_sock srv_sk, cli_sk, listen_sk;
126	__u32 ingress_linum, egress_linum;
127	int err;
128
129	err = bpf_map_lookup_elem(linum_map_fd, &egress_linum_idx,
130	&egress_linum);
131	CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)",
132	"err:%d errno:%d", err, errno);
133
134	err = bpf_map_lookup_elem(linum_map_fd, &ingress_linum_idx,
135	&ingress_linum);
136	CHECK(err == -1, "bpf_map_lookup_elem(linum_map_fd)",
137	"err:%d errno:%d", err, errno);
138
139	err = bpf_map_lookup_elem(sk_map_fd, &egress_srv_idx, &srv_sk);
140	CHECK(err == -1, "bpf_map_lookup_elem(sk_map_fd, &egress_srv_idx)",
141	"err:%d errno:%d", err, errno);
142	err = bpf_map_lookup_elem(tp_map_fd, &egress_srv_idx, &srv_tp);
143	CHECK(err == -1, "bpf_map_lookup_elem(tp_map_fd, &egress_srv_idx)",
144	"err:%d errno:%d", err, errno);
145
146	err = bpf_map_lookup_elem(sk_map_fd, &egress_cli_idx, &cli_sk);
147	CHECK(err == -1, "bpf_map_lookup_elem(sk_map_fd, &egress_cli_idx)",
148	"err:%d errno:%d", err, errno);
149	err = bpf_map_lookup_elem(tp_map_fd, &egress_cli_idx, &cli_tp);
150	CHECK(err == -1, "bpf_map_lookup_elem(tp_map_fd, &egress_cli_idx)",
151	"err:%d errno:%d", err, errno);
152
153	err = bpf_map_lookup_elem(sk_map_fd, &ingress_listen_idx, &listen_sk);
154	CHECK(err == -1, "bpf_map_lookup_elem(sk_map_fd, &ingress_listen_idx)",
155	"err:%d errno:%d", err, errno);
156	err = bpf_map_lookup_elem(tp_map_fd, &ingress_listen_idx, &listen_tp);
157	CHECK(err == -1, "bpf_map_lookup_elem(tp_map_fd, &ingress_listen_idx)",
158	"err:%d errno:%d", err, errno);
159
160	printf("listen_sk: ");
161	print_sk(&listen_sk);
162	printf("\n");
163
164	printf("srv_sk: ");
165	print_sk(&srv_sk);
166	printf("\n");
167
168	printf("cli_sk: ");
169	print_sk(&cli_sk);
170	printf("\n");
171
172	printf("listen_tp: ");
173	print_tp(&listen_tp);
174	printf("\n");
175
176	printf("srv_tp: ");
177	print_tp(&srv_tp);
178	printf("\n");
179
180	printf("cli_tp: ");
181	print_tp(&cli_tp);
182	printf("\n");
183
184	CHECK(listen_sk.state != 10 \|\|
185	listen_sk.family != AF_INET6 \|\|
186	listen_sk.protocol != IPPROTO_TCP \|\|
187	memcmp(listen_sk.src_ip6, &in6addr_loopback,
188	sizeof(listen_sk.src_ip6)) \|\|
189	listen_sk.dst_ip6[0] \|\| listen_sk.dst_ip6[1] \|\|
190	listen_sk.dst_ip6[2] \|\| listen_sk.dst_ip6[3] \|\|
191	listen_sk.src_port != ntohs(srv_sa6.sin6_port) \|\|
192	listen_sk.dst_port,
193	"Unexpected listen_sk",
194	"Check listen_sk output. ingress_linum:%u",
195	ingress_linum);
196
197	CHECK(srv_sk.state == 10 \|\|
198	!srv_sk.state \|\|
199	srv_sk.family != AF_INET6 \|\|
200	srv_sk.protocol != IPPROTO_TCP \|\|
201	memcmp(srv_sk.src_ip6, &in6addr_loopback,
202	sizeof(srv_sk.src_ip6)) \|\|
203	memcmp(srv_sk.dst_ip6, &in6addr_loopback,
204	sizeof(srv_sk.dst_ip6)) \|\|
205	srv_sk.src_port != ntohs(srv_sa6.sin6_port) \|\|
206	srv_sk.dst_port != cli_sa6.sin6_port,
207	"Unexpected srv_sk", "Check srv_sk output. egress_linum:%u",
208	egress_linum);
209
210	CHECK(cli_sk.state == 10 \|\|
211	!cli_sk.state \|\|
212	cli_sk.family != AF_INET6 \|\|
213	cli_sk.protocol != IPPROTO_TCP \|\|
214	memcmp(cli_sk.src_ip6, &in6addr_loopback,
215	sizeof(cli_sk.src_ip6)) \|\|
216	memcmp(cli_sk.dst_ip6, &in6addr_loopback,
217	sizeof(cli_sk.dst_ip6)) \|\|
218	cli_sk.src_port != ntohs(cli_sa6.sin6_port) \|\|
219	cli_sk.dst_port != srv_sa6.sin6_port,
220	"Unexpected cli_sk", "Check cli_sk output. egress_linum:%u",
221	egress_linum);
222
223	CHECK(listen_tp.data_segs_out \|\|
224	listen_tp.data_segs_in \|\|
225	listen_tp.total_retrans \|\|
226	listen_tp.bytes_acked,
227	"Unexpected listen_tp", "Check listen_tp output. ingress_linum:%u",
228	ingress_linum);
229
230	CHECK(srv_tp.data_segs_out != 2 \|\|
231	srv_tp.data_segs_in \|\|
232	srv_tp.snd_cwnd != 10 \|\|
233	srv_tp.total_retrans \|\|
234	srv_tp.bytes_acked != 2 * DATA_LEN,
235	"Unexpected srv_tp", "Check srv_tp output. egress_linum:%u",
236	egress_linum);
237
238	CHECK(cli_tp.data_segs_out \|\|
239	cli_tp.data_segs_in != 2 \|\|
240	cli_tp.snd_cwnd != 10 \|\|
241	cli_tp.total_retrans \|\|
242	cli_tp.bytes_received != 2 * DATA_LEN,
243	"Unexpected cli_tp", "Check cli_tp output. egress_linum:%u",
244	egress_linum);
245	}
246
247	static void check_sk_pkt_out_cnt(int accept_fd, int cli_fd)
248	{
249	struct bpf_spinlock_cnt pkt_out_cnt = {}, pkt_out_cnt10 = {};
250	int err;
251
252	pkt_out_cnt.cnt = ~0;
253	pkt_out_cnt10.cnt = ~0;
254	err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &accept_fd, &pkt_out_cnt);
255	if (!err)
256	err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &accept_fd,
257	&pkt_out_cnt10);
258
259	/* The bpf prog only counts for fullsock and
260	* passive conneciton did not become fullsock until 3WHS
261	* had been finished.
262	* The bpf prog only counted two data packet out but we
263	* specially init accept_fd's pkt_out_cnt by 2 in
264	* init_sk_storage(). Hence, 4 here.
265	*/
266	CHECK(err \|\| pkt_out_cnt.cnt != 4 \|\| pkt_out_cnt10.cnt != 40,
267	"bpf_map_lookup_elem(sk_pkt_out_cnt, &accept_fd)",
268	"err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u",
269	err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt);
270
271	pkt_out_cnt.cnt = ~0;
272	pkt_out_cnt10.cnt = ~0;
273	err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &cli_fd, &pkt_out_cnt);
274	if (!err)
275	err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &cli_fd,
276	&pkt_out_cnt10);
277	/* Active connection is fullsock from the beginning.
278	* 1 SYN and 1 ACK during 3WHS
279	* 2 Acks on data packet.
280	*
281	* The bpf_prog initialized it to 0xeB9F.
282	*/
283	CHECK(err \|\| pkt_out_cnt.cnt != 0xeB9F + 4 \|\|
284	pkt_out_cnt10.cnt != 0xeB9F + 40,
285	"bpf_map_lookup_elem(sk_pkt_out_cnt, &cli_fd)",
286	"err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u",
287	err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt);
288	}
289
290	static void init_sk_storage(int sk_fd, __u32 pkt_out_cnt)
291	{
292	struct bpf_spinlock_cnt scnt = {};
293	int err;
294
295	scnt.cnt = pkt_out_cnt;
296	err = bpf_map_update_elem(sk_pkt_out_cnt_fd, &sk_fd, &scnt,
297	BPF_NOEXIST);
298	CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt_fd)",
299	"err:%d errno:%d", err, errno);
300
301	scnt.cnt *= 10;
302	err = bpf_map_update_elem(sk_pkt_out_cnt10_fd, &sk_fd, &scnt,
303	BPF_NOEXIST);
304	CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt10_fd)",
305	"err:%d errno:%d", err, errno);
306	}
307
308	static void test(void)
309	{
310	int listen_fd, cli_fd, accept_fd, epfd, err;
311	struct epoll_event ev;
312	socklen_t addrlen;
313	int i;
314
315	addrlen = sizeof(struct sockaddr_in6);
316	ev.events = EPOLLIN;
317
318	epfd = epoll_create(1);
319	CHECK(epfd == -1, "epoll_create()", "epfd:%d errno:%d", epfd, errno);
320
321	/* Prepare listen_fd */
322	listen_fd = socket(AF_INET6, SOCK_STREAM \| SOCK_NONBLOCK, 0);
323	CHECK(listen_fd == -1, "socket()", "listen_fd:%d errno:%d",
324	listen_fd, errno);
325
326	init_loopback6(&srv_sa6);
327	err = bind(listen_fd, (struct sockaddr *)&srv_sa6, sizeof(srv_sa6));
328	CHECK(err, "bind(listen_fd)", "err:%d errno:%d", err, errno);
329
330	err = getsockname(listen_fd, (struct sockaddr *)&srv_sa6, &addrlen);
331	CHECK(err, "getsockname(listen_fd)", "err:%d errno:%d", err, errno);
332
333	err = listen(listen_fd, 1);
334	CHECK(err, "listen(listen_fd)", "err:%d errno:%d", err, errno);
335
336	/* Prepare cli_fd */
337	cli_fd = socket(AF_INET6, SOCK_STREAM \| SOCK_NONBLOCK, 0);
338	CHECK(cli_fd == -1, "socket()", "cli_fd:%d errno:%d", cli_fd, errno);
339
340	init_loopback6(&cli_sa6);
341	err = bind(cli_fd, (struct sockaddr *)&cli_sa6, sizeof(cli_sa6));
342	CHECK(err, "bind(cli_fd)", "err:%d errno:%d", err, errno);
343
344	err = getsockname(cli_fd, (struct sockaddr *)&cli_sa6, &addrlen);
345	CHECK(err, "getsockname(cli_fd)", "err:%d errno:%d",
346	err, errno);
347
348	/* Update addr_map with srv_sa6 and cli_sa6 */
349	err = bpf_map_update_elem(addr_map_fd, &addr_srv_idx, &srv_sa6, 0);
350	CHECK(err, "map_update", "err:%d errno:%d", err, errno);
351
352	err = bpf_map_update_elem(addr_map_fd, &addr_cli_idx, &cli_sa6, 0);
353	CHECK(err, "map_update", "err:%d errno:%d", err, errno);
354
355	/* Connect from cli_sa6 to srv_sa6 */
356	err = connect(cli_fd, (struct sockaddr *)&srv_sa6, addrlen);
357	printf("srv_sa6.sin6_port:%u cli_sa6.sin6_port:%u\n\n",
358	ntohs(srv_sa6.sin6_port), ntohs(cli_sa6.sin6_port));
359	CHECK(err && errno != EINPROGRESS,
360	"connect(cli_fd)", "err:%d errno:%d", err, errno);
361
362	ev.data.fd = listen_fd;
363	err = epoll_ctl(epfd, EPOLL_CTL_ADD, listen_fd, &ev);
364	CHECK(err, "epoll_ctl(EPOLL_CTL_ADD, listen_fd)", "err:%d errno:%d",
365	err, errno);
366
367	/* Accept the connection */
368	/* Have some timeout in accept(listen_fd). Just in case. */
369	err = epoll_wait(epfd, &ev, 1, 1000);
370	CHECK(err != 1 \|\| ev.data.fd != listen_fd,
371	"epoll_wait(listen_fd)",
372	"err:%d errno:%d ev.data.fd:%d listen_fd:%d",
373	err, errno, ev.data.fd, listen_fd);
374
375	accept_fd = accept(listen_fd, NULL, NULL);
376	CHECK(accept_fd == -1, "accept(listen_fd)", "accept_fd:%d errno:%d",
377	accept_fd, errno);
378	close(listen_fd);
379
380	ev.data.fd = cli_fd;
381	err = epoll_ctl(epfd, EPOLL_CTL_ADD, cli_fd, &ev);
382	CHECK(err, "epoll_ctl(EPOLL_CTL_ADD, cli_fd)", "err:%d errno:%d",
383	err, errno);
384
385	init_sk_storage(accept_fd, 2);
386
387	for (i = 0; i < 2; i++) {
388	/* Send some data from accept_fd to cli_fd */
389	err = send(accept_fd, DATA, DATA_LEN, 0);
390	CHECK(err != DATA_LEN, "send(accept_fd)", "err:%d errno:%d",
391	err, errno);
392
393	/* Have some timeout in recv(cli_fd). Just in case. */
394	err = epoll_wait(epfd, &ev, 1, 1000);
395	CHECK(err != 1 \|\| ev.data.fd != cli_fd,
396	"epoll_wait(cli_fd)", "err:%d errno:%d ev.data.fd:%d cli_fd:%d",
397	err, errno, ev.data.fd, cli_fd);
398
399	err = recv(cli_fd, NULL, 0, MSG_TRUNC);
400	CHECK(err, "recv(cli_fd)", "err:%d errno:%d", err, errno);
401	}
402
403	check_sk_pkt_out_cnt(accept_fd, cli_fd);
404
405	close(epfd);
406	close(accept_fd);
407	close(cli_fd);
408
409	check_result();
410	}
411
412	int main(int argc, char **argv)
413	{
414	struct bpf_prog_load_attr attr = {
415	.file = "test_sock_fields_kern.o",
416	.prog_type = BPF_PROG_TYPE_CGROUP_SKB,
417	.prog_flags = BPF_F_TEST_RND_HI32,
418	};
419	int cgroup_fd, egress_fd, ingress_fd, err;
420	struct bpf_program *ingress_prog;
421	struct bpf_object *obj;
422	struct bpf_map *map;
423
424	/* Create a cgroup, get fd, and join it */
425	cgroup_fd = cgroup_setup_and_join(TEST_CGROUP);
426	CHECK(cgroup_fd < 0, "cgroup_setup_and_join()",
427	"cgroup_fd:%d errno:%d", cgroup_fd, errno);
428	atexit(cleanup_cgroup_environment);
429
430	err = bpf_prog_load_xattr(&attr, &obj, &egress_fd);
431	CHECK(err, "bpf_prog_load_xattr()", "err:%d", err);
432
433	ingress_prog = bpf_object__find_program_by_title(obj,
434	"cgroup_skb/ingress");
435	CHECK(!ingress_prog,
436	"bpf_object__find_program_by_title(cgroup_skb/ingress)",
437	"not found");
438	ingress_fd = bpf_program__fd(ingress_prog);
439
440	err = bpf_prog_attach(egress_fd, cgroup_fd, BPF_CGROUP_INET_EGRESS, 0);
441	CHECK(err == -1, "bpf_prog_attach(CPF_CGROUP_INET_EGRESS)",
442	"err:%d errno%d", err, errno);
443
444	err = bpf_prog_attach(ingress_fd, cgroup_fd,
445	BPF_CGROUP_INET_INGRESS, 0);
446	CHECK(err == -1, "bpf_prog_attach(CPF_CGROUP_INET_INGRESS)",
447	"err:%d errno%d", err, errno);
448	close(cgroup_fd);
449
450	map = bpf_object__find_map_by_name(obj, "addr_map");
451	CHECK(!map, "cannot find addr_map", "(null)");
452	addr_map_fd = bpf_map__fd(map);
453
454	map = bpf_object__find_map_by_name(obj, "sock_result_map");
455	CHECK(!map, "cannot find sock_result_map", "(null)");
456	sk_map_fd = bpf_map__fd(map);
457
458	map = bpf_object__find_map_by_name(obj, "tcp_sock_result_map");
459	CHECK(!map, "cannot find tcp_sock_result_map", "(null)");
460	tp_map_fd = bpf_map__fd(map);
461
462	map = bpf_object__find_map_by_name(obj, "linum_map");
463	CHECK(!map, "cannot find linum_map", "(null)");
464	linum_map_fd = bpf_map__fd(map);
465
466	map = bpf_object__find_map_by_name(obj, "sk_pkt_out_cnt");
467	CHECK(!map, "cannot find sk_pkt_out_cnt", "(null)");
468	sk_pkt_out_cnt_fd = bpf_map__fd(map);
469
470	map = bpf_object__find_map_by_name(obj, "sk_pkt_out_cnt10");
471	CHECK(!map, "cannot find sk_pkt_out_cnt10", "(null)");
472	sk_pkt_out_cnt10_fd = bpf_map__fd(map);
473
474	test();
475
476	bpf_object__close(obj);
477	cleanup_cgroup_environment();
478
479	printf("PASS\n");
480
481	return 0;
482	}

tools/testing/selftests/bpf/test_tc_neigh.sh+168−0

@@ -0,0 +1,168 @@
	1	#!/bin/bash
	2	# SPDX-License-Identifier: GPL-2.0
	3	#
	4	# This test sets up 3 netns (src <-> fwd <-> dst). There is no direct veth link
	5	# between src and dst. The netns fwd has veth links to each src and dst. The
	6	# client is in src and server in dst. The test installs a TC BPF program to each
	7	# host facing veth in fwd which calls into bpf_redirect_peer() to perform the
	8	# neigh addr population and redirect; it also installs a dropper prog on the
	9	# egress side to drop skbs if neigh addrs were not populated.
	10
	11	if [[ $EUID -ne 0 ]]; then
	12	echo "This script must be run as root"
	13	echo "FAIL"
	14	exit 1
	15	fi
	16
	17	# check that nc, dd, ping, ping6 and timeout are present
	18	command -v nc >/dev/null 2>&1 \|\| \
	19	{ echo >&2 "nc is not available"; exit 1; }
	20	command -v dd >/dev/null 2>&1 \|\| \
	21	{ echo >&2 "dd is not available"; exit 1; }
	22	command -v timeout >/dev/null 2>&1 \|\| \
	23	{ echo >&2 "timeout is not available"; exit 1; }
	24	command -v ping >/dev/null 2>&1 \|\| \
	25	{ echo >&2 "ping is not available"; exit 1; }
	26	command -v ping6 >/dev/null 2>&1 \|\| \
	27	{ echo >&2 "ping6 is not available"; exit 1; }
	28
	29	readonly GREEN='\033[0;92m'
	30	readonly RED='\033[0;31m'
	31	readonly NC='\033[0m' # No Color
	32
	33	readonly PING_ARG="-c 3 -w 10 -q"
	34
	35	readonly TIMEOUT=10
	36
	37	readonly NS_SRC="ns-src-$(mktemp -u XXXXXX)"
	38	readonly NS_FWD="ns-fwd-$(mktemp -u XXXXXX)"
	39	readonly NS_DST="ns-dst-$(mktemp -u XXXXXX)"
	40
	41	readonly IP4_SRC="172.16.1.100"
	42	readonly IP4_DST="172.16.2.100"
	43
	44	readonly IP6_SRC="::1:dead:beef:cafe"
	45	readonly IP6_DST="::2:dead:beef:cafe"
	46
	47	readonly IP4_SLL="169.254.0.1"
	48	readonly IP4_DLL="169.254.0.2"
	49	readonly IP4_NET="169.254.0.0"
	50
	51	cleanup()
	52	{
	53	ip netns del ${NS_SRC}
	54	ip netns del ${NS_FWD}
	55	ip netns del ${NS_DST}
	56	}
	57
	58	trap cleanup EXIT
	59
	60	set -e
	61
	62	ip netns add "${NS_SRC}"
	63	ip netns add "${NS_FWD}"
	64	ip netns add "${NS_DST}"
	65
	66	ip link add veth_src type veth peer name veth_src_fwd
	67	ip link add veth_dst type veth peer name veth_dst_fwd
	68
	69	ip link set veth_src netns ${NS_SRC}
	70	ip link set veth_src_fwd netns ${NS_FWD}
	71
	72	ip link set veth_dst netns ${NS_DST}
	73	ip link set veth_dst_fwd netns ${NS_FWD}
	74
	75	ip -netns ${NS_SRC} addr add ${IP4_SRC}/32 dev veth_src
	76	ip -netns ${NS_DST} addr add ${IP4_DST}/32 dev veth_dst
	77
	78	# The fwd netns automatically get a v6 LL address / routes, but also needs v4
	79	# one in order to start ARP probing. IP4_NET route is added to the endpoints
	80	# so that the ARP processing will reply.
	81
	82	ip -netns ${NS_FWD} addr add ${IP4_SLL}/32 dev veth_src_fwd
	83	ip -netns ${NS_FWD} addr add ${IP4_DLL}/32 dev veth_dst_fwd
	84
	85	ip -netns ${NS_SRC} addr add ${IP6_SRC}/128 dev veth_src nodad
	86	ip -netns ${NS_DST} addr add ${IP6_DST}/128 dev veth_dst nodad
	87
	88	ip -netns ${NS_SRC} link set dev veth_src up
	89	ip -netns ${NS_FWD} link set dev veth_src_fwd up
	90
	91	ip -netns ${NS_DST} link set dev veth_dst up
	92	ip -netns ${NS_FWD} link set dev veth_dst_fwd up
	93
	94	ip -netns ${NS_SRC} route add ${IP4_DST}/32 dev veth_src scope global
	95	ip -netns ${NS_SRC} route add ${IP4_NET}/16 dev veth_src scope global
	96	ip -netns ${NS_FWD} route add ${IP4_SRC}/32 dev veth_src_fwd scope global
	97
	98	ip -netns ${NS_SRC} route add ${IP6_DST}/128 dev veth_src scope global
	99	ip -netns ${NS_FWD} route add ${IP6_SRC}/128 dev veth_src_fwd scope global
	100
	101	ip -netns ${NS_DST} route add ${IP4_SRC}/32 dev veth_dst scope global
	102	ip -netns ${NS_DST} route add ${IP4_NET}/16 dev veth_dst scope global
	103	ip -netns ${NS_FWD} route add ${IP4_DST}/32 dev veth_dst_fwd scope global
	104
	105	ip -netns ${NS_DST} route add ${IP6_SRC}/128 dev veth_dst scope global
	106	ip -netns ${NS_FWD} route add ${IP6_DST}/128 dev veth_dst_fwd scope global
	107
	108	fmac_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/address)
	109	fmac_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/address)
	110
	111	ip -netns ${NS_SRC} neigh add ${IP4_DST} dev veth_src lladdr $fmac_src
	112	ip -netns ${NS_DST} neigh add ${IP4_SRC} dev veth_dst lladdr $fmac_dst
	113
	114	ip -netns ${NS_SRC} neigh add ${IP6_DST} dev veth_src lladdr $fmac_src
	115	ip -netns ${NS_DST} neigh add ${IP6_SRC} dev veth_dst lladdr $fmac_dst
	116
	117	veth_dst=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_dst_fwd/ifindex \| awk '{printf "%08x\n", $1}')
	118	veth_src=$(ip netns exec ${NS_FWD} cat /sys/class/net/veth_src_fwd/ifindex \| awk '{printf "%08x\n", $1}')
	119
	120	xxd -p < test_tc_neigh.o \| sed "s/eeddddee/$veth_src/g" \| xxd -r -p > test_tc_neigh.x.o
	121	xxd -p < test_tc_neigh.x.o \| sed "s/eeffffee/$veth_dst/g" \| xxd -r -p > test_tc_neigh.y.o
	122
	123	ip netns exec ${NS_FWD} tc qdisc add dev veth_src_fwd clsact
	124	ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd ingress bpf da obj test_tc_neigh.y.o sec src_ingress
	125	ip netns exec ${NS_FWD} tc filter add dev veth_src_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh
	126
	127	ip netns exec ${NS_FWD} tc qdisc add dev veth_dst_fwd clsact
	128	ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd ingress bpf da obj test_tc_neigh.y.o sec dst_ingress
	129	ip netns exec ${NS_FWD} tc filter add dev veth_dst_fwd egress bpf da obj test_tc_neigh.y.o sec chk_neigh
	130
	131	rm -f test_tc_neigh.x.o test_tc_neigh.y.o
	132
	133	ip netns exec ${NS_DST} bash -c "nc -4 -l -p 9004 &"
	134	ip netns exec ${NS_DST} bash -c "nc -6 -l -p 9006 &"
	135
	136	set +e
	137
	138	TEST="TCPv4 connectivity test"
	139	ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP4_DST}/9004"
	140	if [ $? -ne 0 ]; then
	141	echo -e "${TEST}: ${RED}FAIL${NC}"
	142	exit 1
	143	fi
	144	echo -e "${TEST}: ${GREEN}PASS${NC}"
	145
	146	TEST="TCPv6 connectivity test"
	147	ip netns exec ${NS_SRC} bash -c "timeout ${TIMEOUT} dd if=/dev/zero bs=1000 count=100 > /dev/tcp/${IP6_DST}/9006"
	148	if [ $? -ne 0 ]; then
	149	echo -e "${TEST}: ${RED}FAIL${NC}"
	150	exit 1
	151	fi
	152	echo -e "${TEST}: ${GREEN}PASS${NC}"
	153
	154	TEST="ICMPv4 connectivity test"
	155	ip netns exec ${NS_SRC} ping $PING_ARG ${IP4_DST}
	156	if [ $? -ne 0 ]; then
	157	echo -e "${TEST}: ${RED}FAIL${NC}"
	158	exit 1
	159	fi
	160	echo -e "${TEST}: ${GREEN}PASS${NC}"
	161
	162	TEST="ICMPv6 connectivity test"
	163	ip netns exec ${NS_SRC} ping6 $PING_ARG ${IP6_DST}
	164	if [ $? -ne 0 ]; then
	165	echo -e "${TEST}: ${RED}FAIL${NC}"
	166	exit 1
	167	fi
	168	echo -e "${TEST}: ${GREEN}PASS${NC}"

tools/testing/selftests/bpf/verifier/and.c+16−0

@@ -48,3 +48,19 @@ · 47 unmodified lines
48	.result = REJECT,	48	.result = REJECT,
49	.flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,	49	.flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS,
50	},	50	},
		51	{
		52	"check known subreg with unknown reg",
		53	.insns = {
		54	BPF_RAW_INSN(BPF_JMP \| BPF_CALL, 0, 0, 0, BPF_FUNC_get_prandom_u32),
		55	BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 32),
		56	BPF_ALU64_IMM(BPF_ADD, BPF_REG_0, 1),
		57	BPF_ALU64_IMM(BPF_AND, BPF_REG_0, 0xFFFF1234),
		58	/* Upper bits are unknown but AND above masks out 1 zero'ing lower bits */
		59	BPF_JMP32_IMM(BPF_JLT, BPF_REG_0, 1, 1),
		60	BPF_LDX_MEM(BPF_W, BPF_REG_1, BPF_REG_1, 512),
		61	BPF_MOV64_IMM(BPF_REG_0, 0),
		62	BPF_EXIT_INSN(),
		63	},
		64	.result = ACCEPT,
		65	.retval = 0
		66	},

tools/testing/selftests/bpf/verifier/ref_tracking.c+47−0

@@ -854,3 +854,50 @@ · 853 unmodified lines
854	.errstr = "Unreleased reference",	854	.errstr = "Unreleased reference",
855	.result = REJECT,	855	.result = REJECT,
856	},	856	},
		857	{
		858	"reference tracking: bpf_sk_release(btf_tcp_sock)",
		859	.insns = {
		860	BPF_SK_LOOKUP(sk_lookup_tcp),
		861	BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
		862	BPF_EXIT_INSN(),
		863	BPF_MOV64_REG(BPF_REG_6, BPF_REG_0),
		864	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
		865	BPF_EMIT_CALL(BPF_FUNC_skc_to_tcp_sock),
		866	BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3),
		867	BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
		868	BPF_EMIT_CALL(BPF_FUNC_sk_release),
		869	BPF_EXIT_INSN(),
		870	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
		871	BPF_EMIT_CALL(BPF_FUNC_sk_release),
		872	BPF_EXIT_INSN(),
		873	},
		874	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
		875	.result = ACCEPT,
		876	.result_unpriv = REJECT,
		877	.errstr_unpriv = "unknown func",
		878	},
		879	{
		880	"reference tracking: use ptr from bpf_skc_to_tcp_sock() after release",
		881	.insns = {
		882	BPF_SK_LOOKUP(sk_lookup_tcp),
		883	BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1),
		884	BPF_EXIT_INSN(),
		885	BPF_MOV64_REG(BPF_REG_6, BPF_REG_0),
		886	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
		887	BPF_EMIT_CALL(BPF_FUNC_skc_to_tcp_sock),
		888	BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 3),
		889	BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
		890	BPF_EMIT_CALL(BPF_FUNC_sk_release),
		891	BPF_EXIT_INSN(),
		892	BPF_MOV64_REG(BPF_REG_7, BPF_REG_0),
		893	BPF_MOV64_REG(BPF_REG_1, BPF_REG_6),
		894	BPF_EMIT_CALL(BPF_FUNC_sk_release),
		895	BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_7, 0),
		896	BPF_EXIT_INSN(),
		897	},
		898	.prog_type = BPF_PROG_TYPE_SCHED_CLS,
		899	.result = REJECT,
		900	.errstr = "invalid mem access",
		901	.result_unpriv = REJECT,
		902	.errstr_unpriv = "unknown func",
		903	},