lael-labs/linux

Documentation/bpf/btf.rst+37−6

     #define BTF_KIND_ARRAY          3       /* Array        */
     #define BTF_KIND_STRUCT         4       /* Struct       */
     #define BTF_KIND_UNION          5       /* Union        */
-    #define BTF_KIND_ENUM           6       /* Enumeration  */
+    #define BTF_KIND_ENUM           6       /* Enumeration up to 32-bit values */
     #define BTF_KIND_FWD            7       /* Forward      */
     #define BTF_KIND_TYPEDEF        8       /* Typedef      */
     #define BTF_KIND_VOLATILE       9       /* Volatile     */
     #define BTF_KIND_FLOAT          16      /* Floating point       */
     #define BTF_KIND_DECL_TAG       17      /* Decl Tag     */
     #define BTF_KIND_TYPE_TAG       18      /* Type Tag     */
+    #define BTF_KIND_ENUM64         19      /* Enumeration up to 64-bit values */
 Note that the type section encodes debug info, not just pure types.
 ``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram.
          * bits 24-28: kind (e.g. int, ptr, array...etc)
          * bits 29-30: unused
          * bit     31: kind_flag, currently used by
-         *             struct, union and fwd
+         *             struct, union, fwd, enum and enum64.
          */
         __u32 info;
-        /* "size" is used by INT, ENUM, STRUCT and UNION.
+        /* "size" is used by INT, ENUM, STRUCT, UNION and ENUM64.
          * "size" tells the size of the type it is describing.
          *
          * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
 ``struct btf_type`` encoding requirement:
   * ``name_off``: 0 or offset to a valid C identifier
-  * ``info.kind_flag``: 0
+  * ``info.kind_flag``: 0 for unsigned, 1 for signed
   * ``info.kind``: BTF_KIND_ENUM
   * ``info.vlen``: number of enum values
-  * ``size``: 4
+  * ``size``: 1/2/4/8
 ``btf_type`` is followed by ``info.vlen`` number of ``struct btf_enum``.::
   * ``name_off``: offset to a valid C identifier
   * ``val``: any value
+If the original enum value is signed and the size is less than 4,
+that value will be sign extended into 4 bytes. If the size is 8,
+the value will be truncated into 4 bytes.
 2.2.7 BTF_KIND_FWD
 ~~~~~~~~~~~~~~~~~~
 a ``func`` argument, and ``btf_decl_tag.component_idx`` should be a
 valid index (starting from 0) pointing to a member or an argument.
-2.2.17 BTF_KIND_TYPE_TAG
+2.2.18 BTF_KIND_TYPE_TAG
 ~~~~~~~~~~~~~~~~~~~~~~~~
 ``struct btf_type`` encoding requirement:
 and finally the base type. The base type is one of
 int, ptr, array, struct, union, enum, func_proto and float types.
+2.2.19 BTF_KIND_ENUM64
+~~~~~~~~~~~~~~~~~~~~~~
+``struct btf_type`` encoding requirement:
+  * ``name_off``: 0 or offset to a valid C identifier
+  * ``info.kind_flag``: 0 for unsigned, 1 for signed
+  * ``info.kind``: BTF_KIND_ENUM64
+  * ``info.vlen``: number of enum values
+  * ``size``: 1/2/4/8
+``btf_type`` is followed by ``info.vlen`` number of ``struct btf_enum64``.::
+    struct btf_enum64 {
+        __u32   name_off;
+        __u32   val_lo32;
+        __u32   val_hi32;
+    };
+The ``btf_enum64`` encoding:
+  * ``name_off``: offset to a valid C identifier
+  * ``val_lo32``: lower 32-bit value for a 64-bit value
+  * ``val_hi32``: high 32-bit value for a 64-bit value
+If the original enum value is signed and the size is less than 8,
+that value will be sign extended into 8 bytes.
 3. BTF Kernel API
 =================

Documentation/bpf/instruction-set.rst+1−1

@@ -127,7 +127,7 @@ BPF_XOR \| BPF_K \| BPF_ALU64 means:: · 126 unmodified lines
127	Byte swap instructions	127	Byte swap instructions
128	----------------------	128	----------------------
129		129
130	The byte swap instructions use an instruction class of ``BFP_ALU`` and a 4-bit	130	The byte swap instructions use an instruction class of ``BPF_ALU`` and a 4-bit
131	code field of ``BPF_END``.	131	code field of ``BPF_END``.
132		132
133	The byte swap instructions operate on the destination register	133	The byte swap instructions operate on the destination register

arch/arm/net/bpf_jit_32.c+0−16

@@ -712,22 +712,6 @@ static inline void emit_alu_r(const u8 dst, const u8 src, const bool is64, · 711 unmodified lines
712	}	712	}
713	}	713	}
714		714
715	/* ALU operation (32 bit)
716	* dst = dst (op) src
717	*/
718	static inline void emit_a32_alu_r(const s8 dst, const s8 src,
719	struct jit_ctx *ctx, const bool is64,
720	const bool hi, const u8 op) {
721	const s8 *tmp = bpf2a32[TMP_REG_1];
722	s8 rn, rd;
723
724	rn = arm_bpf_get_reg32(src, tmp[1], ctx);
725	rd = arm_bpf_get_reg32(dst, tmp[0], ctx);
726	/* ALU operation */
727	emit_alu_r(rd, rn, is64, hi, op, ctx);
728	arm_bpf_put_reg32(dst, rd, ctx);
729	}
730
731	/* ALU operation (64 bit) */	715	/* ALU operation (64 bit) */
732	static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],	716	static inline void emit_a32_alu_r64(const bool is64, const s8 dst[],
733	const s8 src[], struct jit_ctx *ctx,	717	const s8 src[], struct jit_ctx *ctx,

arch/riscv/net/bpf_jit.h+1−0

@@ -69,6 +69,7 @@ struct rv_jit_context { · 68 unmodified lines
69	struct bpf_prog *prog;	69	struct bpf_prog *prog;
70	u16 insns; / RV insns */	70	u16 insns; / RV insns */
71	int ninsns;	71	int ninsns;
		72	int body_len;
72	int epilogue_offset;	73	int epilogue_offset;
73	int offset; / BPF to RV */	74	int offset; / BPF to RV */
74	int nexentries;	75	int nexentries;

arch/riscv/net/bpf_jit_core.c+7−1

 	unsigned int prog_size = 0, extable_size = 0;
 	bool tmp_blinded = false, extra_pass = false;
 	struct bpf_prog *tmp, *orig_prog = prog;
-	int pass = 0, prev_ninsns = 0, i;
+	int pass = 0, prev_ninsns = 0, prologue_len, i;
 	struct rv_jit_data *jit_data;
 	struct rv_jit_context *ctx;
 			prog = orig_prog;
 			goto out_offset;
 		}
+		ctx->body_len = ctx->ninsns;
 		bpf_jit_build_prologue(ctx);
 		ctx->epilogue_offset = ctx->ninsns;
 		bpf_jit_build_epilogue(ctx);
 	if (!prog->is_func || extra_pass) {
 		bpf_jit_binary_lock_ro(jit_data->header);
+		prologue_len = ctx->epilogue_offset - ctx->body_len;
+		for (i = 0; i < prog->len; i++)
+			ctx->offset[i] = ninsns_rvoff(prologue_len +
+						      ctx->offset[i]);
+		bpf_prog_fill_jited_linfo(prog, ctx->offset);
 out_offset:
 		kfree(ctx->offset);
 		kfree(jit_data);

include/linux/bpf.h+102−3

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	#define _LINUX_BPF_H 1	5	#define _LINUX_BPF_H 1
6		6
7	#include <uapi/linux/bpf.h>	7	#include <uapi/linux/bpf.h>
		8	#include <uapi/linux/filter.h>
8		9
9	#include <linux/workqueue.h>	10	#include <linux/workqueue.h>
10	#include <linux/file.h>	11	#include <linux/file.h>
@@ -22,8 +23,10 @@ · 11 unmodified lines
22	#include <linux/sched/mm.h>	23	#include <linux/sched/mm.h>
23	#include <linux/slab.h>	24	#include <linux/slab.h>
24	#include <linux/percpu-refcount.h>	25	#include <linux/percpu-refcount.h>
		26	#include <linux/stddef.h>
25	#include <linux/bpfptr.h>	27	#include <linux/bpfptr.h>
26	#include <linux/btf.h>	28	#include <linux/btf.h>
		29	#include <linux/rcupdate_trace.h>
27		30
28	struct bpf_verifier_env;	31	struct bpf_verifier_env;
29	struct bpf_verifier_log;	32	struct bpf_verifier_log;
@@ -398,6 +401,9 @@ enum bpf_type_flag { · 368 unmodified lines
398	/* DYNPTR points to a ringbuf record. */	401	/* DYNPTR points to a ringbuf record. */
399	DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS),	402	DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS),
400		403
		404	/* Size is known at compile time. */
		405	MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS),
		406
401	__BPF_TYPE_FLAG_MAX,	407	__BPF_TYPE_FLAG_MAX,
402	__BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,	408	__BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1,
403	};	409	};
@@ -461,6 +467,8 @@ enum bpf_arg_type { · 57 unmodified lines
461	* all bytes or clear them in error case.	467	* all bytes or clear them in error case.
462	*/	468	*/
463	ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT \| ARG_PTR_TO_MEM,	469	ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT \| ARG_PTR_TO_MEM,
		470	/* Pointer to valid memory of size known at compile time. */
		471	ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE \| ARG_PTR_TO_MEM,
464		472
465	/* This must be the last entry. Its purpose is to ensure the enum is	473	/* This must be the last entry. Its purpose is to ensure the enum is
466	* wide enough to hold the higher bits reserved for bpf_type_flag.	474	* wide enough to hold the higher bits reserved for bpf_type_flag.
@@ -526,6 +534,14 @@ struct bpf_func_proto { · 59 unmodified lines
526	u32 *arg5_btf_id;	534	u32 *arg5_btf_id;
527	};	535	};
528	u32 *arg_btf_id[5];	536	u32 *arg_btf_id[5];
		537	struct {
		538	size_t arg1_size;
		539	size_t arg2_size;
		540	size_t arg3_size;
		541	size_t arg4_size;
		542	size_t arg5_size;
		543	};
		544	size_t arg_size[5];
529	};	545	};
530	int ret_btf_id; / return value btf_id */	546	int ret_btf_id; / return value btf_id */
531	bool (allowed)(const struct bpf_prog prog);	547	bool (allowed)(const struct bpf_prog prog);
@@ -1084,6 +1100,40 @@ struct bpf_prog_aux { · 552 unmodified lines
1084	};	1100	};
1085	};	1101	};
1086		1102
		1103	struct bpf_prog {
		1104	u16 pages; /* Number of allocated pages */
		1105	u16 jited:1, /* Is our filter JIT'ed? */
		1106	jit_requested:1,/* archs need to JIT the prog */
		1107	gpl_compatible:1, /* Is filter GPL compatible? */
		1108	cb_access:1, /* Is control block accessed? */
		1109	dst_needed:1, /* Do we need dst entry? */
		1110	blinding_requested:1, /* needs constant blinding */
		1111	blinded:1, /* Was blinded */
		1112	is_func:1, /* program is a bpf function */
		1113	kprobe_override:1, /* Do we override a kprobe? */
		1114	has_callchain_buf:1, /* callchain buffer allocated? */
		1115	enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
		1116	call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
		1117	call_get_func_ip:1, /* Do we call get_func_ip() */
		1118	tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */
		1119	enum bpf_prog_type type; /* Type of BPF program */
		1120	enum bpf_attach_type expected_attach_type; /* For some prog types */
		1121	u32 len; /* Number of filter blocks */
		1122	u32 jited_len; /* Size of jited insns in bytes */
		1123	u8 tag[BPF_TAG_SIZE];
		1124	struct bpf_prog_stats __percpu *stats;
		1125	int __percpu *active;
		1126	unsigned int (bpf_func)(const void ctx,
		1127	const struct bpf_insn *insn);
		1128	struct bpf_prog_aux aux; / Auxiliary fields */
		1129	struct sock_fprog_kern orig_prog; / Original BPF program */
		1130	/* Instructions for interpreter */
		1131	union {
		1132	DECLARE_FLEX_ARRAY(struct sock_filter, insns);
		1133	DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
		1134	};
		1135	};
		1136
1087	struct bpf_array_aux {	1137	struct bpf_array_aux {
1088	/* Programs with direct jumps into programs part of this array. */	1138	/* Programs with direct jumps into programs part of this array. */
1089	struct list_head poke_progs;	1139	struct list_head poke_progs;
@@ -1336,6 +1386,8 @@ extern struct bpf_empty_prog_array bpf_empty_prog_array; · 246 unmodified lines
1336		1386
1337	struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);	1387	struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags);
1338	void bpf_prog_array_free(struct bpf_prog_array *progs);	1388	void bpf_prog_array_free(struct bpf_prog_array *progs);
		1389	/* Use when traversal over the bpf_prog_array uses tasks_trace rcu */
		1390	void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs);
1339	int bpf_prog_array_length(struct bpf_prog_array *progs);	1391	int bpf_prog_array_length(struct bpf_prog_array *progs);
1340	bool bpf_prog_array_is_empty(struct bpf_prog_array *array);	1392	bool bpf_prog_array_is_empty(struct bpf_prog_array *array);
1341	int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,	1393	int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs,
@@ -1427,6 +1479,55 @@ bpf_prog_run_array(const struct bpf_prog_array *array, · 85 unmodified lines
1427	return ret;	1479	return ret;
1428	}	1480	}
1429		1481
		1482	/* Notes on RCU design for bpf_prog_arrays containing sleepable programs:
		1483	*
		1484	* We use the tasks_trace rcu flavor read section to protect the bpf_prog_array
		1485	* overall. As a result, we must use the bpf_prog_array_free_sleepable
		1486	* in order to use the tasks_trace rcu grace period.
		1487	*
		1488	* When a non-sleepable program is inside the array, we take the rcu read
		1489	* section and disable preemption for that program alone, so it can access
		1490	* rcu-protected dynamically sized maps.
		1491	*/
		1492	static __always_inline u32
		1493	bpf_prog_run_array_sleepable(const struct bpf_prog_array __rcu *array_rcu,
		1494	const void *ctx, bpf_prog_run_fn run_prog)
		1495	{
		1496	const struct bpf_prog_array_item *item;
		1497	const struct bpf_prog *prog;
		1498	const struct bpf_prog_array *array;
		1499	struct bpf_run_ctx *old_run_ctx;
		1500	struct bpf_trace_run_ctx run_ctx;
		1501	u32 ret = 1;
		1502
		1503	might_fault();
		1504
		1505	rcu_read_lock_trace();
		1506	migrate_disable();
		1507
		1508	array = rcu_dereference_check(array_rcu, rcu_read_lock_trace_held());
		1509	if (unlikely(!array))
		1510	goto out;
		1511	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
		1512	item = &array->items[0];
		1513	while ((prog = READ_ONCE(item->prog))) {
		1514	if (!prog->aux->sleepable)
		1515	rcu_read_lock();
		1516
		1517	run_ctx.bpf_cookie = item->bpf_cookie;
		1518	ret &= run_prog(prog, ctx);
		1519	item++;
		1520
		1521	if (!prog->aux->sleepable)
		1522	rcu_read_unlock();
		1523	}
		1524	bpf_reset_run_ctx(old_run_ctx);
		1525	out:
		1526	migrate_enable();
		1527	rcu_read_unlock_trace();
		1528	return ret;
		1529	}
		1530
1430	#ifdef CONFIG_BPF_SYSCALL	1531	#ifdef CONFIG_BPF_SYSCALL
1431	DECLARE_PER_CPU(int, bpf_prog_active);	1532	DECLARE_PER_CPU(int, bpf_prog_active);
1432	extern struct mutex bpf_stats_enabled_mutex;	1533	extern struct mutex bpf_stats_enabled_mutex;
@@ -2104,6 +2205,7 @@ int sock_map_bpf_prog_query(const union bpf_attr *attr, · 671 unmodified lines
2104	union bpf_attr __user *uattr);	2205	union bpf_attr __user *uattr);
2105		2206
2106	void sock_map_unhash(struct sock *sk);	2207	void sock_map_unhash(struct sock *sk);
		2208	void sock_map_destroy(struct sock *sk);
2107	void sock_map_close(struct sock *sk, long timeout);	2209	void sock_map_close(struct sock *sk, long timeout);
2108	#else	2210	#else
2109	static inline int bpf_prog_offload_init(struct bpf_prog *prog,	2211	static inline int bpf_prog_offload_init(struct bpf_prog *prog,
@@ -2261,12 +2363,9 @@ extern const struct bpf_func_proto bpf_for_each_map_elem_proto; · 151 unmodified lines
2261	extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;	2363	extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto;
2262	extern const struct bpf_func_proto bpf_sk_setsockopt_proto;	2364	extern const struct bpf_func_proto bpf_sk_setsockopt_proto;
2263	extern const struct bpf_func_proto bpf_sk_getsockopt_proto;	2365	extern const struct bpf_func_proto bpf_sk_getsockopt_proto;
2264	extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto;
2265	extern const struct bpf_func_proto bpf_find_vma_proto;	2366	extern const struct bpf_func_proto bpf_find_vma_proto;
2266	extern const struct bpf_func_proto bpf_loop_proto;	2367	extern const struct bpf_func_proto bpf_loop_proto;
2267	extern const struct bpf_func_proto bpf_strncmp_proto;
2268	extern const struct bpf_func_proto bpf_copy_from_user_task_proto;	2368	extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
2269	extern const struct bpf_func_proto bpf_kptr_xchg_proto;
2270		2369
2271	const struct bpf_func_proto *tracing_prog_func_proto(	2370	const struct bpf_func_proto *tracing_prog_func_proto(
2272	enum bpf_func_id func_id, const struct bpf_prog *prog);	2371	enum bpf_func_id func_id, const struct bpf_prog *prog);

include/linux/bpf_verifier.h+1−1

@@ -299,7 +299,7 @@ struct bpf_verifier_state { · 298 unmodified lines
299	* If is_state_visited() sees a state with branches > 0 it means	299	* If is_state_visited() sees a state with branches > 0 it means
300	* there is a loop. If such state is exactly equal to the current state	300	* there is a loop. If such state is exactly equal to the current state
301	* it's an infinite loop. Note states_equal() checks for states	301	* it's an infinite loop. Note states_equal() checks for states
302	* equvalency, so two states being 'states_equal' does not mean	302	* equivalency, so two states being 'states_equal' does not mean
303	* infinite loop. The exact comparison is provided by	303	* infinite loop. The exact comparison is provided by
304	* states_maybe_looping() function. It's a stronger pre-check and	304	* states_maybe_looping() function. It's a stronger pre-check and
305	* much faster than states_equal().	305	* much faster than states_equal().

include/linux/btf.h+28−0

@@ -177,6 +177,19 @@ static inline bool btf_type_is_enum(const struct btf_type *t) · 176 unmodified lines
177	return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM;	177	return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM;
178	}	178	}
179		179
		180	static inline bool btf_is_any_enum(const struct btf_type *t)
		181	{
		182	return BTF_INFO_KIND(t->info) == BTF_KIND_ENUM \|\|
		183	BTF_INFO_KIND(t->info) == BTF_KIND_ENUM64;
		184	}
		185
		186	static inline bool btf_kind_core_compat(const struct btf_type *t1,
		187	const struct btf_type *t2)
		188	{
		189	return BTF_INFO_KIND(t1->info) == BTF_INFO_KIND(t2->info) \|\|
		190	(btf_is_any_enum(t1) && btf_is_any_enum(t2));
		191	}
		192
180	static inline bool str_is_empty(const char *s)	193	static inline bool str_is_empty(const char *s)
181	{	194	{
182	return !s \|\| !s[0];	195	return !s \|\| !s[0];
@@ -192,6 +205,16 @@ static inline bool btf_is_enum(const struct btf_type *t) · 9 unmodified lines
192	return btf_kind(t) == BTF_KIND_ENUM;	205	return btf_kind(t) == BTF_KIND_ENUM;
193	}	206	}
194		207
		208	static inline bool btf_is_enum64(const struct btf_type *t)
		209	{
		210	return btf_kind(t) == BTF_KIND_ENUM64;
		211	}
		212
		213	static inline u64 btf_enum64_value(const struct btf_enum64 *e)
		214	{
		215	return ((u64)e->val_hi32 << 32) \| e->val_lo32;
		216	}
		217
195	static inline bool btf_is_composite(const struct btf_type *t)	218	static inline bool btf_is_composite(const struct btf_type *t)
196	{	219	{
197	u16 kind = btf_kind(t);	220	u16 kind = btf_kind(t);
@@ -332,6 +355,11 @@ static inline struct btf_enum btf_enum(const struct btf_type t) · 134 unmodified lines
332	return (struct btf_enum *)(t + 1);	355	return (struct btf_enum *)(t + 1);
333	}	356	}
334		357
		358	static inline struct btf_enum64 btf_enum64(const struct btf_type t)
		359	{
		360	return (struct btf_enum64 *)(t + 1);
		361	}
		362
335	static inline const struct btf_var_secinfo *btf_type_var_secinfo(	363	static inline const struct btf_var_secinfo *btf_type_var_secinfo(
336	const struct btf_type *t)	364	const struct btf_type *t)
337	{	365	{

include/linux/filter.h+0−34

@@ -559,40 +559,6 @@ struct bpf_prog_stats { · 558 unmodified lines
559	struct u64_stats_sync syncp;	559	struct u64_stats_sync syncp;
560	} __aligned(2 * sizeof(u64));	560	} __aligned(2 * sizeof(u64));
561		561
562	struct bpf_prog {
563	u16 pages; /* Number of allocated pages */
564	u16 jited:1, /* Is our filter JIT'ed? */
565	jit_requested:1,/* archs need to JIT the prog */
566	gpl_compatible:1, /* Is filter GPL compatible? */
567	cb_access:1, /* Is control block accessed? */
568	dst_needed:1, /* Do we need dst entry? */
569	blinding_requested:1, /* needs constant blinding */
570	blinded:1, /* Was blinded */
571	is_func:1, /* program is a bpf function */
572	kprobe_override:1, /* Do we override a kprobe? */
573	has_callchain_buf:1, /* callchain buffer allocated? */
574	enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */
575	call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */
576	call_get_func_ip:1, /* Do we call get_func_ip() */
577	tstamp_type_access:1; /* Accessed __sk_buff->tstamp_type */
578	enum bpf_prog_type type; /* Type of BPF program */
579	enum bpf_attach_type expected_attach_type; /* For some prog types */
580	u32 len; /* Number of filter blocks */
581	u32 jited_len; /* Size of jited insns in bytes */
582	u8 tag[BPF_TAG_SIZE];
583	struct bpf_prog_stats __percpu *stats;
584	int __percpu *active;
585	unsigned int (bpf_func)(const void ctx,
586	const struct bpf_insn *insn);
587	struct bpf_prog_aux aux; / Auxiliary fields */
588	struct sock_fprog_kern orig_prog; / Original BPF program */
589	/* Instructions for interpreter */
590	union {
591	DECLARE_FLEX_ARRAY(struct sock_filter, insns);
592	DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi);
593	};
594	};
595
596	struct sk_filter {	562	struct sk_filter {
597	refcount_t refcnt;	563	refcount_t refcnt;
598	struct rcu_head rcu;	564	struct rcu_head rcu;

include/linux/skmsg.h+1−0

@@ -95,6 +95,7 @@ struct sk_psock { · 94 unmodified lines
95	spinlock_t link_lock;	95	spinlock_t link_lock;
96	refcount_t refcnt;	96	refcount_t refcnt;
97	void (saved_unhash)(struct sock sk);	97	void (saved_unhash)(struct sock sk);
		98	void (saved_destroy)(struct sock sk);
98	void (saved_close)(struct sock sk, long timeout);	99	void (saved_close)(struct sock sk, long timeout);
99	void (saved_write_space)(struct sock sk);	100	void (saved_write_space)(struct sock sk);
100	void (saved_data_ready)(struct sock sk);	101	void (saved_data_ready)(struct sock sk);

include/net/tcp.h+1−0

@@ -434,6 +434,7 @@ u16 tcp_v4_get_syncookie(struct sock sk, struct iphdr iph, · 433 unmodified lines
434	struct tcphdr th, u32 cookie);	434	struct tcphdr th, u32 cookie);
435	u16 tcp_v6_get_syncookie(struct sock sk, struct ipv6hdr iph,	435	u16 tcp_v6_get_syncookie(struct sock sk, struct ipv6hdr iph,
436	struct tcphdr th, u32 cookie);	436	struct tcphdr th, u32 cookie);
		437	u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss);
437	u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,	438	u16 tcp_get_syncookie_mss(struct request_sock_ops *rsk_ops,
438	const struct tcp_request_sock_ops *af_ops,	439	const struct tcp_request_sock_ops *af_ops,
439	struct sock sk, struct tcphdr th);	440	struct sock sk, struct tcphdr th);

include/uapi/linux/bpf.h+84−4

@@ -3597,10 +3597,11 @@ union bpf_attr { · 3596 unmodified lines
3597	*	3597	*
3598	* iph points to the start of the IPv4 or IPv6 header, while	3598	* iph points to the start of the IPv4 or IPv6 header, while
3599	* iph_len contains sizeof\ (struct iphdr) or	3599	* iph_len contains sizeof\ (struct iphdr) or
3600	* sizeof\ (struct ip6hdr).	3600	* sizeof\ (struct ipv6hdr).
3601	*	3601	*
3602	* th points to the start of the TCP header, while th_len	3602	* th points to the start of the TCP header, while th_len
3603	* contains sizeof\ (struct tcphdr).	3603	* contains the length of the TCP header (at least
		3604	* sizeof\ (struct tcphdr)).
3604	* Return	3605	* Return
3605	* 0 if iph and th are a valid SYN cookie ACK, or a negative	3606	* 0 if iph and th are a valid SYN cookie ACK, or a negative
3606	* error otherwise.	3607	* error otherwise.
@@ -3783,10 +3784,11 @@ union bpf_attr { · 176 unmodified lines
3783	*	3784	*
3784	* iph points to the start of the IPv4 or IPv6 header, while	3785	* iph points to the start of the IPv4 or IPv6 header, while
3785	* iph_len contains sizeof\ (struct iphdr) or	3786	* iph_len contains sizeof\ (struct iphdr) or
3786	* sizeof\ (struct ip6hdr).	3787	* sizeof\ (struct ipv6hdr).
3787	*	3788	*
3788	* th points to the start of the TCP header, while th_len	3789	* th points to the start of the TCP header, while th_len
3789	* contains the length of the TCP header.	3790	* contains the length of the TCP header with options (at least
		3791	* sizeof\ (struct tcphdr)).
3790	* Return	3792	* Return
3791	* On success, lower 32 bits hold the generated SYN cookie in	3793	* On success, lower 32 bits hold the generated SYN cookie in
3792	* followed by 16 bits which hold the MSS value for that cookie,	3794	* followed by 16 bits which hold the MSS value for that cookie,
@@ -5249,6 +5251,80 @@ union bpf_attr { · 1456 unmodified lines
5249	* Pointer to the underlying dynptr data, NULL if the dynptr is	5251	* Pointer to the underlying dynptr data, NULL if the dynptr is
5250	* read-only, if the dynptr is invalid, or if the offset and length	5252	* read-only, if the dynptr is invalid, or if the offset and length
5251	* is out of bounds.	5253	* is out of bounds.
		5254	*
		5255	* s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr iph, struct tcphdr th, u32 th_len)
		5256	* Description
		5257	* Try to issue a SYN cookie for the packet with corresponding
		5258	* IPv4/TCP headers, iph and th, without depending on a
		5259	* listening socket.
		5260	*
		5261	* iph points to the IPv4 header.
		5262	*
		5263	* th points to the start of the TCP header, while th_len
		5264	* contains the length of the TCP header (at least
		5265	* sizeof\ (struct tcphdr)).
		5266	* Return
		5267	* On success, lower 32 bits hold the generated SYN cookie in
		5268	* followed by 16 bits which hold the MSS value for that cookie,
		5269	* and the top 16 bits are unused.
		5270	*
		5271	* On failure, the returned value is one of the following:
		5272	*
		5273	* -EINVAL if th_len is invalid.
		5274	*
		5275	* s64 bpf_tcp_raw_gen_syncookie_ipv6(struct ipv6hdr iph, struct tcphdr th, u32 th_len)
		5276	* Description
		5277	* Try to issue a SYN cookie for the packet with corresponding
		5278	* IPv6/TCP headers, iph and th, without depending on a
		5279	* listening socket.
		5280	*
		5281	* iph points to the IPv6 header.
		5282	*
		5283	* th points to the start of the TCP header, while th_len
		5284	* contains the length of the TCP header (at least
		5285	* sizeof\ (struct tcphdr)).
		5286	* Return
		5287	* On success, lower 32 bits hold the generated SYN cookie in
		5288	* followed by 16 bits which hold the MSS value for that cookie,
		5289	* and the top 16 bits are unused.
		5290	*
		5291	* On failure, the returned value is one of the following:
		5292	*
		5293	* -EINVAL if th_len is invalid.
		5294	*
		5295	* -EPROTONOSUPPORT if CONFIG_IPV6 is not builtin.
		5296	*
		5297	* long bpf_tcp_raw_check_syncookie_ipv4(struct iphdr iph, struct tcphdr th)
		5298	* Description
		5299	* Check whether iph and th contain a valid SYN cookie ACK
		5300	* without depending on a listening socket.
		5301	*
		5302	* iph points to the IPv4 header.
		5303	*
		5304	* th points to the TCP header.
		5305	* Return
		5306	* 0 if iph and th are a valid SYN cookie ACK.
		5307	*
		5308	* On failure, the returned value is one of the following:
		5309	*
		5310	* -EACCES if the SYN cookie is not valid.
		5311	*
		5312	* long bpf_tcp_raw_check_syncookie_ipv6(struct ipv6hdr iph, struct tcphdr th)
		5313	* Description
		5314	* Check whether iph and th contain a valid SYN cookie ACK
		5315	* without depending on a listening socket.
		5316	*
		5317	* iph points to the IPv6 header.
		5318	*
		5319	* th points to the TCP header.
		5320	* Return
		5321	* 0 if iph and th are a valid SYN cookie ACK.
		5322	*
		5323	* On failure, the returned value is one of the following:
		5324	*
		5325	* -EACCES if the SYN cookie is not valid.
		5326	*
		5327	* -EPROTONOSUPPORT if CONFIG_IPV6 is not builtin.
5252	*/	5328	*/
5253	#define __BPF_FUNC_MAPPER(FN) \	5329	#define __BPF_FUNC_MAPPER(FN) \
5254	FN(unspec), \	5330	FN(unspec), \
@@ -5455,6 +5531,10 @@ union bpf_attr { · 200 unmodified lines
5455	FN(dynptr_read), \	5531	FN(dynptr_read), \
5456	FN(dynptr_write), \	5532	FN(dynptr_write), \
5457	FN(dynptr_data), \	5533	FN(dynptr_data), \
		5534	FN(tcp_raw_gen_syncookie_ipv4), \
		5535	FN(tcp_raw_gen_syncookie_ipv6), \
		5536	FN(tcp_raw_check_syncookie_ipv4), \
		5537	FN(tcp_raw_check_syncookie_ipv6), \
5458	/* */	5538	/* */
5459		5539
5460	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	5540	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

include/uapi/linux/btf.h+14−3

 	 * bits 24-28: kind (e.g. int, ptr, array...etc)
 	 * bits 29-30: unused
 	 * bit     31: kind_flag, currently used by
-	 *             struct, union and fwd
+	 *             struct, union, enum, fwd and enum64
 	 */
 	__u32 info;
-	/* "size" is used by INT, ENUM, STRUCT, UNION and DATASEC.
+	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.
 	 * "size" tells the size of the type it is describing.
 	 *
 	 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
 	BTF_KIND_ARRAY		= 3,	/* Array	*/
 	BTF_KIND_STRUCT		= 4,	/* Struct	*/
 	BTF_KIND_UNION		= 5,	/* Union	*/
-	BTF_KIND_ENUM		= 6,	/* Enumeration	*/
+	BTF_KIND_ENUM		= 6,	/* Enumeration up to 32-bit values */
 	BTF_KIND_FWD		= 7,	/* Forward	*/
 	BTF_KIND_TYPEDEF	= 8,	/* Typedef	*/
 	BTF_KIND_VOLATILE	= 9,	/* Volatile	*/
 	BTF_KIND_FLOAT		= 16,	/* Floating point	*/
 	BTF_KIND_DECL_TAG	= 17,	/* Decl Tag */
 	BTF_KIND_TYPE_TAG	= 18,	/* Type Tag */
+	BTF_KIND_ENUM64		= 19,	/* Enumeration up to 64-bit values */
 	NR_BTF_KINDS,
 	BTF_KIND_MAX		= NR_BTF_KINDS - 1,
        __s32   component_idx;
 };
+/* BTF_KIND_ENUM64 is followed by multiple "struct btf_enum64".
+ * The exact number of btf_enum64 is stored in the vlen (of the
+ * info in "struct btf_type").
+ */
+struct btf_enum64 {
+	__u32	name_off;
+	__u32	val_lo32;
+	__u32	val_hi32;
+};
 #endif /* _UAPI__LINUX_BTF_H__ */

kernel/bpf/btf.c+128−14

 	[BTF_KIND_FLOAT]	= "FLOAT",
 	[BTF_KIND_DECL_TAG]	= "DECL_TAG",
 	[BTF_KIND_TYPE_TAG]	= "TYPE_TAG",
+	[BTF_KIND_ENUM64]	= "ENUM64",
 };
 const char *btf_type_str(const struct btf_type *t)
 	case BTF_KIND_ENUM:
 	case BTF_KIND_DATASEC:
 	case BTF_KIND_FLOAT:
+	case BTF_KIND_ENUM64:
 		return true;
 	}
 	return (const struct btf_decl_tag *)(t + 1);
 }
+static const struct btf_enum64 *btf_type_enum64(const struct btf_type *t)
+{
+	return (const struct btf_enum64 *)(t + 1);
+}
 static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
 {
 	return kind_ops[BTF_INFO_KIND(t->info)];
 			parens = "{";
 		break;
 	case BTF_KIND_ENUM:
+	case BTF_KIND_ENUM64:
 		prefix = "enum";
 		break;
 	default:
 		case BTF_KIND_UNION:
 		case BTF_KIND_ENUM:
 		case BTF_KIND_FLOAT:
+		case BTF_KIND_ENUM64:
 			size = type->size;
 			goto resolved;
 {
 	const struct btf_enum *enums = btf_type_enum(t);
 	struct btf *btf = env->btf;
+	const char *fmt_str;
 	u16 i, nr_enums;
 	u32 meta_needed;
 		return -EINVAL;
 	}
-	if (btf_type_kflag(t)) {
-		btf_verifier_log_type(env, t, "Invalid btf_info kind_flag");
-		return -EINVAL;
-	}
 	if (t->size > 8 || !is_power_of_2(t->size)) {
 		btf_verifier_log_type(env, t, "Unexpected size");
 		return -EINVAL;
 		if (env->log.level == BPF_LOG_KERNEL)
 			continue;
-		btf_verifier_log(env, "\t%s val=%d\n",
+		fmt_str = btf_type_kflag(t) ? "\t%s val=%d\n" : "\t%s val=%u\n";
+		btf_verifier_log(env, fmt_str,
 				 __btf_name_by_offset(btf, enums[i].name_off),
 				 enums[i].val);
 	}
 		return;
 	}
-	btf_show_type_value(show, "%d", v);
+	if (btf_type_kflag(t))
+		btf_show_type_value(show, "%d", v);
+	else
+		btf_show_type_value(show, "%u", v);
 	btf_show_end_type(show);
 }
 	.show = btf_enum_show,
 };
+static s32 btf_enum64_check_meta(struct btf_verifier_env *env,
+				 const struct btf_type *t,
+				 u32 meta_left)
+{
+	const struct btf_enum64 *enums = btf_type_enum64(t);
+	struct btf *btf = env->btf;
+	const char *fmt_str;
+	u16 i, nr_enums;
+	u32 meta_needed;
+	nr_enums = btf_type_vlen(t);
+	meta_needed = nr_enums * sizeof(*enums);
+	if (meta_left < meta_needed) {
+		btf_verifier_log_basic(env, t,
+				       "meta_left:%u meta_needed:%u",
+				       meta_left, meta_needed);
+		return -EINVAL;
+	}
+	if (t->size > 8 || !is_power_of_2(t->size)) {
+		btf_verifier_log_type(env, t, "Unexpected size");
+		return -EINVAL;
+	}
+	/* enum type either no name or a valid one */
+	if (t->name_off &&
+	    !btf_name_valid_identifier(env->btf, t->name_off)) {
+		btf_verifier_log_type(env, t, "Invalid name");
+		return -EINVAL;
+	}
+	btf_verifier_log_type(env, t, NULL);
+	for (i = 0; i < nr_enums; i++) {
+		if (!btf_name_offset_valid(btf, enums[i].name_off)) {
+			btf_verifier_log(env, "\tInvalid name_offset:%u",
+					 enums[i].name_off);
+			return -EINVAL;
+		}
+		/* enum member must have a valid name */
+		if (!enums[i].name_off ||
+		    !btf_name_valid_identifier(btf, enums[i].name_off)) {
+			btf_verifier_log_type(env, t, "Invalid name");
+			return -EINVAL;
+		}
+		if (env->log.level == BPF_LOG_KERNEL)
+			continue;
+		fmt_str = btf_type_kflag(t) ? "\t%s val=%lld\n" : "\t%s val=%llu\n";
+		btf_verifier_log(env, fmt_str,
+				 __btf_name_by_offset(btf, enums[i].name_off),
+				 btf_enum64_value(enums + i));
+	}
+	return meta_needed;
+}
+static void btf_enum64_show(const struct btf *btf, const struct btf_type *t,
+			    u32 type_id, void *data, u8 bits_offset,
+			    struct btf_show *show)
+{
+	const struct btf_enum64 *enums = btf_type_enum64(t);
+	u32 i, nr_enums = btf_type_vlen(t);
+	void *safe_data;
+	s64 v;
+	safe_data = btf_show_start_type(show, t, type_id, data);
+	if (!safe_data)
+		return;
+	v = *(u64 *)safe_data;
+	for (i = 0; i < nr_enums; i++) {
+		if (v != btf_enum64_value(enums + i))
+			continue;
+		btf_show_type_value(show, "%s",
+				    __btf_name_by_offset(btf,
+							 enums[i].name_off));
+		btf_show_end_type(show);
+		return;
+	}
+	if (btf_type_kflag(t))
+		btf_show_type_value(show, "%lld", v);
+	else
+		btf_show_type_value(show, "%llu", v);
+	btf_show_end_type(show);
+}
+static struct btf_kind_operations enum64_ops = {
+	.check_meta = btf_enum64_check_meta,
+	.resolve = btf_df_resolve,
+	.check_member = btf_enum_check_member,
+	.check_kflag_member = btf_enum_check_kflag_member,
+	.log_details = btf_enum_log,
+	.show = btf_enum64_show,
+};
 static s32 btf_func_proto_check_meta(struct btf_verifier_env *env,
 				     const struct btf_type *t,
 				     u32 meta_left)
 	[BTF_KIND_FLOAT] = &float_ops,
 	[BTF_KIND_DECL_TAG] = &decl_tag_ops,
 	[BTF_KIND_TYPE_TAG] = &modifier_ops,
+	[BTF_KIND_ENUM64] = &enum64_ops,
 };
 static s32 btf_check_meta(struct btf_verifier_env *env,
 	/* skip modifiers */
 	while (btf_type_is_modifier(t))
 		t = btf_type_by_id(btf, t->type);
-	if (btf_type_is_small_int(t) || btf_type_is_enum(t))
+	if (btf_type_is_small_int(t) || btf_is_any_enum(t))
 		/* accessing a scalar */
 		return true;
 	if (!btf_type_is_ptr(t)) {
 	if (btf_type_is_ptr(t))
 		/* kernel size of pointer. Not BPF's size of pointer*/
 		return sizeof(void *);
-	if (btf_type_is_int(t) || btf_type_is_enum(t))
+	if (btf_type_is_int(t) || btf_is_any_enum(t))
 		return t->size;
 	*bad_type = t;
 	return -EINVAL;
 		 * to context only. And only global functions can be replaced.
 		 * Hence type check only those types.
 		 */
-		if (btf_type_is_int(t1) || btf_type_is_enum(t1))
+		if (btf_type_is_int(t1) || btf_is_any_enum(t1))
 			continue;
 		if (!btf_type_is_ptr(t1)) {
 			bpf_log(log,
 	t = btf_type_by_id(btf, t->type);
 	while (btf_type_is_modifier(t))
 		t = btf_type_by_id(btf, t->type);
-	if (!btf_type_is_int(t) && !btf_type_is_enum(t)) {
+	if (!btf_type_is_int(t) && !btf_is_any_enum(t)) {
 		bpf_log(log,
 			"Global function %s() doesn't return scalar. Only those are supported.\n",
 			tname);
 		t = btf_type_by_id(btf, args[i].type);
 		while (btf_type_is_modifier(t))
 			t = btf_type_by_id(btf, t->type);
-		if (btf_type_is_int(t) || btf_type_is_enum(t)) {
+		if (btf_type_is_int(t) || btf_is_any_enum(t)) {
 			reg->type = SCALAR_VALUE;
 			continue;
 		}
 	case BTF_KIND_UNION:
 	case BTF_KIND_ENUM:
 	case BTF_KIND_FWD:
+	case BTF_KIND_ENUM64:
 		return 1;
 	case BTF_KIND_INT:
 		/* just reject deprecated bitfield-like integers; all other
  * field-based relocations. This function assumes that root types were already
  * checked for name match. Beyond that initial root-level name check, names
  * are completely ignored. Compatibility rules are as follows:
- *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
+ *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs/ENUM64s are considered compatible, but
  *     kind should match for local and target types (i.e., STRUCT is not
  *     compatible with UNION);
- *   - for ENUMs, the size is ignored;
+ *   - for ENUMs/ENUM64s, the size is ignored;
  *   - for INT, size and signedness are ignored;
  *   - for ARRAY, dimensionality is ignored, element types are checked for
  *     compatibility recursively;

kernel/bpf/cgroup.c+60−10

@@ -720,6 +720,60 @@ static struct bpf_prog_list find_detach_entry(struct list_head progs, · 719 unmodified lines
720	return ERR_PTR(-ENOENT);	720	return ERR_PTR(-ENOENT);
721	}	721	}
722		722
		723	/**
		724	* purge_effective_progs() - After compute_effective_progs fails to alloc new
		725	* cgrp->bpf.inactive table we can recover by
		726	* recomputing the array in place.
		727	*
		728	* @cgrp: The cgroup which descendants to travers
		729	* @prog: A program to detach or NULL
		730	* @link: A link to detach or NULL
		731	* @atype: Type of detach operation
		732	*/
		733	static void purge_effective_progs(struct cgroup cgrp, struct bpf_prog prog,
		734	struct bpf_cgroup_link *link,
		735	enum cgroup_bpf_attach_type atype)
		736	{
		737	struct cgroup_subsys_state *css;
		738	struct bpf_prog_array *progs;
		739	struct bpf_prog_list *pl;
		740	struct list_head *head;
		741	struct cgroup *cg;
		742	int pos;
		743
		744	/* recompute effective prog array in place */
		745	css_for_each_descendant_pre(css, &cgrp->self) {
		746	struct cgroup *desc = container_of(css, struct cgroup, self);
		747
		748	if (percpu_ref_is_zero(&desc->bpf.refcnt))
		749	continue;
		750
		751	/* find position of link or prog in effective progs array */
		752	for (pos = 0, cg = desc; cg; cg = cgroup_parent(cg)) {
		753	if (pos && !(cg->bpf.flags[atype] & BPF_F_ALLOW_MULTI))
		754	continue;
		755
		756	head = &cg->bpf.progs[atype];
		757	list_for_each_entry(pl, head, node) {
		758	if (!prog_list_prog(pl))
		759	continue;
		760	if (pl->prog == prog && pl->link == link)
		761	goto found;
		762	pos++;
		763	}
		764	}
		765	found:
		766	BUG_ON(!cg);
		767	progs = rcu_dereference_protected(
		768	desc->bpf.effective[atype],
		769	lockdep_is_held(&cgroup_mutex));
		770
		771	/* Remove the program from the array */
		772	WARN_ONCE(bpf_prog_array_delete_safe_at(progs, pos),
		773	"Failed to purge a prog from array at index %d", pos);
		774	}
		775	}
		776
723	/**	777	/**
724	* __cgroup_bpf_detach() - Detach the program or link from a cgroup, and	778	* __cgroup_bpf_detach() - Detach the program or link from a cgroup, and
725	* propagate the change to descendants	779	* propagate the change to descendants
@@ -739,7 +793,6 @@ static int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog, · 13 unmodified lines
739	struct bpf_prog_list *pl;	793	struct bpf_prog_list *pl;
740	struct list_head *progs;	794	struct list_head *progs;
741	u32 flags;	795	u32 flags;
742	int err;
743		796
744	atype = to_cgroup_bpf_attach_type(type);	797	atype = to_cgroup_bpf_attach_type(type);
745	if (atype < 0)	798	if (atype < 0)
@@ -761,9 +814,12 @@ static int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog, · 15 unmodified lines
761	pl->prog = NULL;	814	pl->prog = NULL;
762	pl->link = NULL;	815	pl->link = NULL;
763		816
764	err = update_effective_progs(cgrp, atype);	817	if (update_effective_progs(cgrp, atype)) {
765	if (err)	818	/* if update effective array failed replace the prog with a dummy prog*/
766	goto cleanup;	819	pl->prog = old_prog;
		820	pl->link = link;
		821	purge_effective_progs(cgrp, old_prog, link, atype);
		822	}
767		823
768	/* now can actually delete it from this cgroup list */	824	/* now can actually delete it from this cgroup list */
769	list_del(&pl->node);	825	list_del(&pl->node);
@@ -775,12 +831,6 @@ static int __cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog, · 5 unmodified lines
775	bpf_prog_put(old_prog);	831	bpf_prog_put(old_prog);
776	static_branch_dec(&cgroup_bpf_enabled_key[atype]);	832	static_branch_dec(&cgroup_bpf_enabled_key[atype]);
777	return 0;	833	return 0;
778
779	cleanup:
780	/* restore back prog or link */
781	pl->prog = old_prog;
782	pl->link = link;
783	return err;
784	}	834	}
785		835
786	static int cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog,	836	static int cgroup_bpf_detach(struct cgroup cgrp, struct bpf_prog prog,

kernel/bpf/core.c+16−1

@@ -176,7 +176,7 @@ void bpf_prog_jit_attempt_done(struct bpf_prog *prog) · 175 unmodified lines
176	* here is relative to the prog itself instead of the main prog.	176	* here is relative to the prog itself instead of the main prog.
177	* This array has one entry for each xlated bpf insn.	177	* This array has one entry for each xlated bpf insn.
178	*	178	*
179	* jited_off is the byte off to the last byte of the jited insn.	179	* jited_off is the byte off to the end of the jited insn.
180	*	180	*
181	* Hence, with	181	* Hence, with
182	* insn_start:	182	* insn_start:
@@ -2279,6 +2279,21 @@ void bpf_prog_array_free(struct bpf_prog_array *progs) · 2096 unmodified lines
2279	kfree_rcu(progs, rcu);	2279	kfree_rcu(progs, rcu);
2280	}	2280	}
2281		2281
		2282	static void __bpf_prog_array_free_sleepable_cb(struct rcu_head *rcu)
		2283	{
		2284	struct bpf_prog_array *progs;
		2285
		2286	progs = container_of(rcu, struct bpf_prog_array, rcu);
		2287	kfree_rcu(progs, rcu);
		2288	}
		2289
		2290	void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs)
		2291	{
		2292	if (!progs \|\| progs == &bpf_empty_prog_array.hdr)
		2293	return;
		2294	call_rcu_tasks_trace(&progs->rcu, __bpf_prog_array_free_sleepable_cb);
		2295	}
		2296
2282	int bpf_prog_array_length(struct bpf_prog_array *array)	2297	int bpf_prog_array_length(struct bpf_prog_array *array)
2283	{	2298	{
2284	struct bpf_prog_array_item *item;	2299	struct bpf_prog_array_item *item;

kernel/bpf/helpers.c+6−6

@@ -584,7 +584,7 @@ BPF_CALL_3(bpf_strncmp, const char , s1, u32, s1_sz, const char , s2) · 583 unmodified lines
584	return strncmp(s1, s2, s1_sz);	584	return strncmp(s1, s2, s1_sz);
585	}	585	}
586		586
587	const struct bpf_func_proto bpf_strncmp_proto = {	587	static const struct bpf_func_proto bpf_strncmp_proto = {
588	.func = bpf_strncmp,	588	.func = bpf_strncmp,
589	.gpl_only = false,	589	.gpl_only = false,
590	.ret_type = RET_INTEGER,	590	.ret_type = RET_INTEGER,
@@ -1402,7 +1402,7 @@ BPF_CALL_2(bpf_kptr_xchg, void , map_value, void , ptr) · 811 unmodified lines
1402	*/	1402	*/
1403	#define BPF_PTR_POISON ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))	1403	#define BPF_PTR_POISON ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
1404		1404
1405	const struct bpf_func_proto bpf_kptr_xchg_proto = {	1405	static const struct bpf_func_proto bpf_kptr_xchg_proto = {
1406	.func = bpf_kptr_xchg,	1406	.func = bpf_kptr_xchg,
1407	.gpl_only = false,	1407	.gpl_only = false,
1408	.ret_type = RET_PTR_TO_BTF_ID_OR_NULL,	1408	.ret_type = RET_PTR_TO_BTF_ID_OR_NULL,
@@ -1487,7 +1487,7 @@ error: · 78 unmodified lines
1487	return err;	1487	return err;
1488	}	1488	}
1489		1489
1490	const struct bpf_func_proto bpf_dynptr_from_mem_proto = {	1490	static const struct bpf_func_proto bpf_dynptr_from_mem_proto = {
1491	.func = bpf_dynptr_from_mem,	1491	.func = bpf_dynptr_from_mem,
1492	.gpl_only = false,	1492	.gpl_only = false,
1493	.ret_type = RET_INTEGER,	1493	.ret_type = RET_INTEGER,
@@ -1513,7 +1513,7 @@ BPF_CALL_4(bpf_dynptr_read, void , dst, u32, len, struct bpf_dynptr_kern , src · 19 unmodified lines
1513	return 0;	1513	return 0;
1514	}	1514	}
1515		1515
1516	const struct bpf_func_proto bpf_dynptr_read_proto = {	1516	static const struct bpf_func_proto bpf_dynptr_read_proto = {
1517	.func = bpf_dynptr_read,	1517	.func = bpf_dynptr_read,
1518	.gpl_only = false,	1518	.gpl_only = false,
1519	.ret_type = RET_INTEGER,	1519	.ret_type = RET_INTEGER,
@@ -1539,7 +1539,7 @@ BPF_CALL_4(bpf_dynptr_write, struct bpf_dynptr_kern , dst, u32, offset, void , · 19 unmodified lines
1539	return 0;	1539	return 0;
1540	}	1540	}
1541		1541
1542	const struct bpf_func_proto bpf_dynptr_write_proto = {	1542	static const struct bpf_func_proto bpf_dynptr_write_proto = {
1543	.func = bpf_dynptr_write,	1543	.func = bpf_dynptr_write,
1544	.gpl_only = false,	1544	.gpl_only = false,
1545	.ret_type = RET_INTEGER,	1545	.ret_type = RET_INTEGER,
@@ -1566,7 +1566,7 @@ BPF_CALL_3(bpf_dynptr_data, struct bpf_dynptr_kern *, ptr, u32, offset, u32, len · 20 unmodified lines
1566	return (unsigned long)(ptr->data + ptr->offset + offset);	1566	return (unsigned long)(ptr->data + ptr->offset + offset);
1567	}	1567	}
1568		1568
1569	const struct bpf_func_proto bpf_dynptr_data_proto = {	1569	static const struct bpf_func_proto bpf_dynptr_data_proto = {
1570	.func = bpf_dynptr_data,	1570	.func = bpf_dynptr_data,
1571	.gpl_only = false,	1571	.gpl_only = false,
1572	.ret_type = RET_PTR_TO_DYNPTR_MEM_OR_NULL,	1572	.ret_type = RET_PTR_TO_DYNPTR_MEM_OR_NULL,

kernel/bpf/percpu_freelist.c+14−6

 					   struct pcpu_freelist_node *node)
 {
 	node->next = head->first;
-	head->first = node;
+	WRITE_ONCE(head->first, node);
 }
 static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,
 	orig_cpu = cpu = raw_smp_processor_id();
 	while (1) {
 		head = per_cpu_ptr(s->freelist, cpu);
+		if (!READ_ONCE(head->first))
+			goto next_cpu;
 		raw_spin_lock(&head->lock);
 		node = head->first;
 		if (node) {
-			head->first = node->next;
+			WRITE_ONCE(head->first, node->next);
 			raw_spin_unlock(&head->lock);
 			return node;
 		}
 		raw_spin_unlock(&head->lock);
+next_cpu:
 		cpu = cpumask_next(cpu, cpu_possible_mask);
 		if (cpu >= nr_cpu_ids)
 			cpu = 0;
 	}
 	/* per cpu lists are all empty, try extralist */
+	if (!READ_ONCE(s->extralist.first))
+		return NULL;
 	raw_spin_lock(&s->extralist.lock);
 	node = s->extralist.first;
 	if (node)
-		s->extralist.first = node->next;
+		WRITE_ONCE(s->extralist.first, node->next);
 	raw_spin_unlock(&s->extralist.lock);
 	return node;
 }
 	orig_cpu = cpu = raw_smp_processor_id();
 	while (1) {
 		head = per_cpu_ptr(s->freelist, cpu);
+		if (!READ_ONCE(head->first))
+			goto next_cpu;
 		if (raw_spin_trylock(&head->lock)) {
 			node = head->first;
 			if (node) {
-				head->first = node->next;
+				WRITE_ONCE(head->first, node->next);
 				raw_spin_unlock(&head->lock);
 				return node;
 			}
 			raw_spin_unlock(&head->lock);
 		}
+next_cpu:
 		cpu = cpumask_next(cpu, cpu_possible_mask);
 		if (cpu >= nr_cpu_ids)
 			cpu = 0;
 	}
 	/* cannot pop from per cpu lists, try extralist */
-	if (!raw_spin_trylock(&s->extralist.lock))
+	if (!READ_ONCE(s->extralist.first) || !raw_spin_trylock(&s->extralist.lock))
 		return NULL;
 	node = s->extralist.first;
 	if (node)
-		s->extralist.first = node->next;
+		WRITE_ONCE(s->extralist.first, node->next);
 	raw_spin_unlock(&s->extralist.lock);
 	return node;
 }

kernel/bpf/syscall.c+4−3

 		info.nr_jited_line_info = 0;
 	if (info.nr_jited_line_info && ulen) {
 		if (bpf_dump_raw_ok(file->f_cred)) {
+			unsigned long line_addr;
 			__u64 __user *user_linfo;
 			u32 i;
 			user_linfo = u64_to_user_ptr(info.jited_line_info);
 			ulen = min_t(u32, info.nr_jited_line_info, ulen);
 			for (i = 0; i < ulen; i++) {
-				if (put_user((__u64)(long)prog->aux->jited_linfo[i],
+				line_addr = (unsigned long)prog->aux->jited_linfo[i];
-					     &user_linfo[i]))
+				if (put_user((__u64)line_addr, &user_linfo[i]))
 					return -EFAULT;
 			}
 		} else {
 	return *res ? 0 : -ENOENT;
 }
-const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
+static const struct bpf_func_proto bpf_kallsyms_lookup_name_proto = {
 	.func		= bpf_kallsyms_lookup_name,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,

kernel/bpf/verifier.c+35−14

 	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
 	enum bpf_arg_type arg_type = fn->arg_type[arg];
 	enum bpf_reg_type type = reg->type;
+	u32 *arg_btf_id = NULL;
 	int err = 0;
 	if (arg_type == ARG_DONTCARE)
 		 */
 		goto skip_type_check;
-	err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg], meta);
+	/* arg_btf_id and arg_size are in a union. */
+	if (base_type(arg_type) == ARG_PTR_TO_BTF_ID)
+		arg_btf_id = fn->arg_btf_id[arg];
+	err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
 	if (err)
 		return err;
 		 * next is_mem_size argument below.
 		 */
 		meta->raw_mode = arg_type & MEM_UNINIT;
+		if (arg_type & MEM_FIXED_SIZE) {
+			err = check_helper_mem_access(env, regno,
+						      fn->arg_size[arg], false,
+						      meta);
+		}
 	} else if (arg_type_is_mem_size(arg_type)) {
 		bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
 	return count <= 1;
 }
-static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
+static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
-				    enum bpf_arg_type arg_next)
 {
-	return (base_type(arg_curr) == ARG_PTR_TO_MEM) !=
+	bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
-		arg_type_is_mem_size(arg_next);
+	bool has_size = fn->arg_size[arg] != 0;
+	bool is_next_size = false;
+	if (arg + 1 < ARRAY_SIZE(fn->arg_type))
+		is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
+	if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
+		return is_next_size;
+	return has_size == is_next_size || is_next_size == is_fixed;
 }
 static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
 	 * helper function specification.
 	 */
 	if (arg_type_is_mem_size(fn->arg1_type) ||
-	    base_type(fn->arg5_type) == ARG_PTR_TO_MEM ||
+	    check_args_pair_invalid(fn, 0) ||
-	    check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
+	    check_args_pair_invalid(fn, 1) ||
-	    check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
+	    check_args_pair_invalid(fn, 2) ||
-	    check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
+	    check_args_pair_invalid(fn, 3) ||
-	    check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
+	    check_args_pair_invalid(fn, 4))
 		return false;
 	return true;
 		if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])
 			return false;
-		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i])
+		if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
+		    /* arg_btf_id and arg_size are in a union. */
+		    (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
+		     !(fn->arg_type[i] & MEM_FIXED_SIZE)))
 			return false;
 	}
 			goto err_free;
 		ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
 		scalar_return =
-			btf_type_is_small_int(ret_type) || btf_type_is_enum(ret_type);
+			btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
 		if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
 			verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
 			goto err_free;
 	}
 	if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
-	    prog->type != BPF_PROG_TYPE_LSM) {
+	    prog->type != BPF_PROG_TYPE_LSM && prog->type != BPF_PROG_TYPE_KPROBE) {
-		verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
+		verbose(env, "Only fentry/fexit/fmod_ret, lsm, and kprobe/uprobe programs can be sleepable\n");
 		return -EINVAL;
 	}

kernel/events/core.c+10−6

 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,
 			    u64 bpf_cookie)
 {
-	bool is_kprobe, is_tracepoint, is_syscall_tp;
+	bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp;
 	if (!perf_event_is_tracing(event))
 		return perf_event_set_bpf_handler(event, prog, bpf_cookie);
-	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE;
+	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;
+	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;
 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;
 	is_syscall_tp = is_syscall_trace_event(event->tp_event);
-	if (!is_kprobe && !is_tracepoint && !is_syscall_tp)
+	if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp)
 		/* bpf programs can only be attached to u/kprobe or tracepoint */
 		return -EINVAL;
-	if ((is_kprobe && prog->type != BPF_PROG_TYPE_KPROBE) ||
+	if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) ||
 	    (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) ||
 	    (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT))
 		return -EINVAL;
+	if (prog->type == BPF_PROG_TYPE_KPROBE && prog->aux->sleepable && !is_uprobe)
+		/* only uprobe programs are allowed to be sleepable */
+		return -EINVAL;
 	/* Kprobe override only works for kprobes, not uprobes. */
-	if (prog->kprobe_override &&
+	if (prog->kprobe_override && !is_kprobe)
-	    !(event->tp_event->flags & TRACE_EVENT_FL_KPROBE))
 		return -EINVAL;
 	if (is_tracepoint || is_syscall_tp) {

kernel/trace/bpf_trace.c+2−2

@@ -1936,7 +1936,7 @@ int perf_event_attach_bpf_prog(struct perf_event *event, · 1935 unmodified lines
1936	event->prog = prog;	1936	event->prog = prog;
1937	event->bpf_cookie = bpf_cookie;	1937	event->bpf_cookie = bpf_cookie;
1938	rcu_assign_pointer(event->tp_event->prog_array, new_array);	1938	rcu_assign_pointer(event->tp_event->prog_array, new_array);
1939	bpf_prog_array_free(old_array);	1939	bpf_prog_array_free_sleepable(old_array);
1940		1940
1941	unlock:	1941	unlock:
1942	mutex_unlock(&bpf_event_mutex);	1942	mutex_unlock(&bpf_event_mutex);
@@ -1962,7 +1962,7 @@ void perf_event_detach_bpf_prog(struct perf_event *event) · 19 unmodified lines
1962	bpf_prog_array_delete_safe(old_array, event->prog);	1962	bpf_prog_array_delete_safe(old_array, event->prog);
1963	} else {	1963	} else {
1964	rcu_assign_pointer(event->tp_event->prog_array, new_array);	1964	rcu_assign_pointer(event->tp_event->prog_array, new_array);
1965	bpf_prog_array_free(old_array);	1965	bpf_prog_array_free_sleepable(old_array);
1966	}	1966	}
1967		1967
1968	bpf_prog_put(event->prog);	1968	bpf_prog_put(event->prog);

kernel/trace/trace_uprobe.c+2−3

 #include <linux/namei.h>
 #include <linux/string.h>
 #include <linux/rculist.h>
+#include <linux/filter.h>
 #include "trace_dynevent.h"
 #include "trace_probe.h"
 	if (bpf_prog_array_valid(call)) {
 		u32 ret;
-		preempt_disable();
+		ret = bpf_prog_run_array_sleepable(call->prog_array, regs, bpf_prog_run);
-		ret = trace_call_bpf(call, regs);
-		preempt_enable();
 		if (!ret)
 			return;
 	}

net/bpf/test_run.c+0−6

 	void *data;
 	int ret;
-	if (prog->type != BPF_PROG_TYPE_FLOW_DISSECTOR)
-		return -EINVAL;
 	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
 		return -EINVAL;
 	u32 retval, duration;
 	int ret = -EINVAL;
-	if (prog->type != BPF_PROG_TYPE_SK_LOOKUP)
-		return -EINVAL;
 	if (kattr->test.flags || kattr->test.cpu || kattr->test.batch_size)
 		return -EINVAL;

net/core/filter.c+128−2

@@ -6463,8 +6463,6 @@ static struct sock sk_lookup(struct net net, struct bpf_sock_tuple *tuple, · 6462 unmodified lines
6463		6463
6464	/* bpf_skc_lookup performs the core lookup for different types of sockets,	6464	/* bpf_skc_lookup performs the core lookup for different types of sockets,
6465	* taking a reference on the socket if it doesn't have the flag SOCK_RCU_FREE.	6465	* taking a reference on the socket if it doesn't have the flag SOCK_RCU_FREE.
6466	* Returns the socket as an 'unsigned long' to simplify the casting in the
6467	* callers to satisfy BPF_CALL declarations.
6468	*/	6466	*/
6469	static struct sock *	6467	static struct sock *
6470	__bpf_skc_lookup(struct sk_buff skb, struct bpf_sock_tuple tuple, u32 len,	6468	__bpf_skc_lookup(struct sk_buff skb, struct bpf_sock_tuple tuple, u32 len,
@@ -7444,6 +7442,114 @@ static const struct bpf_func_proto bpf_skb_set_tstamp_proto = { · 973 unmodified lines
7444	.arg3_type = ARG_ANYTHING,	7442	.arg3_type = ARG_ANYTHING,
7445	};	7443	};
7446		7444
		7445	#ifdef CONFIG_SYN_COOKIES
		7446	BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv4, struct iphdr *, iph,
		7447	struct tcphdr *, th, u32, th_len)
		7448	{
		7449	u32 cookie;
		7450	u16 mss;
		7451
		7452	if (unlikely(th_len < sizeof(th) \|\| th_len != th->doff 4))
		7453	return -EINVAL;
		7454
		7455	mss = tcp_parse_mss_option(th, 0) ?: TCP_MSS_DEFAULT;
		7456	cookie = __cookie_v4_init_sequence(iph, th, &mss);
		7457
		7458	return cookie \| ((u64)mss << 32);
		7459	}
		7460
		7461	static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv4_proto = {
		7462	.func = bpf_tcp_raw_gen_syncookie_ipv4,
		7463	.gpl_only = true, /* __cookie_v4_init_sequence() is GPL */
		7464	.pkt_access = true,
		7465	.ret_type = RET_INTEGER,
		7466	.arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
		7467	.arg1_size = sizeof(struct iphdr),
		7468	.arg2_type = ARG_PTR_TO_MEM,
		7469	.arg3_type = ARG_CONST_SIZE,
		7470	};
		7471
		7472	BPF_CALL_3(bpf_tcp_raw_gen_syncookie_ipv6, struct ipv6hdr *, iph,
		7473	struct tcphdr *, th, u32, th_len)
		7474	{
		7475	#if IS_BUILTIN(CONFIG_IPV6)
		7476	const u16 mss_clamp = IPV6_MIN_MTU - sizeof(struct tcphdr) -
		7477	sizeof(struct ipv6hdr);
		7478	u32 cookie;
		7479	u16 mss;
		7480
		7481	if (unlikely(th_len < sizeof(th) \|\| th_len != th->doff 4))
		7482	return -EINVAL;
		7483
		7484	mss = tcp_parse_mss_option(th, 0) ?: mss_clamp;
		7485	cookie = __cookie_v6_init_sequence(iph, th, &mss);
		7486
		7487	return cookie \| ((u64)mss << 32);
		7488	#else
		7489	return -EPROTONOSUPPORT;
		7490	#endif
		7491	}
		7492
		7493	static const struct bpf_func_proto bpf_tcp_raw_gen_syncookie_ipv6_proto = {
		7494	.func = bpf_tcp_raw_gen_syncookie_ipv6,
		7495	.gpl_only = true, /* __cookie_v6_init_sequence() is GPL */
		7496	.pkt_access = true,
		7497	.ret_type = RET_INTEGER,
		7498	.arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
		7499	.arg1_size = sizeof(struct ipv6hdr),
		7500	.arg2_type = ARG_PTR_TO_MEM,
		7501	.arg3_type = ARG_CONST_SIZE,
		7502	};
		7503
		7504	BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv4, struct iphdr *, iph,
		7505	struct tcphdr *, th)
		7506	{
		7507	u32 cookie = ntohl(th->ack_seq) - 1;
		7508
		7509	if (__cookie_v4_check(iph, th, cookie) > 0)
		7510	return 0;
		7511
		7512	return -EACCES;
		7513	}
		7514
		7515	static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv4_proto = {
		7516	.func = bpf_tcp_raw_check_syncookie_ipv4,
		7517	.gpl_only = true, /* __cookie_v4_check is GPL */
		7518	.pkt_access = true,
		7519	.ret_type = RET_INTEGER,
		7520	.arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
		7521	.arg1_size = sizeof(struct iphdr),
		7522	.arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM,
		7523	.arg2_size = sizeof(struct tcphdr),
		7524	};
		7525
		7526	BPF_CALL_2(bpf_tcp_raw_check_syncookie_ipv6, struct ipv6hdr *, iph,
		7527	struct tcphdr *, th)
		7528	{
		7529	#if IS_BUILTIN(CONFIG_IPV6)
		7530	u32 cookie = ntohl(th->ack_seq) - 1;
		7531
		7532	if (__cookie_v6_check(iph, th, cookie) > 0)
		7533	return 0;
		7534
		7535	return -EACCES;
		7536	#else
		7537	return -EPROTONOSUPPORT;
		7538	#endif
		7539	}
		7540
		7541	static const struct bpf_func_proto bpf_tcp_raw_check_syncookie_ipv6_proto = {
		7542	.func = bpf_tcp_raw_check_syncookie_ipv6,
		7543	.gpl_only = true, /* __cookie_v6_check is GPL */
		7544	.pkt_access = true,
		7545	.ret_type = RET_INTEGER,
		7546	.arg1_type = ARG_PTR_TO_FIXED_SIZE_MEM,
		7547	.arg1_size = sizeof(struct ipv6hdr),
		7548	.arg2_type = ARG_PTR_TO_FIXED_SIZE_MEM,
		7549	.arg2_size = sizeof(struct tcphdr),
		7550	};
		7551	#endif /* CONFIG_SYN_COOKIES */
		7552
7447	#endif /* CONFIG_INET */	7553	#endif /* CONFIG_INET */
7448		7554
7449	bool bpf_helper_changes_pkt_data(void *func)	7555	bool bpf_helper_changes_pkt_data(void *func)
@@ -7807,6 +7913,16 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 357 unmodified lines
7807	return &bpf_sk_assign_proto;	7913	return &bpf_sk_assign_proto;
7808	case BPF_FUNC_skb_set_tstamp:	7914	case BPF_FUNC_skb_set_tstamp:
7809	return &bpf_skb_set_tstamp_proto;	7915	return &bpf_skb_set_tstamp_proto;
		7916	#ifdef CONFIG_SYN_COOKIES
		7917	case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
		7918	return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
		7919	case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
		7920	return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
		7921	case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
		7922	return &bpf_tcp_raw_check_syncookie_ipv4_proto;
		7923	case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
		7924	return &bpf_tcp_raw_check_syncookie_ipv6_proto;
		7925	#endif
7810	#endif	7926	#endif
7811	default:	7927	default:
7812	return bpf_sk_base_func_proto(func_id);	7928	return bpf_sk_base_func_proto(func_id);
@@ -7856,6 +7972,16 @@ xdp_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) · 43 unmodified lines
7856	return &bpf_tcp_check_syncookie_proto;	7972	return &bpf_tcp_check_syncookie_proto;
7857	case BPF_FUNC_tcp_gen_syncookie:	7973	case BPF_FUNC_tcp_gen_syncookie:
7858	return &bpf_tcp_gen_syncookie_proto;	7974	return &bpf_tcp_gen_syncookie_proto;
		7975	#ifdef CONFIG_SYN_COOKIES
		7976	case BPF_FUNC_tcp_raw_gen_syncookie_ipv4:
		7977	return &bpf_tcp_raw_gen_syncookie_ipv4_proto;
		7978	case BPF_FUNC_tcp_raw_gen_syncookie_ipv6:
		7979	return &bpf_tcp_raw_gen_syncookie_ipv6_proto;
		7980	case BPF_FUNC_tcp_raw_check_syncookie_ipv4:
		7981	return &bpf_tcp_raw_check_syncookie_ipv4_proto;
		7982	case BPF_FUNC_tcp_raw_check_syncookie_ipv6:
		7983	return &bpf_tcp_raw_check_syncookie_ipv6_proto;
		7984	#endif
7859	#endif	7985	#endif
7860	default:	7986	default:
7861	return bpf_sk_base_func_proto(func_id);	7987	return bpf_sk_base_func_proto(func_id);

net/core/skmsg.c+1−0

@@ -715,6 +715,7 @@ struct sk_psock sk_psock_init(struct sock sk, int node) · 714 unmodified lines
715	psock->eval = __SK_NONE;	715	psock->eval = __SK_NONE;
716	psock->sk_proto = prot;	716	psock->sk_proto = prot;
717	psock->saved_unhash = prot->unhash;	717	psock->saved_unhash = prot->unhash;
		718	psock->saved_destroy = prot->destroy;
718	psock->saved_close = prot->close;	719	psock->saved_close = prot->close;
719	psock->saved_write_space = sk->sk_write_space;	720	psock->saved_write_space = sk->sk_write_space;
720		721

net/core/sock_map.c+23−0

@@ -1561,6 +1561,29 @@ void sock_map_unhash(struct sock *sk) · 1560 unmodified lines
1561	}	1561	}
1562	EXPORT_SYMBOL_GPL(sock_map_unhash);	1562	EXPORT_SYMBOL_GPL(sock_map_unhash);
1563		1563
		1564	void sock_map_destroy(struct sock *sk)
		1565	{
		1566	void (saved_destroy)(struct sock sk);
		1567	struct sk_psock *psock;
		1568
		1569	rcu_read_lock();
		1570	psock = sk_psock_get(sk);
		1571	if (unlikely(!psock)) {
		1572	rcu_read_unlock();
		1573	if (sk->sk_prot->destroy)
		1574	sk->sk_prot->destroy(sk);
		1575	return;
		1576	}
		1577
		1578	saved_destroy = psock->saved_destroy;
		1579	sock_map_remove_links(sk, psock);
		1580	rcu_read_unlock();
		1581	sk_psock_stop(psock, true);
		1582	sk_psock_put(sk, psock);
		1583	saved_destroy(sk);
		1584	}
		1585	EXPORT_SYMBOL_GPL(sock_map_destroy);
		1586
1564	void sock_map_close(struct sock *sk, long timeout)	1587	void sock_map_close(struct sock *sk, long timeout)
1565	{	1588	{
1566	void (saved_close)(struct sock sk, long timeout);	1589	void (saved_close)(struct sock sk, long timeout);

net/ipv4/tcp_bpf.c+1−0

@@ -540,6 +540,7 @@ static void tcp_bpf_rebuild_protos(struct proto prot[TCP_BPF_NUM_CFGS], · 539 unmodified lines
540	struct proto *base)	540	struct proto *base)
541	{	541	{
542	prot[TCP_BPF_BASE] = *base;	542	prot[TCP_BPF_BASE] = *base;
		543	prot[TCP_BPF_BASE].destroy = sock_map_destroy;
543	prot[TCP_BPF_BASE].close = sock_map_close;	544	prot[TCP_BPF_BASE].close = sock_map_close;
544	prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg;	545	prot[TCP_BPF_BASE].recvmsg = tcp_bpf_recvmsg;
545	prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable;	546	prot[TCP_BPF_BASE].sock_is_readable = sk_msg_is_readable;

net/ipv4/tcp_input.c+2−1

 /* Try to parse the MSS option from the TCP header. Return 0 on failure, clamped
  * value on success.
  */
-static u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss)
+u16 tcp_parse_mss_option(const struct tcphdr *th, u16 user_mss)
 {
 	const unsigned char *ptr = (const unsigned char *)(th + 1);
 	int length = (th->doff * 4) - sizeof(struct tcphdr);
 	}
 	return mss;
 }
+EXPORT_SYMBOL_GPL(tcp_parse_mss_option);
 /* Look for tcp options. Normally only called on SYN and SYNACK packets.
  * But, this can also be called on packets in the established flow when

net/xdp/xdp_umem.c+3−3

@@ -57,7 +57,7 @@ static int xdp_umem_addr_map(struct xdp_umem umem, struct page *pages, · 56 unmodified lines
57	static void xdp_umem_release(struct xdp_umem *umem)	57	static void xdp_umem_release(struct xdp_umem *umem)
58	{	58	{
59	umem->zc = false;	59	umem->zc = false;
60	ida_simple_remove(&umem_ida, umem->id);	60	ida_free(&umem_ida, umem->id);
61		61
62	xdp_umem_addr_unmap(umem);	62	xdp_umem_addr_unmap(umem);
63	xdp_umem_unpin_pages(umem);	63	xdp_umem_unpin_pages(umem);
@@ -242,7 +242,7 @@ struct xdp_umem xdp_umem_create(struct xdp_umem_reg mr) · 178 unmodified lines
242	if (!umem)	242	if (!umem)
243	return ERR_PTR(-ENOMEM);	243	return ERR_PTR(-ENOMEM);
244		244
245	err = ida_simple_get(&umem_ida, 0, 0, GFP_KERNEL);	245	err = ida_alloc(&umem_ida, GFP_KERNEL);
246	if (err < 0) {	246	if (err < 0) {
247	kfree(umem);	247	kfree(umem);
248	return ERR_PTR(err);	248	return ERR_PTR(err);
@@ -251,7 +251,7 @@ struct xdp_umem xdp_umem_create(struct xdp_umem_reg mr) · 2 unmodified lines
251		251
252	err = xdp_umem_reg(umem, mr);	252	err = xdp_umem_reg(umem, mr);
253	if (err) {	253	if (err) {
254	ida_simple_remove(&umem_ida, umem->id);	254	ida_free(&umem_ida, umem->id);
255	kfree(umem);	255	kfree(umem);
256	return ERR_PTR(err);	256	return ERR_PTR(err);
257	}	257	}

samples/bpf/xdp_fwd_user.c+49−6

@@ -47,17 +47,60 @@ static int do_attach(int idx, int prog_fd, int map_fd, const char *name) · 46 unmodified lines
47	return err;	47	return err;
48	}	48	}
49		49
50	static int do_detach(int idx, const char *name)	50	static int do_detach(int ifindex, const char ifname, const char app_name)
51	{	51	{
52	int err;	52	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
		53	struct bpf_prog_info prog_info = {};
		54	char prog_name[BPF_OBJ_NAME_LEN];
		55	__u32 info_len, curr_prog_id;
		56	int prog_fd;
		57	int err = 1;
		58
		59	if (bpf_xdp_query_id(ifindex, xdp_flags, &curr_prog_id)) {
		60	printf("ERROR: bpf_xdp_query_id failed (%s)\n",
		61	strerror(errno));
		62	return err;
		63	}
53		64
54	err = bpf_xdp_detach(idx, xdp_flags, NULL);	65	if (!curr_prog_id) {
55	if (err < 0)	66	printf("ERROR: flags(0x%x) xdp prog is not attached to %s\n",
56	printf("ERROR: failed to detach program from %s\n", name);	67	xdp_flags, ifname);
		68	return err;
		69	}
57		70
		71	info_len = sizeof(prog_info);
		72	prog_fd = bpf_prog_get_fd_by_id(curr_prog_id);
		73	if (prog_fd < 0) {
		74	printf("ERROR: bpf_prog_get_fd_by_id failed (%s)\n",
		75	strerror(errno));
		76	return prog_fd;
		77	}
		78
		79	err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
		80	if (err) {
		81	printf("ERROR: bpf_obj_get_info_by_fd failed (%s)\n",
		82	strerror(errno));
		83	goto close_out;
		84	}
		85	snprintf(prog_name, sizeof(prog_name), "%s_prog", app_name);
		86	prog_name[BPF_OBJ_NAME_LEN - 1] = '\0';
		87
		88	if (strcmp(prog_info.name, prog_name)) {
		89	printf("ERROR: %s isn't attached to %s\n", app_name, ifname);
		90	err = 1;
		91	goto close_out;
		92	}
		93
		94	opts.old_prog_fd = prog_fd;
		95	err = bpf_xdp_detach(ifindex, xdp_flags, &opts);
		96	if (err < 0)
		97	printf("ERROR: failed to detach program from %s (%s)\n",
		98	ifname, strerror(errno));
58	/* TODO: Remember to cleanup map, when adding use of shared map	99	/* TODO: Remember to cleanup map, when adding use of shared map
59	* bpf_map_delete_elem((map_fd, &idx);	100	* bpf_map_delete_elem((map_fd, &idx);
60	*/	101	*/
		102	close_out:
		103	close(prog_fd);
61	return err;	104	return err;
62	}	105	}
63		106
@@ -169,7 +212,7 @@ int main(int argc, char **argv) · 105 unmodified lines
169	return 1;	212	return 1;
170	}	213	}
171	if (!attach) {	214	if (!attach) {
172	err = do_detach(idx, argv[i]);	215	err = do_detach(idx, argv[i], prog_name);
173	if (err)	216	if (err)
174	ret = err;	217	ret = err;
175	} else {	218	} else {

samples/bpf/xdp_router_ipv4.bpf.c+9−0

@@ -150,6 +150,15 @@ int xdp_router_ipv4_prog(struct xdp_md *ctx) · 149 unmodified lines
150		150
151	dest_mac = bpf_map_lookup_elem(&arp_table,	151	dest_mac = bpf_map_lookup_elem(&arp_table,
152	&prefix_value->gw);	152	&prefix_value->gw);
		153	if (!dest_mac) {
		154	/* Forward the packet to the kernel in
		155	* order to trigger ARP discovery for
		156	* the default gw.
		157	*/
		158	if (rec)
		159	NO_TEAR_INC(rec->xdp_pass);
		160	return XDP_PASS;
		161	}
153	}	162	}
154	}	163	}
155		164

scripts/bpf_doc.py+4−0

             'struct bpf_timer',
             'struct mptcp_sock',
             'struct bpf_dynptr',
+            'struct iphdr',
+            'struct ipv6hdr',
     ]
     known_types = {
             '...',
             'struct bpf_timer',
             'struct mptcp_sock',
             'struct bpf_dynptr',
+            'struct iphdr',
+            'struct ipv6hdr',
     }
     mapped_types = {
             'u8': '__u8',

tools/bpf/bpftool/Documentation/bpftool-cgroup.rst+11−5

 |	**bpftool** **cgroup help**
 |
 |	*PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* }
-|	*ATTACH_TYPE* := { **ingress** | **egress** | **sock_create** | **sock_ops** | **device** |
+|	*ATTACH_TYPE* := { **cgroup_inet_ingress** | **cgroup_inet_egress** |
-|		**bind4** | **bind6** | **post_bind4** | **post_bind6** | **connect4** | **connect6** |
+|		**cgroup_inet_sock_create** | **cgroup_sock_ops** |
-|		**getpeername4** | **getpeername6** | **getsockname4** | **getsockname6** | **sendmsg4** |
+|		**cgroup_device** | **cgroup_inet4_bind** | **cgroup_inet6_bind** |
-|		**sendmsg6** | **recvmsg4** | **recvmsg6** | **sysctl** | **getsockopt** | **setsockopt** |
+|		**cgroup_inet4_post_bind** | **cgroup_inet6_post_bind** |
-|		**sock_release** }
+|		**cgroup_inet4_connect** | **cgroup_inet6_connect** |
+|		**cgroup_inet4_getpeername** | **cgroup_inet6_getpeername** |
+|		**cgroup_inet4_getsockname** | **cgroup_inet6_getsockname** |
+|		**cgroup_udp4_sendmsg** | **cgroup_udp6_sendmsg** |
+|		**cgroup_udp4_recvmsg** | **cgroup_udp6_recvmsg** |
+|		**cgroup_sysctl** | **cgroup_getsockopt** | **cgroup_setsockopt** |
+|		**cgroup_inet_sock_release** }
 |	*ATTACH_FLAGS* := { **multi** | **override** }
 DESCRIPTION

tools/bpf/bpftool/Documentation/bpftool-prog.rst+3−2

 |		**cgroup/getsockopt** | **cgroup/setsockopt** | **cgroup/sock_release** |
 |		**struct_ops** | **fentry** | **fexit** | **freplace** | **sk_lookup**
 |	}
-|       *ATTACH_TYPE* := {
+|	*ATTACH_TYPE* := {
-|		**msg_verdict** | **skb_verdict** | **stream_verdict** | **stream_parser** | **flow_dissector**
+|		**sk_msg_verdict** | **sk_skb_verdict** | **sk_skb_stream_verdict** |
+|		**sk_skb_stream_parser** | **flow_dissector**
 |	}
 |	*METRICs* := {
 |		**cycles** | **instructions** | **l1d_loads** | **llc_misses** |

tools/bpf/bpftool/Makefile+1−1

@@ -53,7 +53,7 @@ $(LIBBPF_INTERNAL_HDRS): $(LIBBPF_HDRS_DIR)/%.h: $(BPF_DIR)/%.h \| $(LIBBPF_HDRS_ · 52 unmodified lines
53	$(LIBBPF_BOOTSTRAP): $(wildcard $(BPF_DIR)/*.[ch] $(BPF_DIR)/Makefile) \| $(LIBBPF_BOOTSTRAP_OUTPUT)	53	$(LIBBPF_BOOTSTRAP): $(wildcard $(BPF_DIR)/*.[ch] $(BPF_DIR)/Makefile) \| $(LIBBPF_BOOTSTRAP_OUTPUT)
54	$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) \	54	$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) \
55	DESTDIR=$(LIBBPF_BOOTSTRAP_DESTDIR:/=) prefix= \	55	DESTDIR=$(LIBBPF_BOOTSTRAP_DESTDIR:/=) prefix= \
56	ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) $@ install_headers	56	ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) AR=$(HOSTAR) $@ install_headers
57		57
58	$(LIBBPF_BOOTSTRAP_INTERNAL_HDRS): $(LIBBPF_BOOTSTRAP_HDRS_DIR)/%.h: $(BPF_DIR)/%.h \| $(LIBBPF_BOOTSTRAP_HDRS_DIR)	58	$(LIBBPF_BOOTSTRAP_INTERNAL_HDRS): $(LIBBPF_BOOTSTRAP_HDRS_DIR)/%.h: $(BPF_DIR)/%.h \| $(LIBBPF_BOOTSTRAP_HDRS_DIR)
59	$(call QUIET_INSTALL, $@)	59	$(call QUIET_INSTALL, $@)

tools/bpf/bpftool/bash-completion/bpftool+10−8

                             return 0
                             ;;
                         5)
-                            local BPFTOOL_PROG_ATTACH_TYPES='msg_verdict \
+                            local BPFTOOL_PROG_ATTACH_TYPES='sk_msg_verdict \
-                                skb_verdict stream_verdict stream_parser \
+                                sk_skb_verdict sk_skb_stream_verdict sk_skb_stream_parser \
                                 flow_dissector'
                             COMPREPLY=( $( compgen -W "$BPFTOOL_PROG_ATTACH_TYPES" -- "$cur" ) )
                             return 0
                     return 0
                     ;;
                 attach|detach)
-                    local BPFTOOL_CGROUP_ATTACH_TYPES='ingress egress \
+                    local BPFTOOL_CGROUP_ATTACH_TYPES='cgroup_inet_ingress cgroup_inet_egress \
-                        sock_create sock_ops device \
+                        cgroup_inet_sock_create cgroup_sock_ops cgroup_device cgroup_inet4_bind \
-                        bind4 bind6 post_bind4 post_bind6 connect4 connect6 \
+                        cgroup_inet6_bind cgroup_inet4_post_bind cgroup_inet6_post_bind \
-                        getpeername4 getpeername6 getsockname4 getsockname6 \
+                        cgroup_inet4_connect cgroup_inet6_connect cgroup_inet4_getpeername \
-                        sendmsg4 sendmsg6 recvmsg4 recvmsg6 sysctl getsockopt \
+                        cgroup_inet6_getpeername cgroup_inet4_getsockname cgroup_inet6_getsockname \
-                        setsockopt sock_release'
+                        cgroup_udp4_sendmsg cgroup_udp6_sendmsg cgroup_udp4_recvmsg \
+                        cgroup_udp6_recvmsg cgroup_sysctl cgroup_getsockopt cgroup_setsockopt \
+                        cgroup_inet_sock_release'
                     local ATTACH_FLAGS='multi override'
                     local PROG_TYPE='id pinned tag name'
                     # Check for $prev = $command first

tools/bpf/bpftool/btf.c+54−3

@@ -40,6 +40,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { · 39 unmodified lines
40	[BTF_KIND_FLOAT] = "FLOAT",	40	[BTF_KIND_FLOAT] = "FLOAT",
41	[BTF_KIND_DECL_TAG] = "DECL_TAG",	41	[BTF_KIND_DECL_TAG] = "DECL_TAG",
42	[BTF_KIND_TYPE_TAG] = "TYPE_TAG",	42	[BTF_KIND_TYPE_TAG] = "TYPE_TAG",
		43	[BTF_KIND_ENUM64] = "ENUM64",
43	};	44	};
44		45
45	struct btf_attach_point {	46	struct btf_attach_point {
@@ -212,26 +213,76 @@ static int dump_btf_type(const struct btf *btf, __u32 id, · 166 unmodified lines
212	case BTF_KIND_ENUM: {	213	case BTF_KIND_ENUM: {
213	const struct btf_enum v = (const void )(t + 1);	214	const struct btf_enum v = (const void )(t + 1);
214	__u16 vlen = BTF_INFO_VLEN(t->info);	215	__u16 vlen = BTF_INFO_VLEN(t->info);
		216	const char *encoding;
215	int i;	217	int i;
216		218
		219	encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED";
217	if (json_output) {	220	if (json_output) {
		221	jsonw_string_field(w, "encoding", encoding);
218	jsonw_uint_field(w, "size", t->size);	222	jsonw_uint_field(w, "size", t->size);
219	jsonw_uint_field(w, "vlen", vlen);	223	jsonw_uint_field(w, "vlen", vlen);
220	jsonw_name(w, "values");	224	jsonw_name(w, "values");
221	jsonw_start_array(w);	225	jsonw_start_array(w);
222	} else {	226	} else {
223	printf(" size=%u vlen=%u", t->size, vlen);	227	printf(" encoding=%s size=%u vlen=%u", encoding, t->size, vlen);
		228	}
		229	for (i = 0; i < vlen; i++, v++) {
		230	const char *name = btf_str(btf, v->name_off);
		231
		232	if (json_output) {
		233	jsonw_start_object(w);
		234	jsonw_string_field(w, "name", name);
		235	if (btf_kflag(t))
		236	jsonw_int_field(w, "val", v->val);
		237	else
		238	jsonw_uint_field(w, "val", v->val);
		239	jsonw_end_object(w);
		240	} else {
		241	if (btf_kflag(t))
		242	printf("\n\t'%s' val=%d", name, v->val);
		243	else
		244	printf("\n\t'%s' val=%u", name, v->val);
		245	}
		246	}
		247	if (json_output)
		248	jsonw_end_array(w);
		249	break;
		250	}
		251	case BTF_KIND_ENUM64: {
		252	const struct btf_enum64 *v = btf_enum64(t);
		253	__u16 vlen = btf_vlen(t);
		254	const char *encoding;
		255	int i;
		256
		257	encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED";
		258	if (json_output) {
		259	jsonw_string_field(w, "encoding", encoding);
		260	jsonw_uint_field(w, "size", t->size);
		261	jsonw_uint_field(w, "vlen", vlen);
		262	jsonw_name(w, "values");
		263	jsonw_start_array(w);
		264	} else {
		265	printf(" encoding=%s size=%u vlen=%u", encoding, t->size, vlen);
224	}	266	}
225	for (i = 0; i < vlen; i++, v++) {	267	for (i = 0; i < vlen; i++, v++) {
226	const char *name = btf_str(btf, v->name_off);	268	const char *name = btf_str(btf, v->name_off);
		269	__u64 val = ((__u64)v->val_hi32 << 32) \| v->val_lo32;
227		270
228	if (json_output) {	271	if (json_output) {
229	jsonw_start_object(w);	272	jsonw_start_object(w);
230	jsonw_string_field(w, "name", name);	273	jsonw_string_field(w, "name", name);
231	jsonw_uint_field(w, "val", v->val);	274	if (btf_kflag(t))
		275	jsonw_int_field(w, "val", val);
		276	else
		277	jsonw_uint_field(w, "val", val);
232	jsonw_end_object(w);	278	jsonw_end_object(w);
233	} else {	279	} else {
234	printf("\n\t'%s' val=%u", name, v->val);	280	if (btf_kflag(t))
		281	printf("\n\t'%s' val=%lldLL", name,
		282	(unsigned long long)val);
		283	else
		284	printf("\n\t'%s' val=%lluULL", name,
		285	(unsigned long long)val);
235	}	286	}
236	}	287	}
237	if (json_output)	288	if (json_output)

tools/bpf/bpftool/btf_dumper.c+29−0

@@ -182,6 +182,32 @@ static int btf_dumper_enum(const struct btf_dumper *d, · 181 unmodified lines
182	return 0;	182	return 0;
183	}	183	}
184		184
		185	static int btf_dumper_enum64(const struct btf_dumper *d,
		186	const struct btf_type *t,
		187	const void *data)
		188	{
		189	const struct btf_enum64 *enums = btf_enum64(t);
		190	__u32 val_lo32, val_hi32;
		191	__u64 value;
		192	__u16 i;
		193
		194	value = (__u64 )data;
		195	val_lo32 = (__u32)value;
		196	val_hi32 = value >> 32;
		197
		198	for (i = 0; i < btf_vlen(t); i++) {
		199	if (val_lo32 == enums[i].val_lo32 && val_hi32 == enums[i].val_hi32) {
		200	jsonw_string(d->jw,
		201	btf__name_by_offset(d->btf,
		202	enums[i].name_off));
		203	return 0;
		204	}
		205	}
		206
		207	jsonw_int(d->jw, value);
		208	return 0;
		209	}
		210
185	static bool is_str_array(const struct btf btf, const struct btf_array arr,	211	static bool is_str_array(const struct btf btf, const struct btf_array arr,
186	const char *s)	212	const char *s)
187	{	213	{
@@ -542,6 +568,8 @@ static int btf_dumper_do_type(const struct btf_dumper *d, __u32 type_id, · 354 unmodified lines
542	return btf_dumper_array(d, type_id, data);	568	return btf_dumper_array(d, type_id, data);
543	case BTF_KIND_ENUM:	569	case BTF_KIND_ENUM:
544	return btf_dumper_enum(d, t, data);	570	return btf_dumper_enum(d, t, data);
		571	case BTF_KIND_ENUM64:
		572	return btf_dumper_enum64(d, t, data);
545	case BTF_KIND_PTR:	573	case BTF_KIND_PTR:
546	btf_dumper_ptr(d, t, data);	574	btf_dumper_ptr(d, t, data);
547	return 0;	575	return 0;
@@ -618,6 +646,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, · 70 unmodified lines
618	btf__name_by_offset(btf, t->name_off));	646	btf__name_by_offset(btf, t->name_off));
619	break;	647	break;
620	case BTF_KIND_ENUM:	648	case BTF_KIND_ENUM:
		649	case BTF_KIND_ENUM64:
621	BTF_PRINT_ARG("enum %s ",	650	BTF_PRINT_ARG("enum %s ",
622	btf__name_by_offset(btf, t->name_off));	651	btf__name_by_offset(btf, t->name_off));
623	break;	652	break;

tools/bpf/bpftool/cgroup.c+36−17

 #define HELP_SPEC_ATTACH_FLAGS						\
 	"ATTACH_FLAGS := { multi | override }"
-#define HELP_SPEC_ATTACH_TYPES						       \
+#define HELP_SPEC_ATTACH_TYPES						\
-	"       ATTACH_TYPE := { ingress | egress | sock_create |\n"	       \
+	"       ATTACH_TYPE := { cgroup_inet_ingress | cgroup_inet_egress |\n" \
-	"                        sock_ops | device | bind4 | bind6 |\n"	       \
+	"                        cgroup_inet_sock_create | cgroup_sock_ops |\n" \
-	"                        post_bind4 | post_bind6 | connect4 |\n"       \
+	"                        cgroup_device | cgroup_inet4_bind |\n" \
-	"                        connect6 | getpeername4 | getpeername6 |\n"   \
+	"                        cgroup_inet6_bind | cgroup_inet4_post_bind |\n" \
-	"                        getsockname4 | getsockname6 | sendmsg4 |\n"   \
+	"                        cgroup_inet6_post_bind | cgroup_inet4_connect |\n" \
-	"                        sendmsg6 | recvmsg4 | recvmsg6 |\n"           \
+	"                        cgroup_inet6_connect | cgroup_inet4_getpeername |\n" \
-	"                        sysctl | getsockopt | setsockopt |\n"	       \
+	"                        cgroup_inet6_getpeername | cgroup_inet4_getsockname |\n" \
-	"                        sock_release }"
+	"                        cgroup_inet6_getsockname | cgroup_udp4_sendmsg |\n" \
+	"                        cgroup_udp6_sendmsg | cgroup_udp4_recvmsg |\n" \
+	"                        cgroup_udp6_recvmsg | cgroup_sysctl |\n" \
+	"                        cgroup_getsockopt | cgroup_setsockopt |\n" \
+	"                        cgroup_inet_sock_release }"
 static unsigned int query_flags;
 static enum bpf_attach_type parse_attach_type(const char *str)
 {
+	const char *attach_type_str;
 	enum bpf_attach_type type;
-	for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
+	for (type = 0; ; type++) {
-		if (attach_type_name[type] &&
+		attach_type_str = libbpf_bpf_attach_type_str(type);
-		    is_prefix(str, attach_type_name[type]))
+		if (!attach_type_str)
+			break;
+		if (!strcmp(str, attach_type_str))
+			return type;
+	}
+	/* Also check traditionally used attach type strings. For these we keep
+	 * allowing prefixed usage.
+	 */
+	for (type = 0; ; type++) {
+		attach_type_str = bpf_attach_type_input_str(type);
+		if (!attach_type_str)
+			break;
+		if (is_prefix(str, attach_type_str))
 			return type;
 	}
 {
 	char prog_name[MAX_PROG_FULL_NAME];
 	struct bpf_prog_info info = {};
+	const char *attach_type_str;
 	__u32 info_len = sizeof(info);
 	int prog_fd;
 		return -1;
 	}
+	attach_type_str = libbpf_bpf_attach_type_str(attach_type);
 	get_prog_full_name(&info, prog_fd, prog_name, sizeof(prog_name));
 	if (json_output) {
 		jsonw_start_object(json_wtr);
 		jsonw_uint_field(json_wtr, "id", info.id);
-		if (attach_type < ARRAY_SIZE(attach_type_name))
+		if (attach_type_str)
-			jsonw_string_field(json_wtr, "attach_type",
+			jsonw_string_field(json_wtr, "attach_type", attach_type_str);
-					   attach_type_name[attach_type]);
 		else
 			jsonw_uint_field(json_wtr, "attach_type", attach_type);
 		jsonw_string_field(json_wtr, "attach_flags",
 		jsonw_end_object(json_wtr);
 	} else {
 		printf("%s%-8u ", level ? "    " : "", info.id);
-		if (attach_type < ARRAY_SIZE(attach_type_name))
+		if (attach_type_str)
-			printf("%-15s", attach_type_name[attach_type]);
+			printf("%-15s", attach_type_str);
 		else
 			printf("type %-10u", attach_type);
 		printf(" %-15s %-15s\n", attach_flags_str, prog_name);

tools/bpf/bpftool/common.c+44−46

 #include <linux/magic.h>
 #include <net/if.h>
 #include <sys/mount.h>
+#include <sys/resource.h>
 #include <sys/stat.h>
 #include <sys/vfs.h>
 #define BPF_FS_MAGIC		0xcafe4a11
 #endif
-const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = {
-	[BPF_CGROUP_INET_INGRESS]	= "ingress",
-	[BPF_CGROUP_INET_EGRESS]	= "egress",
-	[BPF_CGROUP_INET_SOCK_CREATE]	= "sock_create",
-	[BPF_CGROUP_INET_SOCK_RELEASE]	= "sock_release",
-	[BPF_CGROUP_SOCK_OPS]		= "sock_ops",
-	[BPF_CGROUP_DEVICE]		= "device",
-	[BPF_CGROUP_INET4_BIND]		= "bind4",
-	[BPF_CGROUP_INET6_BIND]		= "bind6",
-	[BPF_CGROUP_INET4_CONNECT]	= "connect4",
-	[BPF_CGROUP_INET6_CONNECT]	= "connect6",
-	[BPF_CGROUP_INET4_POST_BIND]	= "post_bind4",
-	[BPF_CGROUP_INET6_POST_BIND]	= "post_bind6",
-	[BPF_CGROUP_INET4_GETPEERNAME]	= "getpeername4",
-	[BPF_CGROUP_INET6_GETPEERNAME]	= "getpeername6",
-	[BPF_CGROUP_INET4_GETSOCKNAME]	= "getsockname4",
-	[BPF_CGROUP_INET6_GETSOCKNAME]	= "getsockname6",
-	[BPF_CGROUP_UDP4_SENDMSG]	= "sendmsg4",
-	[BPF_CGROUP_UDP6_SENDMSG]	= "sendmsg6",
-	[BPF_CGROUP_SYSCTL]		= "sysctl",
-	[BPF_CGROUP_UDP4_RECVMSG]	= "recvmsg4",
-	[BPF_CGROUP_UDP6_RECVMSG]	= "recvmsg6",
-	[BPF_CGROUP_GETSOCKOPT]		= "getsockopt",
-	[BPF_CGROUP_SETSOCKOPT]		= "setsockopt",
-	[BPF_SK_SKB_STREAM_PARSER]	= "sk_skb_stream_parser",
-	[BPF_SK_SKB_STREAM_VERDICT]	= "sk_skb_stream_verdict",
-	[BPF_SK_SKB_VERDICT]		= "sk_skb_verdict",
-	[BPF_SK_MSG_VERDICT]		= "sk_msg_verdict",
-	[BPF_LIRC_MODE2]		= "lirc_mode2",
-	[BPF_FLOW_DISSECTOR]		= "flow_dissector",
-	[BPF_TRACE_RAW_TP]		= "raw_tp",
-	[BPF_TRACE_FENTRY]		= "fentry",
-	[BPF_TRACE_FEXIT]		= "fexit",
-	[BPF_MODIFY_RETURN]		= "mod_ret",
-	[BPF_LSM_MAC]			= "lsm_mac",
-	[BPF_SK_LOOKUP]			= "sk_lookup",
-	[BPF_TRACE_ITER]		= "trace_iter",
-	[BPF_XDP_DEVMAP]		= "xdp_devmap",
-	[BPF_XDP_CPUMAP]		= "xdp_cpumap",
-	[BPF_XDP]			= "xdp",
-	[BPF_SK_REUSEPORT_SELECT]	= "sk_skb_reuseport_select",
-	[BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]	= "sk_skb_reuseport_select_or_migrate",
-	[BPF_PERF_EVENT]		= "perf_event",
-	[BPF_TRACE_KPROBE_MULTI]	= "trace_kprobe_multi",
-};
 void p_err(const char *fmt, ...)
 {
 	va_list ap;
 	return (unsigned long)st_fs.f_type == BPF_FS_MAGIC;
 }
+void set_max_rlimit(void)
+{
+	struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY };
+	setrlimit(RLIMIT_MEMLOCK, &rinf);
+}
 static int
 mnt_fs(const char *target, const char *type, char *buff, size_t bufflen)
 {
 {
 	return k1 == k2;
 }
+const char *bpf_attach_type_input_str(enum bpf_attach_type t)
+{
+	switch (t) {
+	case BPF_CGROUP_INET_INGRESS:		return "ingress";
+	case BPF_CGROUP_INET_EGRESS:		return "egress";
+	case BPF_CGROUP_INET_SOCK_CREATE:	return "sock_create";
+	case BPF_CGROUP_INET_SOCK_RELEASE:	return "sock_release";
+	case BPF_CGROUP_SOCK_OPS:		return "sock_ops";
+	case BPF_CGROUP_DEVICE:			return "device";
+	case BPF_CGROUP_INET4_BIND:		return "bind4";
+	case BPF_CGROUP_INET6_BIND:		return "bind6";
+	case BPF_CGROUP_INET4_CONNECT:		return "connect4";
+	case BPF_CGROUP_INET6_CONNECT:		return "connect6";
+	case BPF_CGROUP_INET4_POST_BIND:	return "post_bind4";
+	case BPF_CGROUP_INET6_POST_BIND:	return "post_bind6";
+	case BPF_CGROUP_INET4_GETPEERNAME:	return "getpeername4";
+	case BPF_CGROUP_INET6_GETPEERNAME:	return "getpeername6";
+	case BPF_CGROUP_INET4_GETSOCKNAME:	return "getsockname4";
+	case BPF_CGROUP_INET6_GETSOCKNAME:	return "getsockname6";
+	case BPF_CGROUP_UDP4_SENDMSG:		return "sendmsg4";
+	case BPF_CGROUP_UDP6_SENDMSG:		return "sendmsg6";
+	case BPF_CGROUP_SYSCTL:			return "sysctl";
+	case BPF_CGROUP_UDP4_RECVMSG:		return "recvmsg4";
+	case BPF_CGROUP_UDP6_RECVMSG:		return "recvmsg6";
+	case BPF_CGROUP_GETSOCKOPT:		return "getsockopt";
+	case BPF_CGROUP_SETSOCKOPT:		return "setsockopt";
+	case BPF_TRACE_RAW_TP:			return "raw_tp";
+	case BPF_TRACE_FENTRY:			return "fentry";
+	case BPF_TRACE_FEXIT:			return "fexit";
+	case BPF_MODIFY_RETURN:			return "mod_ret";
+	case BPF_SK_REUSEPORT_SELECT:		return "sk_skb_reuseport_select";
+	case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE:	return "sk_skb_reuseport_select_or_migrate";
+	default:	return libbpf_bpf_attach_type_str(t);
+	}
+}

tools/bpf/bpftool/feature.c+55−34

 }
 static void
-probe_prog_type(enum bpf_prog_type prog_type, bool *supported_types,
+probe_prog_type(enum bpf_prog_type prog_type, const char *prog_type_str,
-		const char *define_prefix, __u32 ifindex)
+		bool *supported_types, const char *define_prefix, __u32 ifindex)
 {
 	char feat_name[128], plain_desc[128], define_name[128];
 	const char *plain_comment = "eBPF program_type ";
 	supported_types[prog_type] |= res;
-	if (!prog_type_name[prog_type]) {
-		p_info("program type name not found (type %d)", prog_type);
-		return;
-	}
 	maxlen = sizeof(plain_desc) - strlen(plain_comment) - 1;
-	if (strlen(prog_type_name[prog_type]) > maxlen) {
+	if (strlen(prog_type_str) > maxlen) {
 		p_info("program type name too long");
 		return;
 	}
-	sprintf(feat_name, "have_%s_prog_type", prog_type_name[prog_type]);
+	sprintf(feat_name, "have_%s_prog_type", prog_type_str);
-	sprintf(define_name, "%s_prog_type", prog_type_name[prog_type]);
+	sprintf(define_name, "%s_prog_type", prog_type_str);
 	uppercase(define_name, sizeof(define_name));
-	sprintf(plain_desc, "%s%s", plain_comment, prog_type_name[prog_type]);
+	sprintf(plain_desc, "%s%s", plain_comment, prog_type_str);
 	print_bool_feature(feat_name, plain_desc, define_name, res,
 			   define_prefix);
 }
 }
 static void
-probe_map_type(enum bpf_map_type map_type, const char *define_prefix,
+probe_map_type(enum bpf_map_type map_type, char const *map_type_str,
-	       __u32 ifindex)
+	       const char *define_prefix, __u32 ifindex)
 {
 	char feat_name[128], plain_desc[128], define_name[128];
 	const char *plain_comment = "eBPF map_type ";
 	 * check required for unprivileged users
 	 */
-	if (!map_type_name[map_type]) {
-		p_info("map type name not found (type %d)", map_type);
-		return;
-	}
 	maxlen = sizeof(plain_desc) - strlen(plain_comment) - 1;
-	if (strlen(map_type_name[map_type]) > maxlen) {
+	if (strlen(map_type_str) > maxlen) {
 		p_info("map type name too long");
 		return;
 	}
-	sprintf(feat_name, "have_%s_map_type", map_type_name[map_type]);
+	sprintf(feat_name, "have_%s_map_type", map_type_str);
-	sprintf(define_name, "%s_map_type", map_type_name[map_type]);
+	sprintf(define_name, "%s_map_type", map_type_str);
 	uppercase(define_name, sizeof(define_name));
-	sprintf(plain_desc, "%s%s", plain_comment, map_type_name[map_type]);
+	sprintf(plain_desc, "%s%s", plain_comment, map_type_str);
 	print_bool_feature(feat_name, plain_desc, define_name, res,
 			   define_prefix);
 }
 }
 static void
-probe_helpers_for_progtype(enum bpf_prog_type prog_type, bool supported_type,
+probe_helpers_for_progtype(enum bpf_prog_type prog_type,
+			   const char *prog_type_str, bool supported_type,
 			   const char *define_prefix, __u32 ifindex)
 {
-	const char *ptype_name = prog_type_name[prog_type];
 	char feat_name[128];
 	unsigned int id;
 	bool probe_res = false;
 		}
 	if (json_output) {
-		sprintf(feat_name, "%s_available_helpers", ptype_name);
+		sprintf(feat_name, "%s_available_helpers", prog_type_str);
 		jsonw_name(json_wtr, feat_name);
 		jsonw_start_array(json_wtr);
 	} else if (!define_prefix) {
 		printf("eBPF helpers supported for program type %s:",
-		       ptype_name);
+		       prog_type_str);
 	}
 	for (id = 1; id < ARRAY_SIZE(helper_name); id++) {
 			/* fallthrough */
 		default:
 			probe_res |= probe_helper_for_progtype(prog_type, supported_type,
-						  define_prefix, id, ptype_name,
+						  define_prefix, id, prog_type_str,
 						  ifindex);
 		}
 	}
 section_program_types(bool *supported_types, const char *define_prefix,
 		      __u32 ifindex)
 {
-	unsigned int i;
+	unsigned int prog_type = BPF_PROG_TYPE_UNSPEC;
+	const char *prog_type_str;
 	print_start_section("program_types",
 			    "Scanning eBPF program types...",
 			    "/*** eBPF program types ***/",
 			    define_prefix);
-	for (i = BPF_PROG_TYPE_UNSPEC + 1; i < prog_type_name_size; i++)
+	while (true) {
-		probe_prog_type(i, supported_types, define_prefix, ifindex);
+		prog_type++;
+		prog_type_str = libbpf_bpf_prog_type_str(prog_type);
+		/* libbpf will return NULL for variants unknown to it. */
+		if (!prog_type_str)
+			break;
+		probe_prog_type(prog_type, prog_type_str, supported_types, define_prefix,
+				ifindex);
+	}
 	print_end_section();
 }
 static void section_map_types(const char *define_prefix, __u32 ifindex)
 {
-	unsigned int i;
+	unsigned int map_type = BPF_MAP_TYPE_UNSPEC;
+	const char *map_type_str;
 	print_start_section("map_types",
 			    "Scanning eBPF map types...",
 			    "/*** eBPF map types ***/",
 			    define_prefix);
-	for (i = BPF_MAP_TYPE_UNSPEC + 1; i < map_type_name_size; i++)
+	while (true) {
-		probe_map_type(i, define_prefix, ifindex);
+		map_type++;
+		map_type_str = libbpf_bpf_map_type_str(map_type);
+		/* libbpf will return NULL for variants unknown to it. */
+		if (!map_type_str)
+			break;
+		probe_map_type(map_type, map_type_str, define_prefix, ifindex);
+	}
 	print_end_section();
 }
 static void
 section_helpers(bool *supported_types, const char *define_prefix, __u32 ifindex)
 {
-	unsigned int i;
+	unsigned int prog_type = BPF_PROG_TYPE_UNSPEC;
+	const char *prog_type_str;
 	print_start_section("helpers",
 			    "Scanning eBPF helper functions...",
 		       "	%sBPF__PROG_TYPE_ ## prog_type ## __HELPER_ ## helper\n",
 		       define_prefix, define_prefix, define_prefix,
 		       define_prefix);
-	for (i = BPF_PROG_TYPE_UNSPEC + 1; i < prog_type_name_size; i++)
+	while (true) {
-		probe_helpers_for_progtype(i, supported_types[i], define_prefix,
+		prog_type++;
+		prog_type_str = libbpf_bpf_prog_type_str(prog_type);
+		/* libbpf will return NULL for variants unknown to it. */
+		if (!prog_type_str)
+			break;
+		probe_helpers_for_progtype(prog_type, prog_type_str,
+					   supported_types[prog_type],
+					   define_prefix,
 					   ifindex);
+	}
 	print_end_section();
 }
 	__u32 ifindex = 0;
 	char *ifname;
+	set_max_rlimit();
 	while (argc) {
 		if (is_prefix(*argv, "kernel")) {
 			if (target != COMPONENT_UNSPEC) {

tools/bpf/bpftool/gen.c+4−0

 	const struct btf_type *sec;
 	char map_ident[256], var_ident[256];
+	if (!btf)
+		return;
 	codegen("\
 		\n\
 		__attribute__((unused)) static void			    \n\
 	case BTF_KIND_INT:
 	case BTF_KIND_FLOAT:
 	case BTF_KIND_ENUM:
+	case BTF_KIND_ENUM64:
 	case BTF_KIND_STRUCT:
 	case BTF_KIND_UNION:
 		break;

tools/bpf/bpftool/link.c+31−30

@@ -13,19 +13,6 @@ · 12 unmodified lines
13	#include "json_writer.h"	13	#include "json_writer.h"
14	#include "main.h"	14	#include "main.h"
15		15
16	static const char * const link_type_name[] = {
17	[BPF_LINK_TYPE_UNSPEC] = "unspec",
18	[BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
19	[BPF_LINK_TYPE_TRACING] = "tracing",
20	[BPF_LINK_TYPE_CGROUP] = "cgroup",
21	[BPF_LINK_TYPE_ITER] = "iter",
22	[BPF_LINK_TYPE_NETNS] = "netns",
23	[BPF_LINK_TYPE_XDP] = "xdp",
24	[BPF_LINK_TYPE_PERF_EVENT] = "perf_event",
25	[BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
26	[BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
27	};
28
29	static struct hashmap *link_table;	16	static struct hashmap *link_table;
30		17
31	static int link_parse_fd(int argc, char **argv)	18	static int link_parse_fd(int argc, char **argv)
@@ -67,9 +54,12 @@ static int link_parse_fd(int argc, char **argv) · 35 unmodified lines
67	static void	54	static void
68	show_link_header_json(struct bpf_link_info info, json_writer_t wtr)	55	show_link_header_json(struct bpf_link_info info, json_writer_t wtr)
69	{	56	{
		57	const char *link_type_str;
		58
70	jsonw_uint_field(wtr, "id", info->id);	59	jsonw_uint_field(wtr, "id", info->id);
71	if (info->type < ARRAY_SIZE(link_type_name))	60	link_type_str = libbpf_bpf_link_type_str(info->type);
72	jsonw_string_field(wtr, "type", link_type_name[info->type]);	61	if (link_type_str)
		62	jsonw_string_field(wtr, "type", link_type_str);
73	else	63	else
74	jsonw_uint_field(wtr, "type", info->type);	64	jsonw_uint_field(wtr, "type", info->type);
75		65
@@ -78,9 +68,11 @@ show_link_header_json(struct bpf_link_info info, json_writer_t wtr) · 2 unmodified lines
78		68
79	static void show_link_attach_type_json(__u32 attach_type, json_writer_t *wtr)	69	static void show_link_attach_type_json(__u32 attach_type, json_writer_t *wtr)
80	{	70	{
81	if (attach_type < ARRAY_SIZE(attach_type_name))	71	const char *attach_type_str;
82	jsonw_string_field(wtr, "attach_type",	72
83	attach_type_name[attach_type]);	73	attach_type_str = libbpf_bpf_attach_type_str(attach_type);
		74	if (attach_type_str)
		75	jsonw_string_field(wtr, "attach_type", attach_type_str);
84	else	76	else
85	jsonw_uint_field(wtr, "attach_type", attach_type);	77	jsonw_uint_field(wtr, "attach_type", attach_type);
86	}	78	}
@@ -121,6 +113,7 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info) · 34 unmodified lines
121	static int show_link_close_json(int fd, struct bpf_link_info *info)	113	static int show_link_close_json(int fd, struct bpf_link_info *info)
122	{	114	{
123	struct bpf_prog_info prog_info;	115	struct bpf_prog_info prog_info;
		116	const char *prog_type_str;
124	int err;	117	int err;
125		118
126	jsonw_start_object(json_wtr);	119	jsonw_start_object(json_wtr);
@@ -137,12 +130,12 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) · 10 unmodified lines
137	if (err)	130	if (err)
138	return err;	131	return err;
139		132
140	if (prog_info.type < prog_type_name_size)	133	prog_type_str = libbpf_bpf_prog_type_str(prog_info.type);
141	jsonw_string_field(json_wtr, "prog_type",	134	/* libbpf will return NULL for variants unknown to it. */
142	prog_type_name[prog_info.type]);	135	if (prog_type_str)
		136	jsonw_string_field(json_wtr, "prog_type", prog_type_str);
143	else	137	else
144	jsonw_uint_field(json_wtr, "prog_type",	138	jsonw_uint_field(json_wtr, "prog_type", prog_info.type);
145	prog_info.type);
146		139
147	show_link_attach_type_json(info->tracing.attach_type,	140	show_link_attach_type_json(info->tracing.attach_type,
148	json_wtr);	141	json_wtr);
@@ -184,9 +177,12 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) · 35 unmodified lines
184		177
185	static void show_link_header_plain(struct bpf_link_info *info)	178	static void show_link_header_plain(struct bpf_link_info *info)
186	{	179	{
		180	const char *link_type_str;
		181
187	printf("%u: ", info->id);	182	printf("%u: ", info->id);
188	if (info->type < ARRAY_SIZE(link_type_name))	183	link_type_str = libbpf_bpf_link_type_str(info->type);
189	printf("%s ", link_type_name[info->type]);	184	if (link_type_str)
		185	printf("%s ", link_type_str);
190	else	186	else
191	printf("type %u ", info->type);	187	printf("type %u ", info->type);
192		188
@@ -195,8 +191,11 @@ static void show_link_header_plain(struct bpf_link_info *info) · 2 unmodified lines
195		191
196	static void show_link_attach_type_plain(__u32 attach_type)	192	static void show_link_attach_type_plain(__u32 attach_type)
197	{	193	{
198	if (attach_type < ARRAY_SIZE(attach_type_name))	194	const char *attach_type_str;
199	printf("attach_type %s ", attach_type_name[attach_type]);	195
		196	attach_type_str = libbpf_bpf_attach_type_str(attach_type);
		197	if (attach_type_str)
		198	printf("attach_type %s ", attach_type_str);
200	else	199	else
201	printf("attach_type %u ", attach_type);	200	printf("attach_type %u ", attach_type);
202	}	201	}
@@ -214,6 +213,7 @@ static void show_iter_plain(struct bpf_link_info *info) · 11 unmodified lines
214	static int show_link_close_plain(int fd, struct bpf_link_info *info)	213	static int show_link_close_plain(int fd, struct bpf_link_info *info)
215	{	214	{
216	struct bpf_prog_info prog_info;	215	struct bpf_prog_info prog_info;
		216	const char *prog_type_str;
217	int err;	217	int err;
218		218
219	show_link_header_plain(info);	219	show_link_header_plain(info);
@@ -228,9 +228,10 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) · 8 unmodified lines
228	if (err)	228	if (err)
229	return err;	229	return err;
230		230
231	if (prog_info.type < prog_type_name_size)	231	prog_type_str = libbpf_bpf_prog_type_str(prog_info.type);
232	printf("\n\tprog_type %s ",	232	/* libbpf will return NULL for variants unknown to it. */
233	prog_type_name[prog_info.type]);	233	if (prog_type_str)
		234	printf("\n\tprog_type %s ", prog_type_str);
234	else	235	else
235	printf("\n\tprog_type %u ", prog_info.type);	236	printf("\n\tprog_type %u ", prog_info.type);
236		237

tools/bpf/bpftool/main.c+0−2

@@ -508,8 +508,6 @@ int main(int argc, char **argv) · 507 unmodified lines
508	* mode for loading generated skeleton.	508	* mode for loading generated skeleton.
509	*/	509	*/
510	libbpf_set_strict_mode(LIBBPF_STRICT_ALL & ~LIBBPF_STRICT_MAP_DEFINITIONS);	510	libbpf_set_strict_mode(LIBBPF_STRICT_ALL & ~LIBBPF_STRICT_MAP_DEFINITIONS);
511	} else {
512	libbpf_set_strict_mode(LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK);
513	}	511	}
514		512
515	argc -= optind;	513	argc -= optind;

tools/bpf/bpftool/main.h+16−6

 #define HELP_SPEC_LINK							\
 	"LINK := { id LINK_ID | pinned FILE }"
-extern const char * const prog_type_name[];
-extern const size_t prog_type_name_size;
 extern const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE];
-extern const char * const map_type_name[];
-extern const size_t map_type_name_size;
 /* keep in sync with the definition in skeleton/pid_iter.bpf.c */
 enum bpf_obj_type {
 	BPF_OBJ_UNKNOWN,
 void fprint_hex(FILE *f, void *arg, unsigned int n, const char *sep);
 void usage(void) __noreturn;
+void set_max_rlimit(void);
 int mount_tracefs(const char *target);
 struct obj_ref {
 size_t hash_fn_for_key_as_id(const void *key, void *ctx);
 bool equal_fn_for_key_as_id(const void *k1, const void *k2, void *ctx);
+/* bpf_attach_type_input_str - convert the provided attach type value into a
+ * textual representation that we accept for input purposes.
+ *
+ * This function is similar in nature to libbpf_bpf_attach_type_str, but
+ * recognizes some attach type names that have been used by the program in the
+ * past and which do not follow the string inference scheme that libbpf uses.
+ * These textual representations should only be used for user input.
+ *
+ * @t: The attach type
+ * Returns a pointer to a static string identifying the attach type. NULL is
+ * returned for unknown bpf_attach_type values.
+ */
+const char *bpf_attach_type_input_str(enum bpf_attach_type t);
 static inline void *u32_as_hash_field(__u32 x)
 {
 	return (void *)(uintptr_t)x;

tools/bpf/bpftool/map.c+35−49

@@ -22,42 +22,6 @@ · 21 unmodified lines
22	#include "json_writer.h"	22	#include "json_writer.h"
23	#include "main.h"	23	#include "main.h"
24		24
25	const char * const map_type_name[] = {
26	[BPF_MAP_TYPE_UNSPEC] = "unspec",
27	[BPF_MAP_TYPE_HASH] = "hash",
28	[BPF_MAP_TYPE_ARRAY] = "array",
29	[BPF_MAP_TYPE_PROG_ARRAY] = "prog_array",
30	[BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array",
31	[BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash",
32	[BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array",
33	[BPF_MAP_TYPE_STACK_TRACE] = "stack_trace",
34	[BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array",
35	[BPF_MAP_TYPE_LRU_HASH] = "lru_hash",
36	[BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash",
37	[BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie",
38	[BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps",
39	[BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps",
40	[BPF_MAP_TYPE_DEVMAP] = "devmap",
41	[BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash",
42	[BPF_MAP_TYPE_SOCKMAP] = "sockmap",
43	[BPF_MAP_TYPE_CPUMAP] = "cpumap",
44	[BPF_MAP_TYPE_XSKMAP] = "xskmap",
45	[BPF_MAP_TYPE_SOCKHASH] = "sockhash",
46	[BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage",
47	[BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray",
48	[BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage",
49	[BPF_MAP_TYPE_QUEUE] = "queue",
50	[BPF_MAP_TYPE_STACK] = "stack",
51	[BPF_MAP_TYPE_SK_STORAGE] = "sk_storage",
52	[BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops",
53	[BPF_MAP_TYPE_RINGBUF] = "ringbuf",
54	[BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage",
55	[BPF_MAP_TYPE_TASK_STORAGE] = "task_storage",
56	[BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter",
57	};
58
59	const size_t map_type_name_size = ARRAY_SIZE(map_type_name);
60
61	static struct hashmap *map_table;	25	static struct hashmap *map_table;
62		26
63	static bool map_is_per_cpu(__u32 type)	27	static bool map_is_per_cpu(__u32 type)
@@ -81,12 +45,18 @@ static bool map_is_map_of_progs(__u32 type) · 17 unmodified lines
81		45
82	static int map_type_from_str(const char *type)	46	static int map_type_from_str(const char *type)
83	{	47	{
		48	const char *map_type_str;
84	unsigned int i;	49	unsigned int i;
85		50
86	for (i = 0; i < ARRAY_SIZE(map_type_name); i++)	51	for (i = 0; ; i++) {
		52	map_type_str = libbpf_bpf_map_type_str(i);
		53	if (!map_type_str)
		54	break;
		55
87	/* Don't allow prefixing in case of possible future shadowing */	56	/* Don't allow prefixing in case of possible future shadowing */
88	if (map_type_name[i] && !strcmp(map_type_name[i], type))	57	if (!strcmp(map_type_str, type))
89	return i;	58	return i;
		59	}
90	return -1;	60	return -1;
91	}	61	}
92		62
@@ -472,9 +442,12 @@ static int parse_elem(char *argv, struct bpf_map_info info, · 379 unmodified lines
472		442
473	static void show_map_header_json(struct bpf_map_info info, json_writer_t wtr)	443	static void show_map_header_json(struct bpf_map_info info, json_writer_t wtr)
474	{	444	{
		445	const char *map_type_str;
		446
475	jsonw_uint_field(wtr, "id", info->id);	447	jsonw_uint_field(wtr, "id", info->id);
476	if (info->type < ARRAY_SIZE(map_type_name))	448	map_type_str = libbpf_bpf_map_type_str(info->type);
477	jsonw_string_field(wtr, "type", map_type_name[info->type]);	449	if (map_type_str)
		450	jsonw_string_field(wtr, "type", map_type_str);
478	else	451	else
479	jsonw_uint_field(wtr, "type", info->type);	452	jsonw_uint_field(wtr, "type", info->type);
480		453
@@ -513,10 +486,12 @@ static int show_map_close_json(int fd, struct bpf_map_info *info) · 32 unmodified lines
513		486
514	if (owner_prog_type) {	487	if (owner_prog_type) {
515	unsigned int prog_type = atoi(owner_prog_type);	488	unsigned int prog_type = atoi(owner_prog_type);
		489	const char *prog_type_str;
516		490
517	if (prog_type < prog_type_name_size)	491	prog_type_str = libbpf_bpf_prog_type_str(prog_type);
		492	if (prog_type_str)
518	jsonw_string_field(json_wtr, "owner_prog_type",	493	jsonw_string_field(json_wtr, "owner_prog_type",
519	prog_type_name[prog_type]);	494	prog_type_str);
520	else	495	else
521	jsonw_uint_field(json_wtr, "owner_prog_type",	496	jsonw_uint_field(json_wtr, "owner_prog_type",
522	prog_type);	497	prog_type);
@@ -559,9 +534,13 @@ static int show_map_close_json(int fd, struct bpf_map_info *info) · 36 unmodified lines
559		534
560	static void show_map_header_plain(struct bpf_map_info *info)	535	static void show_map_header_plain(struct bpf_map_info *info)
561	{	536	{
		537	const char *map_type_str;
		538
562	printf("%u: ", info->id);	539	printf("%u: ", info->id);
563	if (info->type < ARRAY_SIZE(map_type_name))	540
564	printf("%s ", map_type_name[info->type]);	541	map_type_str = libbpf_bpf_map_type_str(info->type);
		542	if (map_type_str)
		543	printf("%s ", map_type_str);
565	else	544	else
566	printf("type %u ", info->type);	545	printf("type %u ", info->type);
567		546
@@ -597,10 +576,11 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info) · 29 unmodified lines
597	printf("\n\t");	576	printf("\n\t");
598	if (owner_prog_type) {	577	if (owner_prog_type) {
599	unsigned int prog_type = atoi(owner_prog_type);	578	unsigned int prog_type = atoi(owner_prog_type);
		579	const char *prog_type_str;
600		580
601	if (prog_type < prog_type_name_size)	581	prog_type_str = libbpf_bpf_prog_type_str(prog_type);
602	printf("owner_prog_type %s ",	582	if (prog_type_str)
603	prog_type_name[prog_type]);	583	printf("owner_prog_type %s ", prog_type_str);
604	else	584	else
605	printf("owner_prog_type %d ", prog_type);	585	printf("owner_prog_type %d ", prog_type);
606	}	586	}
@@ -876,9 +856,13 @@ map_dump(int fd, struct bpf_map_info info, json_writer_t wtr, · 269 unmodified lines
876	}	856	}
877		857
878	if (info->type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&	858	if (info->type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
879	info->value_size != 8)	859	info->value_size != 8) {
		860	const char *map_type_str;
		861
		862	map_type_str = libbpf_bpf_map_type_str(info->type);
880	p_info("Warning: cannot read values from %s map with value_size != 8",	863	p_info("Warning: cannot read values from %s map with value_size != 8",
881	map_type_name[info->type]);	864	map_type_str);
		865	}
882	while (true) {	866	while (true) {
883	err = bpf_map_get_next_key(fd, prev_key, key);	867	err = bpf_map_get_next_key(fd, prev_key, key);
884	if (err) {	868	if (err) {
@@ -1342,6 +1326,8 @@ static int do_create(int argc, char **argv) · 457 unmodified lines
1342	goto exit;	1326	goto exit;
1343	}	1327	}
1344		1328
		1329	set_max_rlimit();
		1330
1345	fd = bpf_map_create(map_type, map_name, key_size, value_size, max_entries, &attr);	1331	fd = bpf_map_create(map_type, map_name, key_size, value_size, max_entries, &attr);
1346	if (fd < 0) {	1332	if (fd < 0) {
1347	p_err("map create failed: %s", strerror(errno));	1333	p_err("map create failed: %s", strerror(errno));

tools/bpf/bpftool/pids.c+1−0

@@ -108,6 +108,7 @@ int build_obj_refs_table(struct hashmap **map, enum bpf_obj_type type) · 107 unmodified lines
108	p_err("failed to create hashmap for PID references");	108	p_err("failed to create hashmap for PID references");
109	return -1;	109	return -1;
110	}	110	}
		111	set_max_rlimit();
111		112
112	skel = pid_iter_bpf__open();	113	skel = pid_iter_bpf__open();
113	if (!skel) {	114	if (!skel) {

tools/bpf/bpftool/prog.c+34−45

@@ -36,54 +36,28 @@ · 35 unmodified lines
36	#define BPF_METADATA_PREFIX "bpf_metadata_"	36	#define BPF_METADATA_PREFIX "bpf_metadata_"
37	#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1)	37	#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1)
38		38
39	const char * const prog_type_name[] = {
40	[BPF_PROG_TYPE_UNSPEC] = "unspec",
41	[BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
42	[BPF_PROG_TYPE_KPROBE] = "kprobe",
43	[BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
44	[BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
45	[BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
46	[BPF_PROG_TYPE_XDP] = "xdp",
47	[BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
48	[BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
49	[BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
50	[BPF_PROG_TYPE_LWT_IN] = "lwt_in",
51	[BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
52	[BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
53	[BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
54	[BPF_PROG_TYPE_SK_SKB] = "sk_skb",
55	[BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
56	[BPF_PROG_TYPE_SK_MSG] = "sk_msg",
57	[BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
58	[BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
59	[BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local",
60	[BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
61	[BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
62	[BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
63	[BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",
64	[BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
65	[BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt",
66	[BPF_PROG_TYPE_TRACING] = "tracing",
67	[BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops",
68	[BPF_PROG_TYPE_EXT] = "ext",
69	[BPF_PROG_TYPE_LSM] = "lsm",
70	[BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup",
71	[BPF_PROG_TYPE_SYSCALL] = "syscall",
72	};
73
74	const size_t prog_type_name_size = ARRAY_SIZE(prog_type_name);
75
76	enum dump_mode {	39	enum dump_mode {
77	DUMP_JITED,	40	DUMP_JITED,
78	DUMP_XLATED,	41	DUMP_XLATED,
79	};	42	};
80		43
		44	static const bool attach_types[] = {
		45	[BPF_SK_SKB_STREAM_PARSER] = true,
		46	[BPF_SK_SKB_STREAM_VERDICT] = true,
		47	[BPF_SK_SKB_VERDICT] = true,
		48	[BPF_SK_MSG_VERDICT] = true,
		49	[BPF_FLOW_DISSECTOR] = true,
		50	[__MAX_BPF_ATTACH_TYPE] = false,
		51	};
		52
		53	/* Textual representations traditionally used by the program and kept around
		54	* for the sake of backwards compatibility.
		55	*/
81	static const char * const attach_type_strings[] = {	56	static const char * const attach_type_strings[] = {
82	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",	57	[BPF_SK_SKB_STREAM_PARSER] = "stream_parser",
83	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",	58	[BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict",
84	[BPF_SK_SKB_VERDICT] = "skb_verdict",	59	[BPF_SK_SKB_VERDICT] = "skb_verdict",
85	[BPF_SK_MSG_VERDICT] = "msg_verdict",	60	[BPF_SK_MSG_VERDICT] = "msg_verdict",
86	[BPF_FLOW_DISSECTOR] = "flow_dissector",
87	[__MAX_BPF_ATTACH_TYPE] = NULL,	61	[__MAX_BPF_ATTACH_TYPE] = NULL,
88	};	62	};
89		63
@@ -94,6 +68,14 @@ static enum bpf_attach_type parse_attach_type(const char *str) · 4 unmodified lines
94	enum bpf_attach_type type;	68	enum bpf_attach_type type;
95		69
96	for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {	70	for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) {
		71	if (attach_types[type]) {
		72	const char *attach_type_str;
		73
		74	attach_type_str = libbpf_bpf_attach_type_str(type);
		75	if (!strcmp(str, attach_type_str))
		76	return type;
		77	}
		78
97	if (attach_type_strings[type] &&	79	if (attach_type_strings[type] &&
98	is_prefix(str, attach_type_strings[type]))	80	is_prefix(str, attach_type_strings[type]))
99	return type;	81	return type;
@@ -428,12 +410,14 @@ out_free: · 328 unmodified lines
428		410
429	static void print_prog_header_json(struct bpf_prog_info *info, int fd)	411	static void print_prog_header_json(struct bpf_prog_info *info, int fd)
430	{	412	{
		413	const char *prog_type_str;
431	char prog_name[MAX_PROG_FULL_NAME];	414	char prog_name[MAX_PROG_FULL_NAME];
432		415
433	jsonw_uint_field(json_wtr, "id", info->id);	416	jsonw_uint_field(json_wtr, "id", info->id);
434	if (info->type < ARRAY_SIZE(prog_type_name))	417	prog_type_str = libbpf_bpf_prog_type_str(info->type);
435	jsonw_string_field(json_wtr, "type",	418
436	prog_type_name[info->type]);	419	if (prog_type_str)
		420	jsonw_string_field(json_wtr, "type", prog_type_str);
437	else	421	else
438	jsonw_uint_field(json_wtr, "type", info->type);	422	jsonw_uint_field(json_wtr, "type", info->type);
439		423
@@ -515,11 +499,13 @@ static void print_prog_json(struct bpf_prog_info *info, int fd) · 75 unmodified lines
515		499
516	static void print_prog_header_plain(struct bpf_prog_info *info, int fd)	500	static void print_prog_header_plain(struct bpf_prog_info *info, int fd)
517	{	501	{
		502	const char *prog_type_str;
518	char prog_name[MAX_PROG_FULL_NAME];	503	char prog_name[MAX_PROG_FULL_NAME];
519		504
520	printf("%u: ", info->id);	505	printf("%u: ", info->id);
521	if (info->type < ARRAY_SIZE(prog_type_name))	506	prog_type_str = libbpf_bpf_prog_type_str(info->type);
522	printf("%s ", prog_type_name[info->type]);	507	if (prog_type_str)
		508	printf("%s ", prog_type_str);
523	else	509	else
524	printf("type %u ", info->type);	510	printf("type %u ", info->type);
525		511
@@ -1604,6 +1590,8 @@ static int load_with_options(int argc, char **argv, bool first_prog_only) · 1078 unmodified lines
1604	}	1590	}
1605	}	1591	}
1606		1592
		1593	set_max_rlimit();
		1594
1607	if (verifier_logs)	1595	if (verifier_logs)
1608	/* log_level1 + log_level2 + stats, but not stable UAPI */	1596	/* log_level1 + log_level2 + stats, but not stable UAPI */
1609	open_opts.kernel_log_level = 1 + 2 + 4;	1597	open_opts.kernel_log_level = 1 + 2 + 4;
@@ -2301,6 +2289,7 @@ static int do_profile(int argc, char **argv) · 691 unmodified lines
2301	}	2289	}
2302	}	2290	}
2303		2291
		2292	set_max_rlimit();
2304	err = profiler_bpf__load(profile_obj);	2293	err = profiler_bpf__load(profile_obj);
2305	if (err) {	2294	if (err) {
2306	p_err("failed to load profile_obj");	2295	p_err("failed to load profile_obj");
@@ -2374,8 +2363,8 @@ static int do_help(int argc, char **argv) · 67 unmodified lines
2374	" cgroup/sendmsg6 \| cgroup/recvmsg4 \| cgroup/recvmsg6 \|\n"	2363	" cgroup/sendmsg6 \| cgroup/recvmsg4 \| cgroup/recvmsg6 \|\n"
2375	" cgroup/getsockopt \| cgroup/setsockopt \| cgroup/sock_release \|\n"	2364	" cgroup/getsockopt \| cgroup/setsockopt \| cgroup/sock_release \|\n"
2376	" struct_ops \| fentry \| fexit \| freplace \| sk_lookup }\n"	2365	" struct_ops \| fentry \| fexit \| freplace \| sk_lookup }\n"
2377	" ATTACH_TYPE := { msg_verdict \| skb_verdict \| stream_verdict \|\n"	2366	" ATTACH_TYPE := { sk_msg_verdict \| sk_skb_verdict \| sk_skb_stream_verdict \|\n"
2378	" stream_parser \| flow_dissector }\n"	2367	" sk_skb_stream_parser \| flow_dissector }\n"
2379	" METRIC := { cycles \| instructions \| l1d_loads \| llc_misses \| itlb_misses \| dtlb_misses }\n"	2368	" METRIC := { cycles \| instructions \| l1d_loads \| llc_misses \| itlb_misses \| dtlb_misses }\n"
2380	" " HELP_SPEC_OPTIONS " \|\n"	2369	" " HELP_SPEC_OPTIONS " \|\n"
2381	" {-f\|--bpffs} \| {-m\|--mapcompat} \| {-n\|--nomount} \|\n"	2370	" {-f\|--bpffs} \| {-m\|--mapcompat} \| {-n\|--nomount} \|\n"

tools/bpf/bpftool/struct_ops.c+2−0

@@ -501,6 +501,8 @@ static int do_register(int argc, char **argv) · 500 unmodified lines
501	if (libbpf_get_error(obj))	501	if (libbpf_get_error(obj))
502	return -1;	502	return -1;
503		503
		504	set_max_rlimit();
		505
504	if (bpf_object__load(obj)) {	506	if (bpf_object__load(obj)) {
505	bpf_object__close(obj);	507	bpf_object__close(obj);
506	return -1;	508	return -1;

tools/include/uapi/linux/bpf.h+84−4

@@ -3597,10 +3597,11 @@ union bpf_attr { · 3596 unmodified lines
3597	*	3597	*
3598	* iph points to the start of the IPv4 or IPv6 header, while	3598	* iph points to the start of the IPv4 or IPv6 header, while
3599	* iph_len contains sizeof\ (struct iphdr) or	3599	* iph_len contains sizeof\ (struct iphdr) or
3600	* sizeof\ (struct ip6hdr).	3600	* sizeof\ (struct ipv6hdr).
3601	*	3601	*
3602	* th points to the start of the TCP header, while th_len	3602	* th points to the start of the TCP header, while th_len
3603	* contains sizeof\ (struct tcphdr).	3603	* contains the length of the TCP header (at least
		3604	* sizeof\ (struct tcphdr)).
3604	* Return	3605	* Return
3605	* 0 if iph and th are a valid SYN cookie ACK, or a negative	3606	* 0 if iph and th are a valid SYN cookie ACK, or a negative
3606	* error otherwise.	3607	* error otherwise.
@@ -3783,10 +3784,11 @@ union bpf_attr { · 176 unmodified lines
3783	*	3784	*
3784	* iph points to the start of the IPv4 or IPv6 header, while	3785	* iph points to the start of the IPv4 or IPv6 header, while
3785	* iph_len contains sizeof\ (struct iphdr) or	3786	* iph_len contains sizeof\ (struct iphdr) or
3786	* sizeof\ (struct ip6hdr).	3787	* sizeof\ (struct ipv6hdr).
3787	*	3788	*
3788	* th points to the start of the TCP header, while th_len	3789	* th points to the start of the TCP header, while th_len
3789	* contains the length of the TCP header.	3790	* contains the length of the TCP header with options (at least
		3791	* sizeof\ (struct tcphdr)).
3790	* Return	3792	* Return
3791	* On success, lower 32 bits hold the generated SYN cookie in	3793	* On success, lower 32 bits hold the generated SYN cookie in
3792	* followed by 16 bits which hold the MSS value for that cookie,	3794	* followed by 16 bits which hold the MSS value for that cookie,
@@ -5249,6 +5251,80 @@ union bpf_attr { · 1456 unmodified lines
5249	* Pointer to the underlying dynptr data, NULL if the dynptr is	5251	* Pointer to the underlying dynptr data, NULL if the dynptr is
5250	* read-only, if the dynptr is invalid, or if the offset and length	5252	* read-only, if the dynptr is invalid, or if the offset and length
5251	* is out of bounds.	5253	* is out of bounds.
		5254	*
		5255	* s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr iph, struct tcphdr th, u32 th_len)
		5256	* Description
		5257	* Try to issue a SYN cookie for the packet with corresponding
		5258	* IPv4/TCP headers, iph and th, without depending on a
		5259	* listening socket.
		5260	*
		5261	* iph points to the IPv4 header.
		5262	*
		5263	* th points to the start of the TCP header, while th_len
		5264	* contains the length of the TCP header (at least
		5265	* sizeof\ (struct tcphdr)).
		5266	* Return
		5267	* On success, lower 32 bits hold the generated SYN cookie in
		5268	* followed by 16 bits which hold the MSS value for that cookie,
		5269	* and the top 16 bits are unused.
		5270	*
		5271	* On failure, the returned value is one of the following:
		5272	*
		5273	* -EINVAL if th_len is invalid.
		5274	*
		5275	* s64 bpf_tcp_raw_gen_syncookie_ipv6(struct ipv6hdr iph, struct tcphdr th, u32 th_len)
		5276	* Description
		5277	* Try to issue a SYN cookie for the packet with corresponding
		5278	* IPv6/TCP headers, iph and th, without depending on a
		5279	* listening socket.
		5280	*
		5281	* iph points to the IPv6 header.
		5282	*
		5283	* th points to the start of the TCP header, while th_len
		5284	* contains the length of the TCP header (at least
		5285	* sizeof\ (struct tcphdr)).
		5286	* Return
		5287	* On success, lower 32 bits hold the generated SYN cookie in
		5288	* followed by 16 bits which hold the MSS value for that cookie,
		5289	* and the top 16 bits are unused.
		5290	*
		5291	* On failure, the returned value is one of the following:
		5292	*
		5293	* -EINVAL if th_len is invalid.
		5294	*
		5295	* -EPROTONOSUPPORT if CONFIG_IPV6 is not builtin.
		5296	*
		5297	* long bpf_tcp_raw_check_syncookie_ipv4(struct iphdr iph, struct tcphdr th)
		5298	* Description
		5299	* Check whether iph and th contain a valid SYN cookie ACK
		5300	* without depending on a listening socket.
		5301	*
		5302	* iph points to the IPv4 header.
		5303	*
		5304	* th points to the TCP header.
		5305	* Return
		5306	* 0 if iph and th are a valid SYN cookie ACK.
		5307	*
		5308	* On failure, the returned value is one of the following:
		5309	*
		5310	* -EACCES if the SYN cookie is not valid.
		5311	*
		5312	* long bpf_tcp_raw_check_syncookie_ipv6(struct ipv6hdr iph, struct tcphdr th)
		5313	* Description
		5314	* Check whether iph and th contain a valid SYN cookie ACK
		5315	* without depending on a listening socket.
		5316	*
		5317	* iph points to the IPv6 header.
		5318	*
		5319	* th points to the TCP header.
		5320	* Return
		5321	* 0 if iph and th are a valid SYN cookie ACK.
		5322	*
		5323	* On failure, the returned value is one of the following:
		5324	*
		5325	* -EACCES if the SYN cookie is not valid.
		5326	*
		5327	* -EPROTONOSUPPORT if CONFIG_IPV6 is not builtin.
5252	*/	5328	*/
5253	#define __BPF_FUNC_MAPPER(FN) \	5329	#define __BPF_FUNC_MAPPER(FN) \
5254	FN(unspec), \	5330	FN(unspec), \
@@ -5455,6 +5531,10 @@ union bpf_attr { · 200 unmodified lines
5455	FN(dynptr_read), \	5531	FN(dynptr_read), \
5456	FN(dynptr_write), \	5532	FN(dynptr_write), \
5457	FN(dynptr_data), \	5533	FN(dynptr_data), \
		5534	FN(tcp_raw_gen_syncookie_ipv4), \
		5535	FN(tcp_raw_gen_syncookie_ipv6), \
		5536	FN(tcp_raw_check_syncookie_ipv4), \
		5537	FN(tcp_raw_check_syncookie_ipv6), \
5458	/* */	5538	/* */
5459		5539
5460	/* integer value in 'imm' field of BPF_CALL instruction selects which helper	5540	/* integer value in 'imm' field of BPF_CALL instruction selects which helper

tools/include/uapi/linux/btf.h+14−3

 	 * bits 24-28: kind (e.g. int, ptr, array...etc)
 	 * bits 29-30: unused
 	 * bit     31: kind_flag, currently used by
-	 *             struct, union and fwd
+	 *             struct, union, enum, fwd and enum64
 	 */
 	__u32 info;
-	/* "size" is used by INT, ENUM, STRUCT, UNION and DATASEC.
+	/* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64.
 	 * "size" tells the size of the type it is describing.
 	 *
 	 * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
 	BTF_KIND_ARRAY		= 3,	/* Array	*/
 	BTF_KIND_STRUCT		= 4,	/* Struct	*/
 	BTF_KIND_UNION		= 5,	/* Union	*/
-	BTF_KIND_ENUM		= 6,	/* Enumeration	*/
+	BTF_KIND_ENUM		= 6,	/* Enumeration up to 32-bit values */
 	BTF_KIND_FWD		= 7,	/* Forward	*/
 	BTF_KIND_TYPEDEF	= 8,	/* Typedef	*/
 	BTF_KIND_VOLATILE	= 9,	/* Volatile	*/
 	BTF_KIND_FLOAT		= 16,	/* Floating point	*/
 	BTF_KIND_DECL_TAG	= 17,	/* Decl Tag */
 	BTF_KIND_TYPE_TAG	= 18,	/* Type Tag */
+	BTF_KIND_ENUM64		= 19,	/* Enumeration up to 64-bit values */
 	NR_BTF_KINDS,
 	BTF_KIND_MAX		= NR_BTF_KINDS - 1,
        __s32   component_idx;
 };
+/* BTF_KIND_ENUM64 is followed by multiple "struct btf_enum64".
+ * The exact number of btf_enum64 is stored in the vlen (of the
+ * info in "struct btf_type").
+ */
+struct btf_enum64 {
+	__u32	name_off;
+	__u32	val_lo32;
+	__u32	val_hi32;
+};
 #endif /* _UAPI__LINUX_BTF_H__ */

tools/lib/bpf/btf.c+205−24

@@ -130,7 +130,7 @@ static inline __u64 ptr_to_u64(const void *ptr) · 129 unmodified lines
130		130
131	/* Ensure given dynamically allocated memory region pointed to by data with	131	/* Ensure given dynamically allocated memory region pointed to by data with
132	* capacity of cap_cnt elements each taking elem_sz bytes has enough	132	* capacity of cap_cnt elements each taking elem_sz bytes has enough
133	* memory to accomodate add_cnt new elements, assuming cur_cnt elements	133	* memory to accommodate add_cnt new elements, assuming cur_cnt elements
134	* are already used. At most max_cnt elements can be ever allocated.	134	* are already used. At most max_cnt elements can be ever allocated.
135	* If necessary, memory is reallocated and all existing data is copied over,	135	* If necessary, memory is reallocated and all existing data is copied over,
136	* new pointer to the memory region is stored at *data, new memory region	136	* new pointer to the memory region is stored at *data, new memory region
@@ -305,6 +305,8 @@ static int btf_type_size(const struct btf_type *t) · 168 unmodified lines
305	return base_size + sizeof(__u32);	305	return base_size + sizeof(__u32);
306	case BTF_KIND_ENUM:	306	case BTF_KIND_ENUM:
307	return base_size + vlen * sizeof(struct btf_enum);	307	return base_size + vlen * sizeof(struct btf_enum);
		308	case BTF_KIND_ENUM64:
		309	return base_size + vlen * sizeof(struct btf_enum64);
308	case BTF_KIND_ARRAY:	310	case BTF_KIND_ARRAY:
309	return base_size + sizeof(struct btf_array);	311	return base_size + sizeof(struct btf_array);
310	case BTF_KIND_STRUCT:	312	case BTF_KIND_STRUCT:
@@ -334,6 +336,7 @@ static void btf_bswap_type_base(struct btf_type *t) · 23 unmodified lines
334	static int btf_bswap_type_rest(struct btf_type *t)	336	static int btf_bswap_type_rest(struct btf_type *t)
335	{	337	{
336	struct btf_var_secinfo *v;	338	struct btf_var_secinfo *v;
		339	struct btf_enum64 *e64;
337	struct btf_member *m;	340	struct btf_member *m;
338	struct btf_array *a;	341	struct btf_array *a;
339	struct btf_param *p;	342	struct btf_param *p;
@@ -361,6 +364,13 @@ static int btf_bswap_type_rest(struct btf_type *t) · 21 unmodified lines
361	e->val = bswap_32(e->val);	364	e->val = bswap_32(e->val);
362	}	365	}
363	return 0;	366	return 0;
		367	case BTF_KIND_ENUM64:
		368	for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) {
		369	e64->name_off = bswap_32(e64->name_off);
		370	e64->val_lo32 = bswap_32(e64->val_lo32);
		371	e64->val_hi32 = bswap_32(e64->val_hi32);
		372	}
		373	return 0;
364	case BTF_KIND_ARRAY:	374	case BTF_KIND_ARRAY:
365	a = btf_array(t);	375	a = btf_array(t);
366	a->type = bswap_32(a->type);	376	a->type = bswap_32(a->type);
@@ -472,9 +482,22 @@ const struct btf_type btf__type_by_id(const struct btf btf, __u32 type_id) · 105 unmodified lines
472		482
473	static int determine_ptr_size(const struct btf *btf)	483	static int determine_ptr_size(const struct btf *btf)
474	{	484	{
		485	static const char * const long_aliases[] = {
		486	"long",
		487	"long int",
		488	"int long",
		489	"unsigned long",
		490	"long unsigned",
		491	"unsigned long int",
		492	"unsigned int long",
		493	"long unsigned int",
		494	"long int unsigned",
		495	"int unsigned long",
		496	"int long unsigned",
		497	};
475	const struct btf_type *t;	498	const struct btf_type *t;
476	const char *name;	499	const char *name;
477	int i, n;	500	int i, j, n;
478		501
479	if (btf->base_btf && btf->base_btf->ptr_sz > 0)	502	if (btf->base_btf && btf->base_btf->ptr_sz > 0)
480	return btf->base_btf->ptr_sz;	503	return btf->base_btf->ptr_sz;
@@ -485,15 +508,16 @@ static int determine_ptr_size(const struct btf *btf) · 4 unmodified lines
485	if (!btf_is_int(t))	508	if (!btf_is_int(t))
486	continue;	509	continue;
487		510
		511	if (t->size != 4 && t->size != 8)
		512	continue;
		513
488	name = btf__name_by_offset(btf, t->name_off);	514	name = btf__name_by_offset(btf, t->name_off);
489	if (!name)	515	if (!name)
490	continue;	516	continue;
491		517
492	if (strcmp(name, "long int") == 0 \|\|	518	for (j = 0; j < ARRAY_SIZE(long_aliases); j++) {
493	strcmp(name, "long unsigned int") == 0) {	519	if (strcmp(name, long_aliases[j]) == 0)
494	if (t->size != 4 && t->size != 8)	520	return t->size;
495	continue;
496	return t->size;
497	}	521	}
498	}	522	}
499		523
@@ -597,6 +621,7 @@ __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) · 97 unmodified lines
597	case BTF_KIND_STRUCT:	621	case BTF_KIND_STRUCT:
598	case BTF_KIND_UNION:	622	case BTF_KIND_UNION:
599	case BTF_KIND_ENUM:	623	case BTF_KIND_ENUM:
		624	case BTF_KIND_ENUM64:
600	case BTF_KIND_DATASEC:	625	case BTF_KIND_DATASEC:
601	case BTF_KIND_FLOAT:	626	case BTF_KIND_FLOAT:
602	size = t->size;	627	size = t->size;
@@ -644,6 +669,7 @@ int btf__align_of(const struct btf *btf, __u32 id) · 41 unmodified lines
644	switch (kind) {	669	switch (kind) {
645	case BTF_KIND_INT:	670	case BTF_KIND_INT:
646	case BTF_KIND_ENUM:	671	case BTF_KIND_ENUM:
		672	case BTF_KIND_ENUM64:
647	case BTF_KIND_FLOAT:	673	case BTF_KIND_FLOAT:
648	return min(btf_ptr_sz(btf), (size_t)t->size);	674	return min(btf_ptr_sz(btf), (size_t)t->size);
649	case BTF_KIND_PTR:	675	case BTF_KIND_PTR:
@@ -2115,20 +2141,8 @@ int btf__add_field(struct btf btf, const char name, int type_id, · 1465 unmodified lines
2115	return 0;	2141	return 0;
2116	}	2142	}
2117		2143
2118	/*	2144	static int btf_add_enum_common(struct btf btf, const char name, __u32 byte_sz,
2119	* Append new BTF_KIND_ENUM type with:	2145	bool is_signed, __u8 kind)
2120	* - name - name of the enum, can be NULL or empty for anonymous enums;
2121	* - byte_sz - size of the enum, in bytes.
2122	*
2123	* Enum initially has no enum values in it (and corresponds to enum forward
2124	* declaration). Enumerator values can be added by btf__add_enum_value()
2125	* immediately after btf__add_enum() succeeds.
2126	*
2127	* Returns:
2128	* - >0, type ID of newly added BTF type;
2129	* - <0, on error.
2130	*/
2131	int btf__add_enum(struct btf btf, const char name, __u32 byte_sz)
2132	{	2146	{
2133	struct btf_type *t;	2147	struct btf_type *t;
2134	int sz, name_off = 0;	2148	int sz, name_off = 0;
@@ -2153,12 +2167,34 @@ int btf__add_enum(struct btf btf, const char name, __u32 byte_sz) · 18 unmodified lines
2153		2167
2154	/* start out with vlen=0; it will be adjusted when adding enum values */	2168	/* start out with vlen=0; it will be adjusted when adding enum values */
2155	t->name_off = name_off;	2169	t->name_off = name_off;
2156	t->info = btf_type_info(BTF_KIND_ENUM, 0, 0);	2170	t->info = btf_type_info(kind, 0, is_signed);
2157	t->size = byte_sz;	2171	t->size = byte_sz;
2158		2172
2159	return btf_commit_type(btf, sz);	2173	return btf_commit_type(btf, sz);
2160	}	2174	}
2161		2175
		2176	/*
		2177	* Append new BTF_KIND_ENUM type with:
		2178	* - name - name of the enum, can be NULL or empty for anonymous enums;
		2179	* - byte_sz - size of the enum, in bytes.
		2180	*
		2181	* Enum initially has no enum values in it (and corresponds to enum forward
		2182	* declaration). Enumerator values can be added by btf__add_enum_value()
		2183	* immediately after btf__add_enum() succeeds.
		2184	*
		2185	* Returns:
		2186	* - >0, type ID of newly added BTF type;
		2187	* - <0, on error.
		2188	*/
		2189	int btf__add_enum(struct btf btf, const char name, __u32 byte_sz)
		2190	{
		2191	/*
		2192	* set the signedness to be unsigned, it will change to signed
		2193	* if any later enumerator is negative.
		2194	*/
		2195	return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM);
		2196	}
		2197
2162	/*	2198	/*
2163	* Append new enum value for the current ENUM type with:	2199	* Append new enum value for the current ENUM type with:
2164	* - name - name of the enumerator value, can't be NULL or empty;	2200	* - name - name of the enumerator value, can't be NULL or empty;
@@ -2206,6 +2242,82 @@ int btf__add_enum_value(struct btf btf, const char name, __s64 value) · 41 unmodified lines
2206	t = btf_last_type(btf);	2242	t = btf_last_type(btf);
2207	btf_type_inc_vlen(t);	2243	btf_type_inc_vlen(t);
2208		2244
		2245	/* if negative value, set signedness to signed */
		2246	if (value < 0)
		2247	t->info = btf_type_info(btf_kind(t), btf_vlen(t), true);
		2248
		2249	btf->hdr->type_len += sz;
		2250	btf->hdr->str_off += sz;
		2251	return 0;
		2252	}
		2253
		2254	/*
		2255	* Append new BTF_KIND_ENUM64 type with:
		2256	* - name - name of the enum, can be NULL or empty for anonymous enums;
		2257	* - byte_sz - size of the enum, in bytes.
		2258	* - is_signed - whether the enum values are signed or not;
		2259	*
		2260	* Enum initially has no enum values in it (and corresponds to enum forward
		2261	* declaration). Enumerator values can be added by btf__add_enum64_value()
		2262	* immediately after btf__add_enum64() succeeds.
		2263	*
		2264	* Returns:
		2265	* - >0, type ID of newly added BTF type;
		2266	* - <0, on error.
		2267	*/
		2268	int btf__add_enum64(struct btf btf, const char name, __u32 byte_sz,
		2269	bool is_signed)
		2270	{
		2271	return btf_add_enum_common(btf, name, byte_sz, is_signed,
		2272	BTF_KIND_ENUM64);
		2273	}
		2274
		2275	/*
		2276	* Append new enum value for the current ENUM64 type with:
		2277	* - name - name of the enumerator value, can't be NULL or empty;
		2278	* - value - integer value corresponding to enum value name;
		2279	* Returns:
		2280	* - 0, on success;
		2281	* - <0, on error.
		2282	*/
		2283	int btf__add_enum64_value(struct btf btf, const char name, __u64 value)
		2284	{
		2285	struct btf_enum64 *v;
		2286	struct btf_type *t;
		2287	int sz, name_off;
		2288
		2289	/* last type should be BTF_KIND_ENUM64 */
		2290	if (btf->nr_types == 0)
		2291	return libbpf_err(-EINVAL);
		2292	t = btf_last_type(btf);
		2293	if (!btf_is_enum64(t))
		2294	return libbpf_err(-EINVAL);
		2295
		2296	/* non-empty name */
		2297	if (!name \|\| !name[0])
		2298	return libbpf_err(-EINVAL);
		2299
		2300	/* decompose and invalidate raw data */
		2301	if (btf_ensure_modifiable(btf))
		2302	return libbpf_err(-ENOMEM);
		2303
		2304	sz = sizeof(struct btf_enum64);
		2305	v = btf_add_type_mem(btf, sz);
		2306	if (!v)
		2307	return libbpf_err(-ENOMEM);
		2308
		2309	name_off = btf__add_str(btf, name);
		2310	if (name_off < 0)
		2311	return name_off;
		2312
		2313	v->name_off = name_off;
		2314	v->val_lo32 = (__u32)value;
		2315	v->val_hi32 = value >> 32;
		2316
		2317	/* update parent type's vlen */
		2318	t = btf_last_type(btf);
		2319	btf_type_inc_vlen(t);
		2320
2209	btf->hdr->type_len += sz;	2321	btf->hdr->type_len += sz;
2210	btf->hdr->str_off += sz;	2322	btf->hdr->str_off += sz;
2211	return 0;	2323	return 0;
@@ -3470,7 +3582,7 @@ static bool btf_equal_int_tag(struct btf_type t1, struct btf_type t2) · 1258 unmodified lines
3470	return info1 == info2;	3582	return info1 == info2;
3471	}	3583	}
3472		3584
3473	/* Calculate type signature hash of ENUM. */	3585	/* Calculate type signature hash of ENUM/ENUM64. */
3474	static long btf_hash_enum(struct btf_type *t)	3586	static long btf_hash_enum(struct btf_type *t)
3475	{	3587	{
3476	long h;	3588	long h;
@@ -3504,9 +3616,31 @@ static bool btf_equal_enum(struct btf_type t1, struct btf_type t2) · 27 unmodified lines
3504	return true;	3616	return true;
3505	}	3617	}
3506		3618
		3619	static bool btf_equal_enum64(struct btf_type t1, struct btf_type t2)
		3620	{
		3621	const struct btf_enum64 m1, m2;
		3622	__u16 vlen;
		3623	int i;
		3624
		3625	if (!btf_equal_common(t1, t2))
		3626	return false;
		3627
		3628	vlen = btf_vlen(t1);
		3629	m1 = btf_enum64(t1);
		3630	m2 = btf_enum64(t2);
		3631	for (i = 0; i < vlen; i++) {
		3632	if (m1->name_off != m2->name_off \|\| m1->val_lo32 != m2->val_lo32 \|\|
		3633	m1->val_hi32 != m2->val_hi32)
		3634	return false;
		3635	m1++;
		3636	m2++;
		3637	}
		3638	return true;
		3639	}
		3640
3507	static inline bool btf_is_enum_fwd(struct btf_type *t)	3641	static inline bool btf_is_enum_fwd(struct btf_type *t)
3508	{	3642	{
3509	return btf_is_enum(t) && btf_vlen(t) == 0;	3643	return btf_is_any_enum(t) && btf_vlen(t) == 0;
3510	}	3644	}
3511		3645
3512	static bool btf_compat_enum(struct btf_type t1, struct btf_type t2)	3646	static bool btf_compat_enum(struct btf_type t1, struct btf_type t2)
@@ -3519,6 +3653,17 @@ static bool btf_compat_enum(struct btf_type t1, struct btf_type t2) · 6 unmodified lines
3519	t1->size == t2->size;	3653	t1->size == t2->size;
3520	}	3654	}
3521		3655
		3656	static bool btf_compat_enum64(struct btf_type t1, struct btf_type t2)
		3657	{
		3658	if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2))
		3659	return btf_equal_enum64(t1, t2);
		3660
		3661	/* ignore vlen when comparing */
		3662	return t1->name_off == t2->name_off &&
		3663	(t1->info & ~0xffff) == (t2->info & ~0xffff) &&
		3664	t1->size == t2->size;
		3665	}
		3666
3522	/*	3667	/*
3523	* Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs,	3668	* Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs,
3524	* as referenced type IDs equivalence is established separately during type	3669	* as referenced type IDs equivalence is established separately during type
@@ -3731,6 +3876,7 @@ static int btf_dedup_prep(struct btf_dedup *d) · 206 unmodified lines
3731	h = btf_hash_int_decl_tag(t);	3876	h = btf_hash_int_decl_tag(t);
3732	break;	3877	break;
3733	case BTF_KIND_ENUM:	3878	case BTF_KIND_ENUM:
		3879	case BTF_KIND_ENUM64:
3734	h = btf_hash_enum(t);	3880	h = btf_hash_enum(t);
3735	break;	3881	break;
3736	case BTF_KIND_STRUCT:	3882	case BTF_KIND_STRUCT:
@@ -3820,6 +3966,27 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) · 83 unmodified lines
3820	}	3966	}
3821	break;	3967	break;
3822		3968
		3969	case BTF_KIND_ENUM64:
		3970	h = btf_hash_enum(t);
		3971	for_each_dedup_cand(d, hash_entry, h) {
		3972	cand_id = (__u32)(long)hash_entry->value;
		3973	cand = btf_type_by_id(d->btf, cand_id);
		3974	if (btf_equal_enum64(t, cand)) {
		3975	new_id = cand_id;
		3976	break;
		3977	}
		3978	if (btf_compat_enum64(t, cand)) {
		3979	if (btf_is_enum_fwd(t)) {
		3980	/* resolve fwd to full enum */
		3981	new_id = cand_id;
		3982	break;
		3983	}
		3984	/* resolve canonical enum fwd to full enum */
		3985	d->map[cand_id] = type_id;
		3986	}
		3987	}
		3988	break;
		3989
3823	case BTF_KIND_FWD:	3990	case BTF_KIND_FWD:
3824	case BTF_KIND_FLOAT:	3991	case BTF_KIND_FLOAT:
3825	h = btf_hash_common(t);	3992	h = btf_hash_common(t);
@@ -4115,6 +4282,9 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, · 289 unmodified lines
4115	case BTF_KIND_ENUM:	4282	case BTF_KIND_ENUM:
4116	return btf_compat_enum(cand_type, canon_type);	4283	return btf_compat_enum(cand_type, canon_type);
4117		4284
		4285	case BTF_KIND_ENUM64:
		4286	return btf_compat_enum64(cand_type, canon_type);
		4287
4118	case BTF_KIND_FWD:	4288	case BTF_KIND_FWD:
4119	case BTF_KIND_FLOAT:	4289	case BTF_KIND_FLOAT:
4120	return btf_equal_common(cand_type, canon_type);	4290	return btf_equal_common(cand_type, canon_type);
@@ -4717,6 +4887,7 @@ int btf_type_visit_type_ids(struct btf_type t, type_id_visit_fn visit, void ct · 596 unmodified lines
4717	case BTF_KIND_INT:	4887	case BTF_KIND_INT:
4718	case BTF_KIND_FLOAT:	4888	case BTF_KIND_FLOAT:
4719	case BTF_KIND_ENUM:	4889	case BTF_KIND_ENUM:
		4890	case BTF_KIND_ENUM64:
4720	return 0;	4891	return 0;
4721		4892
4722	case BTF_KIND_FWD:	4893	case BTF_KIND_FWD:
@@ -4811,6 +4982,16 @@ int btf_type_visit_str_offs(struct btf_type t, str_off_visit_fn visit, void ct · 88 unmodified lines
4811	}	4982	}
4812	break;	4983	break;
4813	}	4984	}
		4985	case BTF_KIND_ENUM64: {
		4986	struct btf_enum64 *m = btf_enum64(t);
		4987
		4988	for (i = 0, n = btf_vlen(t); i < n; i++, m++) {
		4989	err = visit(&m->name_off, ctx);
		4990	if (err)
		4991	return err;
		4992	}
		4993	break;
		4994	}
4814	case BTF_KIND_FUNC_PROTO: {	4995	case BTF_KIND_FUNC_PROTO: {
4815	struct btf_param *m = btf_params(t);	4996	struct btf_param *m = btf_params(t);
4816		4997

tools/lib/bpf/btf.h+31−1

 /* enum construction APIs */
 LIBBPF_API int btf__add_enum(struct btf *btf, const char *name, __u32 bytes_sz);
 LIBBPF_API int btf__add_enum_value(struct btf *btf, const char *name, __s64 value);
+LIBBPF_API int btf__add_enum64(struct btf *btf, const char *name, __u32 bytes_sz, bool is_signed);
+LIBBPF_API int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value);
 enum btf_fwd_kind {
 	BTF_FWD_STRUCT = 0,
 #ifndef BTF_KIND_FLOAT
 #define BTF_KIND_FLOAT		16	/* Floating point	*/
 #endif
-/* The kernel header switched to enums, so these two were never #defined */
+/* The kernel header switched to enums, so the following were never #defined */
 #define BTF_KIND_DECL_TAG	17	/* Decl Tag */
 #define BTF_KIND_TYPE_TAG	18	/* Type Tag */
+#define BTF_KIND_ENUM64		19	/* Enum for up-to 64bit values */
 static inline __u16 btf_kind(const struct btf_type *t)
 {
 	return btf_kind(t) == BTF_KIND_ENUM;
 }
+static inline bool btf_is_enum64(const struct btf_type *t)
+{
+	return btf_kind(t) == BTF_KIND_ENUM64;
+}
 static inline bool btf_is_fwd(const struct btf_type *t)
 {
 	return btf_kind(t) == BTF_KIND_FWD;
 	return btf_kind(t) == BTF_KIND_TYPE_TAG;
 }
+static inline bool btf_is_any_enum(const struct btf_type *t)
+{
+	return btf_is_enum(t) || btf_is_enum64(t);
+}
+static inline bool btf_kind_core_compat(const struct btf_type *t1,
+					const struct btf_type *t2)
+{
+	return btf_kind(t1) == btf_kind(t2) ||
+	       (btf_is_any_enum(t1) && btf_is_any_enum(t2));
+}
 static inline __u8 btf_int_encoding(const struct btf_type *t)
 {
 	return BTF_INT_ENCODING(*(__u32 *)(t + 1));
 	return (struct btf_enum *)(t + 1);
 }
+static inline struct btf_enum64 *btf_enum64(const struct btf_type *t)
+{
+	return (struct btf_enum64 *)(t + 1);
+}
+static inline __u64 btf_enum64_value(const struct btf_enum64 *e)
+{
+	return ((__u64)e->val_hi32 << 32) | e->val_lo32;
+}
 static inline struct btf_member *btf_members(const struct btf_type *t)
 {
 	return (struct btf_member *)(t + 1);

tools/lib/bpf/btf_dump.c+103−34

@@ -318,6 +318,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d) · 317 unmodified lines
318	switch (btf_kind(t)) {	318	switch (btf_kind(t)) {
319	case BTF_KIND_INT:	319	case BTF_KIND_INT:
320	case BTF_KIND_ENUM:	320	case BTF_KIND_ENUM:
		321	case BTF_KIND_ENUM64:
321	case BTF_KIND_FWD:	322	case BTF_KIND_FWD:
322	case BTF_KIND_FLOAT:	323	case BTF_KIND_FLOAT:
323	break;	324	break;
@@ -538,6 +539,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) · 214 unmodified lines
538	return 1;	539	return 1;
539	}	540	}
540	case BTF_KIND_ENUM:	541	case BTF_KIND_ENUM:
		542	case BTF_KIND_ENUM64:
541	case BTF_KIND_FWD:	543	case BTF_KIND_FWD:
542	/*	544	/*
543	* non-anonymous or non-referenced enums are top-level	545	* non-anonymous or non-referenced enums are top-level
@@ -739,6 +741,7 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) · 195 unmodified lines
739	tstate->emit_state = EMITTED;	741	tstate->emit_state = EMITTED;
740	break;	742	break;
741	case BTF_KIND_ENUM:	743	case BTF_KIND_ENUM:
		744	case BTF_KIND_ENUM64:
742	if (top_level_def) {	745	if (top_level_def) {
743	btf_dump_emit_enum_def(d, id, t, 0);	746	btf_dump_emit_enum_def(d, id, t, 0);
744	btf_dump_printf(d, ";\n\n");	747	btf_dump_printf(d, ";\n\n");
@@ -989,38 +992,81 @@ static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id, · 244 unmodified lines
989	btf_dump_printf(d, "enum %s", btf_dump_type_name(d, id));	992	btf_dump_printf(d, "enum %s", btf_dump_type_name(d, id));
990	}	993	}
991		994
992	static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id,	995	static void btf_dump_emit_enum32_val(struct btf_dump *d,
993	const struct btf_type *t,	996	const struct btf_type *t,
994	int lvl)	997	int lvl, __u16 vlen)
995	{	998	{
996	const struct btf_enum *v = btf_enum(t);	999	const struct btf_enum *v = btf_enum(t);
997	__u16 vlen = btf_vlen(t);	1000	bool is_signed = btf_kflag(t);
		1001	const char *fmt_str;
998	const char *name;	1002	const char *name;
999	size_t dup_cnt;	1003	size_t dup_cnt;
1000	int i;	1004	int i;
1001		1005
		1006	for (i = 0; i < vlen; i++, v++) {
		1007	name = btf_name_of(d, v->name_off);
		1008	/* enumerators share namespace with typedef idents */
		1009	dup_cnt = btf_dump_name_dups(d, d->ident_names, name);
		1010	if (dup_cnt > 1) {
		1011	fmt_str = is_signed ? "\n%s%s___%zd = %d," : "\n%s%s___%zd = %u,";
		1012	btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, dup_cnt, v->val);
		1013	} else {
		1014	fmt_str = is_signed ? "\n%s%s = %d," : "\n%s%s = %u,";
		1015	btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, v->val);
		1016	}
		1017	}
		1018	}
		1019
		1020	static void btf_dump_emit_enum64_val(struct btf_dump *d,
		1021	const struct btf_type *t,
		1022	int lvl, __u16 vlen)
		1023	{
		1024	const struct btf_enum64 *v = btf_enum64(t);
		1025	bool is_signed = btf_kflag(t);
		1026	const char *fmt_str;
		1027	const char *name;
		1028	size_t dup_cnt;
		1029	__u64 val;
		1030	int i;
		1031
		1032	for (i = 0; i < vlen; i++, v++) {
		1033	name = btf_name_of(d, v->name_off);
		1034	dup_cnt = btf_dump_name_dups(d, d->ident_names, name);
		1035	val = btf_enum64_value(v);
		1036	if (dup_cnt > 1) {
		1037	fmt_str = is_signed ? "\n%s%s___%zd = %lldLL,"
		1038	: "\n%s%s___%zd = %lluULL,";
		1039	btf_dump_printf(d, fmt_str,
		1040	pfx(lvl + 1), name, dup_cnt,
		1041	(unsigned long long)val);
		1042	} else {
		1043	fmt_str = is_signed ? "\n%s%s = %lldLL,"
		1044	: "\n%s%s = %lluULL,";
		1045	btf_dump_printf(d, fmt_str,
		1046	pfx(lvl + 1), name,
		1047	(unsigned long long)val);
		1048	}
		1049	}
		1050	}
		1051	static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id,
		1052	const struct btf_type *t,
		1053	int lvl)
		1054	{
		1055	__u16 vlen = btf_vlen(t);
		1056
1002	btf_dump_printf(d, "enum%s%s",	1057	btf_dump_printf(d, "enum%s%s",
1003	t->name_off ? " " : "",	1058	t->name_off ? " " : "",
1004	btf_dump_type_name(d, id));	1059	btf_dump_type_name(d, id));
1005		1060
1006	if (vlen) {	1061	if (!vlen)
1007	btf_dump_printf(d, " {");	1062	return;
1008	for (i = 0; i < vlen; i++, v++) {	1063
1009	name = btf_name_of(d, v->name_off);	1064	btf_dump_printf(d, " {");
1010	/* enumerators share namespace with typedef idents */	1065	if (btf_is_enum(t))
1011	dup_cnt = btf_dump_name_dups(d, d->ident_names, name);	1066	btf_dump_emit_enum32_val(d, t, lvl, vlen);
1012	if (dup_cnt > 1) {	1067	else
1013	btf_dump_printf(d, "\n%s%s___%zu = %u,",	1068	btf_dump_emit_enum64_val(d, t, lvl, vlen);
1014	pfx(lvl + 1), name, dup_cnt,	1069	btf_dump_printf(d, "\n%s}", pfx(lvl));
1015	(__u32)v->val);
1016	} else {
1017	btf_dump_printf(d, "\n%s%s = %u,",
1018	pfx(lvl + 1), name,
1019	(__u32)v->val);
1020	}
1021	}
1022	btf_dump_printf(d, "\n%s}", pfx(lvl));
1023	}
1024	}	1070	}
1025		1071
1026	static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id,	1072	static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id,
@@ -1178,6 +1224,7 @@ skip_mod: · 151 unmodified lines
1178	break;	1224	break;
1179	case BTF_KIND_INT:	1225	case BTF_KIND_INT:
1180	case BTF_KIND_ENUM:	1226	case BTF_KIND_ENUM:
		1227	case BTF_KIND_ENUM64:
1181	case BTF_KIND_FWD:	1228	case BTF_KIND_FWD:
1182	case BTF_KIND_STRUCT:	1229	case BTF_KIND_STRUCT:
1183	case BTF_KIND_UNION:	1230	case BTF_KIND_UNION:
@@ -1312,6 +1359,7 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, · 128 unmodified lines
1312	btf_dump_emit_struct_fwd(d, id, t);	1359	btf_dump_emit_struct_fwd(d, id, t);
1313	break;	1360	break;
1314	case BTF_KIND_ENUM:	1361	case BTF_KIND_ENUM:
		1362	case BTF_KIND_ENUM64:
1315	btf_dump_emit_mods(d, decls);	1363	btf_dump_emit_mods(d, decls);
1316	/* inline anonymous enum */	1364	/* inline anonymous enum */
1317	if (t->name_off == 0 && !d->skip_anon_defs)	1365	if (t->name_off == 0 && !d->skip_anon_defs)
@@ -1988,7 +2036,8 @@ static int btf_dump_get_enum_value(struct btf_dump *d, · 670 unmodified lines
1988	__u32 id,	2036	__u32 id,
1989	__s64 *value)	2037	__s64 *value)
1990	{	2038	{
1991	/* handle unaligned enum value */	2039	bool is_signed = btf_kflag(t);
		2040
1992	if (!ptr_is_aligned(d->btf, id, data)) {	2041	if (!ptr_is_aligned(d->btf, id, data)) {
1993	__u64 val;	2042	__u64 val;
1994	int err;	2043	int err;
@@ -2005,13 +2054,13 @@ static int btf_dump_get_enum_value(struct btf_dump *d, · 10 unmodified lines
2005	value = (__s64 *)data;	2054	value = (__s64 *)data;
2006	return 0;	2055	return 0;
2007	case 4:	2056	case 4:
2008	value = (__s32 *)data;	2057	value = is_signed ? (__s32 )data : (__u32 *)data;
2009	return 0;	2058	return 0;
2010	case 2:	2059	case 2:
2011	value = (__s16 *)data;	2060	value = is_signed ? (__s16 )data : (__u16 *)data;
2012	return 0;	2061	return 0;
2013	case 1:	2062	case 1:
2014	value = (__s8 *)data;	2063	value = is_signed ? (__s8 )data : (__u8 *)data;
2015	return 0;	2064	return 0;
2016	default:	2065	default:
2017	pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id);	2066	pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id);
@@ -2024,7 +2073,7 @@ static int btf_dump_enum_data(struct btf_dump *d, · 6 unmodified lines
2024	__u32 id,	2073	__u32 id,
2025	const void *data)	2074	const void *data)
2026	{	2075	{
2027	const struct btf_enum *e;	2076	bool is_signed;
2028	__s64 value;	2077	__s64 value;
2029	int i, err;	2078	int i, err;
2030		2079
@@ -2032,14 +2081,31 @@ static int btf_dump_enum_data(struct btf_dump *d, · 1 unmodified line
2032	if (err)	2081	if (err)
2033	return err;	2082	return err;
2034		2083
2035	for (i = 0, e = btf_enum(t); i < btf_vlen(t); i++, e++) {	2084	is_signed = btf_kflag(t);
2036	if (value != e->val)	2085	if (btf_is_enum(t)) {
2037	continue;	2086	const struct btf_enum *e;
2038	btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off));	2087
2039	return 0;	2088	for (i = 0, e = btf_enum(t); i < btf_vlen(t); i++, e++) {
2040	}	2089	if (value != e->val)
		2090	continue;
		2091	btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off));
		2092	return 0;
		2093	}
2041		2094
2042	btf_dump_type_values(d, "%d", value);	2095	btf_dump_type_values(d, is_signed ? "%d" : "%u", value);
		2096	} else {
		2097	const struct btf_enum64 *e;
		2098
		2099	for (i = 0, e = btf_enum64(t); i < btf_vlen(t); i++, e++) {
		2100	if (value != btf_enum64_value(e))
		2101	continue;
		2102	btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off));
		2103	return 0;
		2104	}
		2105
		2106	btf_dump_type_values(d, is_signed ? "%lldLL" : "%lluULL",
		2107	(unsigned long long)value);
		2108	}
2043	return 0;	2109	return 0;
2044	}	2110	}
2045		2111
@@ -2099,6 +2165,7 @@ static int btf_dump_type_data_check_overflow(struct btf_dump *d, · 53 unmodified lines
2099	case BTF_KIND_FLOAT:	2165	case BTF_KIND_FLOAT:
2100	case BTF_KIND_PTR:	2166	case BTF_KIND_PTR:
2101	case BTF_KIND_ENUM:	2167	case BTF_KIND_ENUM:
		2168	case BTF_KIND_ENUM64:
2102	if (data + bits_offset / 8 + size > d->typed_dump->data_end)	2169	if (data + bits_offset / 8 + size > d->typed_dump->data_end)
2103	return -E2BIG;	2170	return -E2BIG;
2104	break;	2171	break;
@@ -2203,6 +2270,7 @@ static int btf_dump_type_data_check_zero(struct btf_dump *d, · 98 unmodified lines
2203	return -ENODATA;	2270	return -ENODATA;
2204	}	2271	}
2205	case BTF_KIND_ENUM:	2272	case BTF_KIND_ENUM:
		2273	case BTF_KIND_ENUM64:
2206	err = btf_dump_get_enum_value(d, t, data, id, &value);	2274	err = btf_dump_get_enum_value(d, t, data, id, &value);
2207	if (err)	2275	if (err)
2208	return err;	2276	return err;
@@ -2275,6 +2343,7 @@ static int btf_dump_dump_type_data(struct btf_dump *d, · 66 unmodified lines
2275	err = btf_dump_struct_data(d, t, id, data);	2343	err = btf_dump_struct_data(d, t, id, data);
2276	break;	2344	break;
2277	case BTF_KIND_ENUM:	2345	case BTF_KIND_ENUM:
		2346	case BTF_KIND_ENUM64:
2278	/* handle bitfield and int enum values */	2347	/* handle bitfield and int enum values */
2279	if (bit_sz) {	2348	if (bit_sz) {
2280	__u64 print_num;	2349	__u64 print_num;

tools/lib/bpf/libbpf.c+242−54

@@ -72,6 +72,134 @@ · 71 unmodified lines
72	static struct bpf_map bpf_object__add_map(struct bpf_object obj);	72	static struct bpf_map bpf_object__add_map(struct bpf_object obj);
73	static bool prog_is_subprog(const struct bpf_object obj, const struct bpf_program prog);	73	static bool prog_is_subprog(const struct bpf_object obj, const struct bpf_program prog);
74		74
		75	static const char * const attach_type_name[] = {
		76	[BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress",
		77	[BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress",
		78	[BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create",
		79	[BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release",
		80	[BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops",
		81	[BPF_CGROUP_DEVICE] = "cgroup_device",
		82	[BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind",
		83	[BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind",
		84	[BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect",
		85	[BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect",
		86	[BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind",
		87	[BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind",
		88	[BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername",
		89	[BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername",
		90	[BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname",
		91	[BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname",
		92	[BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg",
		93	[BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg",
		94	[BPF_CGROUP_SYSCTL] = "cgroup_sysctl",
		95	[BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg",
		96	[BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg",
		97	[BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt",
		98	[BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt",
		99	[BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser",
		100	[BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
		101	[BPF_SK_SKB_VERDICT] = "sk_skb_verdict",
		102	[BPF_SK_MSG_VERDICT] = "sk_msg_verdict",
		103	[BPF_LIRC_MODE2] = "lirc_mode2",
		104	[BPF_FLOW_DISSECTOR] = "flow_dissector",
		105	[BPF_TRACE_RAW_TP] = "trace_raw_tp",
		106	[BPF_TRACE_FENTRY] = "trace_fentry",
		107	[BPF_TRACE_FEXIT] = "trace_fexit",
		108	[BPF_MODIFY_RETURN] = "modify_return",
		109	[BPF_LSM_MAC] = "lsm_mac",
		110	[BPF_SK_LOOKUP] = "sk_lookup",
		111	[BPF_TRACE_ITER] = "trace_iter",
		112	[BPF_XDP_DEVMAP] = "xdp_devmap",
		113	[BPF_XDP_CPUMAP] = "xdp_cpumap",
		114	[BPF_XDP] = "xdp",
		115	[BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select",
		116	[BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate",
		117	[BPF_PERF_EVENT] = "perf_event",
		118	[BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi",
		119	};
		120
		121	static const char * const link_type_name[] = {
		122	[BPF_LINK_TYPE_UNSPEC] = "unspec",
		123	[BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
		124	[BPF_LINK_TYPE_TRACING] = "tracing",
		125	[BPF_LINK_TYPE_CGROUP] = "cgroup",
		126	[BPF_LINK_TYPE_ITER] = "iter",
		127	[BPF_LINK_TYPE_NETNS] = "netns",
		128	[BPF_LINK_TYPE_XDP] = "xdp",
		129	[BPF_LINK_TYPE_PERF_EVENT] = "perf_event",
		130	[BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi",
		131	[BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops",
		132	};
		133
		134	static const char * const map_type_name[] = {
		135	[BPF_MAP_TYPE_UNSPEC] = "unspec",
		136	[BPF_MAP_TYPE_HASH] = "hash",
		137	[BPF_MAP_TYPE_ARRAY] = "array",
		138	[BPF_MAP_TYPE_PROG_ARRAY] = "prog_array",
		139	[BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array",
		140	[BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash",
		141	[BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array",
		142	[BPF_MAP_TYPE_STACK_TRACE] = "stack_trace",
		143	[BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array",
		144	[BPF_MAP_TYPE_LRU_HASH] = "lru_hash",
		145	[BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash",
		146	[BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie",
		147	[BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps",
		148	[BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps",
		149	[BPF_MAP_TYPE_DEVMAP] = "devmap",
		150	[BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash",
		151	[BPF_MAP_TYPE_SOCKMAP] = "sockmap",
		152	[BPF_MAP_TYPE_CPUMAP] = "cpumap",
		153	[BPF_MAP_TYPE_XSKMAP] = "xskmap",
		154	[BPF_MAP_TYPE_SOCKHASH] = "sockhash",
		155	[BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage",
		156	[BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray",
		157	[BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage",
		158	[BPF_MAP_TYPE_QUEUE] = "queue",
		159	[BPF_MAP_TYPE_STACK] = "stack",
		160	[BPF_MAP_TYPE_SK_STORAGE] = "sk_storage",
		161	[BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops",
		162	[BPF_MAP_TYPE_RINGBUF] = "ringbuf",
		163	[BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage",
		164	[BPF_MAP_TYPE_TASK_STORAGE] = "task_storage",
		165	[BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter",
		166	};
		167
		168	static const char * const prog_type_name[] = {
		169	[BPF_PROG_TYPE_UNSPEC] = "unspec",
		170	[BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
		171	[BPF_PROG_TYPE_KPROBE] = "kprobe",
		172	[BPF_PROG_TYPE_SCHED_CLS] = "sched_cls",
		173	[BPF_PROG_TYPE_SCHED_ACT] = "sched_act",
		174	[BPF_PROG_TYPE_TRACEPOINT] = "tracepoint",
		175	[BPF_PROG_TYPE_XDP] = "xdp",
		176	[BPF_PROG_TYPE_PERF_EVENT] = "perf_event",
		177	[BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb",
		178	[BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock",
		179	[BPF_PROG_TYPE_LWT_IN] = "lwt_in",
		180	[BPF_PROG_TYPE_LWT_OUT] = "lwt_out",
		181	[BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit",
		182	[BPF_PROG_TYPE_SOCK_OPS] = "sock_ops",
		183	[BPF_PROG_TYPE_SK_SKB] = "sk_skb",
		184	[BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device",
		185	[BPF_PROG_TYPE_SK_MSG] = "sk_msg",
		186	[BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint",
		187	[BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr",
		188	[BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local",
		189	[BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2",
		190	[BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport",
		191	[BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector",
		192	[BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl",
		193	[BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
		194	[BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt",
		195	[BPF_PROG_TYPE_TRACING] = "tracing",
		196	[BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops",
		197	[BPF_PROG_TYPE_EXT] = "ext",
		198	[BPF_PROG_TYPE_LSM] = "lsm",
		199	[BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup",
		200	[BPF_PROG_TYPE_SYSCALL] = "syscall",
		201	};
		202
75	static int __base_pr(enum libbpf_print_level level, const char *format,	203	static int __base_pr(enum libbpf_print_level level, const char *format,
76	va_list args)	204	va_list args)
77	{	205	{
@@ -2114,6 +2242,7 @@ static const char *__btf_kind_str(__u16 kind) · 2036 unmodified lines
2114	case BTF_KIND_FLOAT: return "float";	2242	case BTF_KIND_FLOAT: return "float";
2115	case BTF_KIND_DECL_TAG: return "decl_tag";	2243	case BTF_KIND_DECL_TAG: return "decl_tag";
2116	case BTF_KIND_TYPE_TAG: return "type_tag";	2244	case BTF_KIND_TYPE_TAG: return "type_tag";
		2245	case BTF_KIND_ENUM64: return "enum64";
2117	default: return "unknown";	2246	default: return "unknown";
2118	}	2247	}
2119	}	2248	}
@@ -2642,12 +2771,13 @@ static bool btf_needs_sanitization(struct bpf_object *obj) · 522 unmodified lines
2642	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);	2771	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2643	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);	2772	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2644	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);	2773	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
		2774	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2645		2775
2646	return !has_func \|\| !has_datasec \|\| !has_func_global \|\| !has_float \|\|	2776	return !has_func \|\| !has_datasec \|\| !has_func_global \|\| !has_float \|\|
2647	!has_decl_tag \|\| !has_type_tag;	2777	!has_decl_tag \|\| !has_type_tag \|\| !has_enum64;
2648	}	2778	}
2649		2779
2650	static void bpf_object__sanitize_btf(struct bpf_object obj, struct btf btf)	2780	static int bpf_object__sanitize_btf(struct bpf_object obj, struct btf btf)
2651	{	2781	{
2652	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);	2782	bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2653	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);	2783	bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
@@ -2655,6 +2785,8 @@ static void bpf_object__sanitize_btf(struct bpf_object obj, struct btf btf) · 1 unmodified line
2655	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);	2785	bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2656	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);	2786	bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2657	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);	2787	bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
		2788	bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
		2789	int enum64_placeholder_id = 0;
2658	struct btf_type *t;	2790	struct btf_type *t;
2659	int i, j, vlen;	2791	int i, j, vlen;
2660		2792
@@ -2717,8 +2849,32 @@ static void bpf_object__sanitize_btf(struct bpf_object obj, struct btf btf) · 56 unmodified lines
2717	/* replace TYPE_TAG with a CONST */	2849	/* replace TYPE_TAG with a CONST */
2718	t->name_off = 0;	2850	t->name_off = 0;
2719	t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);	2851	t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2720	}	2852	} else if (!has_enum64 && btf_is_enum(t)) {
		2853	/* clear the kflag */
		2854	t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
		2855	} else if (!has_enum64 && btf_is_enum64(t)) {
		2856	/* replace ENUM64 with a union */
		2857	struct btf_member *m;
		2858
		2859	if (enum64_placeholder_id == 0) {
		2860	enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
		2861	if (enum64_placeholder_id < 0)
		2862	return enum64_placeholder_id;
		2863
		2864	t = (struct btf_type *)btf__type_by_id(btf, i);
		2865	}
		2866
		2867	m = btf_members(t);
		2868	vlen = btf_vlen(t);
		2869	t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
		2870	for (j = 0; j < vlen; j++, m++) {
		2871	m->type = enum64_placeholder_id;
		2872	m->offset = 0;
		2873	}
		2874	}
2721	}	2875	}
		2876
		2877	return 0;
2722	}	2878	}
2723		2879
2724	static bool libbpf_needs_btf(const struct bpf_object *obj)	2880	static bool libbpf_needs_btf(const struct bpf_object *obj)
@@ -3056,7 +3212,9 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) · 331 unmodified lines
3056		3212
3057	/* enforce 8-byte pointers for BPF-targeted BTFs */	3213	/* enforce 8-byte pointers for BPF-targeted BTFs */
3058	btf__set_pointer_size(obj->btf, 8);	3214	btf__set_pointer_size(obj->btf, 8);
3059	bpf_object__sanitize_btf(obj, kern_btf);	3215	err = bpf_object__sanitize_btf(obj, kern_btf);
		3216	if (err)
		3217	return err;
3060	}	3218	}
3061		3219
3062	if (obj->gen_loader) {	3220	if (obj->gen_loader) {
@@ -3563,6 +3721,10 @@ static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, · 500 unmodified lines
3563	if (strcmp(name, "libbpf_tristate"))	3721	if (strcmp(name, "libbpf_tristate"))
3564	return KCFG_UNKNOWN;	3722	return KCFG_UNKNOWN;
3565	return KCFG_TRISTATE;	3723	return KCFG_TRISTATE;
		3724	case BTF_KIND_ENUM64:
		3725	if (strcmp(name, "libbpf_tristate"))
		3726	return KCFG_UNKNOWN;
		3727	return KCFG_TRISTATE;
3566	case BTF_KIND_ARRAY:	3728	case BTF_KIND_ARRAY:
3567	if (btf_array(t)->nelems == 0)	3729	if (btf_array(t)->nelems == 0)
3568	return KCFG_UNKNOWN;	3730	return KCFG_UNKNOWN;
@@ -4746,6 +4908,17 @@ static int probe_kern_bpf_cookie(void) · 1177 unmodified lines
4746	return probe_fd(ret);	4908	return probe_fd(ret);
4747	}	4909	}
4748		4910
		4911	static int probe_kern_btf_enum64(void)
		4912	{
		4913	static const char strs[] = "\0enum64";
		4914	__u32 types[] = {
		4915	BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
		4916	};
		4917
		4918	return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
		4919	strs, sizeof(strs)));
		4920	}
		4921
4749	enum kern_feature_result {	4922	enum kern_feature_result {
4750	FEAT_UNKNOWN = 0,	4923	FEAT_UNKNOWN = 0,
4751	FEAT_SUPPORTED = 1,	4924	FEAT_SUPPORTED = 1,
@@ -4811,6 +4984,9 @@ static struct kern_feature_desc { · 59 unmodified lines
4811	[FEAT_BPF_COOKIE] = {	4984	[FEAT_BPF_COOKIE] = {
4812	"BPF cookie support", probe_kern_bpf_cookie,	4985	"BPF cookie support", probe_kern_bpf_cookie,
4813	},	4986	},
		4987	[FEAT_BTF_ENUM64] = {
		4988	"BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
		4989	},
4814	};	4990	};
4815		4991
4816	bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)	4992	bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
@@ -4943,11 +5119,6 @@ bpf_object__populate_internal_map(struct bpf_object obj, struct bpf_map map) · 126 unmodified lines
4943		5119
4944	static void bpf_map__destroy(struct bpf_map *map);	5120	static void bpf_map__destroy(struct bpf_map *map);
4945		5121
4946	static bool is_pow_of_2(size_t x)
4947	{
4948	return x && (x & (x - 1));
4949	}
4950
4951	static size_t adjust_ringbuf_sz(size_t sz)	5122	static size_t adjust_ringbuf_sz(size_t sz)
4952	{	5123	{
4953	__u32 page_sz = sysconf(_SC_PAGE_SIZE);	5124	__u32 page_sz = sysconf(_SC_PAGE_SIZE);
@@ -5353,7 +5524,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, · 399 unmodified lines
5353	n = btf__type_cnt(targ_btf);	5524	n = btf__type_cnt(targ_btf);
5354	for (i = targ_start_id; i < n; i++) {	5525	for (i = targ_start_id; i < n; i++) {
5355	t = btf__type_by_id(targ_btf, i);	5526	t = btf__type_by_id(targ_btf, i);
5356	if (btf_kind(t) != btf_kind(local_t))	5527	if (!btf_kind_core_compat(t, local_t))
5357	continue;	5528	continue;
5358		5529
5359	targ_name = btf__name_by_offset(targ_btf, t->name_off);	5530	targ_name = btf__name_by_offset(targ_btf, t->name_off);
@@ -5567,7 +5738,7 @@ int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, · 207 unmodified lines
5567	/* caller made sure that names match (ignoring flavor suffix) */	5738	/* caller made sure that names match (ignoring flavor suffix) */
5568	local_type = btf__type_by_id(local_btf, local_id);	5739	local_type = btf__type_by_id(local_btf, local_id);
5569	targ_type = btf__type_by_id(targ_btf, targ_id);	5740	targ_type = btf__type_by_id(targ_btf, targ_id);
5570	if (btf_kind(local_type) != btf_kind(targ_type))	5741	if (!btf_kind_core_compat(local_type, targ_type))
5571	return 0;	5742	return 0;
5572		5743
5573	recur:	5744	recur:
@@ -5580,7 +5751,7 @@ recur: · 6 unmodified lines
5580	if (!local_type \|\| !targ_type)	5751	if (!local_type \|\| !targ_type)
5581	return -EINVAL;	5752	return -EINVAL;
5582		5753
5583	if (btf_kind(local_type) != btf_kind(targ_type))	5754	if (!btf_kind_core_compat(local_type, targ_type))
5584	return 0;	5755	return 0;
5585		5756
5586	switch (btf_kind(local_type)) {	5757	switch (btf_kind(local_type)) {
@@ -5588,6 +5759,7 @@ recur: · 1 unmodified line
5588	case BTF_KIND_STRUCT:	5759	case BTF_KIND_STRUCT:
5589	case BTF_KIND_UNION:	5760	case BTF_KIND_UNION:
5590	case BTF_KIND_ENUM:	5761	case BTF_KIND_ENUM:
		5762	case BTF_KIND_ENUM64:
5591	case BTF_KIND_FWD:	5763	case BTF_KIND_FWD:
5592	return 1;	5764	return 1;
5593	case BTF_KIND_INT:	5765	case BTF_KIND_INT:
@@ -9005,8 +9177,10 @@ static const struct bpf_sec_def section_defs[] = { · 3411 unmodified lines
9005	SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE \| SEC_SLOPPY_PFX),	9177	SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE \| SEC_SLOPPY_PFX),
9006	SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),	9178	SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
9007	SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),	9179	SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
		9180	SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
9008	SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),	9181	SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe),
9009	SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),	9182	SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe),
		9183	SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe),
9010	SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),	9184	SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
9011	SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),	9185	SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi),
9012	SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),	9186	SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt),
@@ -9300,6 +9474,38 @@ int libbpf_prog_type_by_name(const char name, enum bpf_prog_type prog_type, · 287 unmodified lines
9300	return libbpf_err(-ESRCH);	9474	return libbpf_err(-ESRCH);
9301	}	9475	}
9302		9476
		9477	const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t)
		9478	{
		9479	if (t < 0 \|\| t >= ARRAY_SIZE(attach_type_name))
		9480	return NULL;
		9481
		9482	return attach_type_name[t];
		9483	}
		9484
		9485	const char *libbpf_bpf_link_type_str(enum bpf_link_type t)
		9486	{
		9487	if (t < 0 \|\| t >= ARRAY_SIZE(link_type_name))
		9488	return NULL;
		9489
		9490	return link_type_name[t];
		9491	}
		9492
		9493	const char *libbpf_bpf_map_type_str(enum bpf_map_type t)
		9494	{
		9495	if (t < 0 \|\| t >= ARRAY_SIZE(map_type_name))
		9496	return NULL;
		9497
		9498	return map_type_name[t];
		9499	}
		9500
		9501	const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t)
		9502	{
		9503	if (t < 0 \|\| t >= ARRAY_SIZE(prog_type_name))
		9504	return NULL;
		9505
		9506	return prog_type_name[t];
		9507	}
		9508
9303	static struct bpf_map find_struct_ops_map_by_offset(struct bpf_object obj,	9509	static struct bpf_map find_struct_ops_map_by_offset(struct bpf_object obj,
9304	size_t offset)	9510	size_t offset)
9305	{	9511	{
@@ -10988,43 +11194,6 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, · 1682 unmodified lines
10988	return pfd;	11194	return pfd;
10989	}	11195	}
10990		11196
10991	/* uprobes deal in relative offsets; subtract the base address associated with
10992	* the mapped binary. See Documentation/trace/uprobetracer.rst for more
10993	* details.
10994	*/
10995	static long elf_find_relative_offset(const char filename, Elf elf, long addr)
10996	{
10997	size_t n;
10998	int i;
10999
11000	if (elf_getphdrnum(elf, &n)) {
11001	pr_warn("elf: failed to find program headers for '%s': %s\n", filename,
11002	elf_errmsg(-1));
11003	return -ENOENT;
11004	}
11005
11006	for (i = 0; i < n; i++) {
11007	int seg_start, seg_end, seg_offset;
11008	GElf_Phdr phdr;
11009
11010	if (!gelf_getphdr(elf, i, &phdr)) {
11011	pr_warn("elf: failed to get program header %d from '%s': %s\n", i, filename,
11012	elf_errmsg(-1));
11013	return -ENOENT;
11014	}
11015	if (phdr.p_type != PT_LOAD \|\| !(phdr.p_flags & PF_X))
11016	continue;
11017
11018	seg_start = phdr.p_vaddr;
11019	seg_end = seg_start + phdr.p_memsz;
11020	seg_offset = phdr.p_offset;
11021	if (addr >= seg_start && addr < seg_end)
11022	return addr - seg_start + seg_offset;
11023	}
11024	pr_warn("elf: failed to find prog header containing 0x%lx in '%s'\n", addr, filename);
11025	return -ENOENT;
11026	}
11027
11028	/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */	11197	/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */
11029	static Elf_Scn elf_find_next_scn_by_type(Elf elf, int sh_type, Elf_Scn *scn)	11198	static Elf_Scn elf_find_next_scn_by_type(Elf elf, int sh_type, Elf_Scn *scn)
11030	{	11199	{
@@ -11111,6 +11280,8 @@ static long elf_find_func_offset(const char binary_path, const char name) · 80 unmodified lines
11111	for (idx = 0; idx < nr_syms; idx++) {	11280	for (idx = 0; idx < nr_syms; idx++) {
11112	int curr_bind;	11281	int curr_bind;
11113	GElf_Sym sym;	11282	GElf_Sym sym;
		11283	Elf_Scn *sym_scn;
		11284	GElf_Shdr sym_sh;
11114		11285
11115	if (!gelf_getsym(symbols, idx, &sym))	11286	if (!gelf_getsym(symbols, idx, &sym))
11116	continue;	11287	continue;
@@ -11148,12 +11319,28 @@ static long elf_find_func_offset(const char binary_path, const char name) · 31 unmodified lines
11148	continue;	11319	continue;
11149	}	11320	}
11150	}	11321	}
11151	ret = sym.st_value;	11322
		11323	/* Transform symbol's virtual address (absolute for
		11324	* binaries and relative for shared libs) into file
		11325	* offset, which is what kernel is expecting for
		11326	* uprobe/uretprobe attachment.
		11327	* See Documentation/trace/uprobetracer.rst for more
		11328	* details.
		11329	* This is done by looking up symbol's containing
		11330	* section's header and using it's virtual address
		11331	* (sh_addr) and corresponding file offset (sh_offset)
		11332	* to transform sym.st_value (virtual address) into
		11333	* desired final file offset.
		11334	*/
		11335	sym_scn = elf_getscn(elf, sym.st_shndx);
		11336	if (!sym_scn)
		11337	continue;
		11338	if (!gelf_getshdr(sym_scn, &sym_sh))
		11339	continue;
		11340
		11341	ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset;
11152	last_bind = curr_bind;	11342	last_bind = curr_bind;
11153	}	11343	}
11154	/* For binaries that are not shared libraries, we need relative offset */
11155	if (ret > 0 && !is_shared_lib)
11156	ret = elf_find_relative_offset(binary_path, elf, ret);
11157	if (ret > 0)	11344	if (ret > 0)
11158	break;	11345	break;
11159	}	11346	}
@@ -11386,7 +11573,8 @@ static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf · 226 unmodified lines
11386	break;	11573	break;
11387	case 3:	11574	case 3:
11388	case 4:	11575	case 4:
11389	opts.retprobe = strcmp(probe_type, "uretprobe") == 0;	11576	opts.retprobe = strcmp(probe_type, "uretprobe") == 0 \|\|
		11577	strcmp(probe_type, "uretprobe.s") == 0;
11390	if (opts.retprobe && offset != 0) {	11578	if (opts.retprobe && offset != 0) {
11391	pr_warn("prog '%s': uretprobes do not support offset specification\n",	11579	pr_warn("prog '%s': uretprobes do not support offset specification\n",
11392	prog->name);	11580	prog->name);

tools/lib/bpf/libbpf.h+37−1

@@ -51,6 +51,42 @@ enum libbpf_errno { · 50 unmodified lines
51		51
52	LIBBPF_API int libbpf_strerror(int err, char *buf, size_t size);	52	LIBBPF_API int libbpf_strerror(int err, char *buf, size_t size);
53		53
		54	/**
		55	* @brief libbpf_bpf_attach_type_str() converts the provided attach type
		56	* value into a textual representation.
		57	* @param t The attach type.
		58	* @return Pointer to a static string identifying the attach type. NULL is
		59	* returned for unknown bpf_attach_type values.
		60	*/
		61	LIBBPF_API const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t);
		62
		63	/**
		64	* @brief libbpf_bpf_link_type_str() converts the provided link type value
		65	* into a textual representation.
		66	* @param t The link type.
		67	* @return Pointer to a static string identifying the link type. NULL is
		68	* returned for unknown bpf_link_type values.
		69	*/
		70	LIBBPF_API const char *libbpf_bpf_link_type_str(enum bpf_link_type t);
		71
		72	/**
		73	* @brief libbpf_bpf_map_type_str() converts the provided map type value
		74	* into a textual representation.
		75	* @param t The map type.
		76	* @return Pointer to a static string identifying the map type. NULL is
		77	* returned for unknown bpf_map_type values.
		78	*/
		79	LIBBPF_API const char *libbpf_bpf_map_type_str(enum bpf_map_type t);
		80
		81	/**
		82	* @brief libbpf_bpf_prog_type_str() converts the provided program type
		83	* value into a textual representation.
		84	* @param t The program type.
		85	* @return Pointer to a static string identifying the program type. NULL is
		86	* returned for unknown bpf_prog_type values.
		87	*/
		88	LIBBPF_API const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t);
		89
54	enum libbpf_print_level {	90	enum libbpf_print_level {
55	LIBBPF_WARN,	91	LIBBPF_WARN,
56	LIBBPF_INFO,	92	LIBBPF_INFO,
@@ -71,7 +107,7 @@ struct bpf_object_open_attr { · 14 unmodified lines
71	};	107	};
72		108
73	struct bpf_object_open_opts {	109	struct bpf_object_open_opts {
74	/* size of this struct, for forward/backward compatiblity */	110	/* size of this struct, for forward/backward compatibility */
75	size_t sz;	111	size_t sz;
76	/* object name override, if provided:	112	/* object name override, if provided:
77	* - for object open from file, this will override setting object	113	* - for object open from file, this will override setting object

tools/lib/bpf/libbpf.map+8−0

@@ -461,5 +461,13 @@ LIBBPF_0.8.0 { · 460 unmodified lines
461	} LIBBPF_0.7.0;	461	} LIBBPF_0.7.0;
462		462
463	LIBBPF_1.0.0 {	463	LIBBPF_1.0.0 {
		464	global:
		465	btf__add_enum64;
		466	btf__add_enum64_value;
		467	libbpf_bpf_attach_type_str;
		468	libbpf_bpf_link_type_str;
		469	libbpf_bpf_map_type_str;
		470	libbpf_bpf_prog_type_str;
		471
464	local: *;	472	local: *;
465	};	473	};

tools/lib/bpf/libbpf_internal.h+7−0

 	FEAT_MEMCG_ACCOUNT,
 	/* BPF cookie (bpf_get_attach_cookie() BPF helper) support */
 	FEAT_BPF_COOKIE,
+	/* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */
+	FEAT_BTF_ENUM64,
 	__FEAT_CNT,
 };
 					   const char *usdt_provider, const char *usdt_name,
 					   __u64 usdt_cookie);
+static inline bool is_pow_of_2(size_t x)
+{
+	return x && (x & (x - 1)) == 0;
+}
 #endif /* __LIBBPF_LIBBPF_INTERNAL_H */

tools/lib/bpf/linker.c+2−5

 	return err;
 }
-static bool is_pow_of_2(size_t x)
-{
-	return x && (x & (x - 1)) == 0;
-}
 static int linker_sanity_check_elf(struct src_obj *obj)
 {
 	struct src_sec *sec;
 	case BTF_KIND_STRUCT:
 	case BTF_KIND_UNION:
 	case BTF_KIND_ENUM:
+	case BTF_KIND_ENUM64:
 	case BTF_KIND_FWD:
 	case BTF_KIND_FUNC:
 	case BTF_KIND_VAR:
 	case BTF_KIND_INT:
 	case BTF_KIND_FLOAT:
 	case BTF_KIND_ENUM:
+	case BTF_KIND_ENUM64:
 		/* ignore encoding for int and enum values for enum */
 		if (t1->size != t2->size) {
 			pr_warn("global '%s': incompatible %s '%s' size %u and %u\n",

tools/lib/bpf/relo_core.c+69−44

  * just a parsed access string representation): [0, 1, 2, 3].
  *
  * High-level spec will capture only 3 points:
- *   - intial zero-index access by pointer (&s->... is the same as &s[0]...);
+ *   - initial zero-index access by pointer (&s->... is the same as &s[0]...);
  *   - field 'a' access (corresponds to '2' in low-level spec);
  *   - array element #3 access (corresponds to '3' in low-level spec).
  *
 	struct bpf_core_accessor *acc;
 	const struct btf_type *t;
 	const char *name, *spec_str;
-	__u32 id;
+	__u32 id, name_off;
 	__s64 sz;
 	spec_str = btf__name_by_offset(btf, relo->access_str_off);
 	spec->len++;
 	if (core_relo_is_enumval_based(relo->kind)) {
-		if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
+		if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t))
 			return -EINVAL;
 		/* record enumerator name in a first accessor */
-		acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off);
+		name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off
+					  : btf_enum64(t)[access_idx].name_off;
+		acc->name = btf__name_by_offset(btf, name_off);
 		return 0;
 	}
 	if (btf_is_composite(local_type) && btf_is_composite(targ_type))
 		return 1;
-	if (btf_kind(local_type) != btf_kind(targ_type))
+	if (!btf_kind_core_compat(local_type, targ_type))
 		return 0;
 	switch (btf_kind(local_type)) {
 	case BTF_KIND_FLOAT:
 		return 1;
 	case BTF_KIND_FWD:
+	case BTF_KIND_ENUM64:
 	case BTF_KIND_ENUM: {
 		const char *local_name, *targ_name;
 		size_t local_len, targ_len;
 	const struct bpf_core_accessor *local_acc;
 	struct bpf_core_accessor *targ_acc;
 	int i, sz, matched;
+	__u32 name_off;
 	memset(targ_spec, 0, sizeof(*targ_spec));
 	targ_spec->btf = targ_btf;
 	if (core_relo_is_enumval_based(local_spec->relo_kind)) {
 		size_t local_essent_len, targ_essent_len;
-		const struct btf_enum *e;
 		const char *targ_name;
 		/* has to resolve to an enum */
 		targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id);
-		if (!btf_is_enum(targ_type))
+		if (!btf_is_any_enum(targ_type))
 			return 0;
 		local_essent_len = bpf_core_essential_name_len(local_acc->name);
-		for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) {
+		for (i = 0; i < btf_vlen(targ_type); i++) {
-			targ_name = btf__name_by_offset(targ_spec->btf, e->name_off);
+			if (btf_is_enum(targ_type))
+				name_off = btf_enum(targ_type)[i].name_off;
+			else
+				name_off = btf_enum64(targ_type)[i].name_off;
+			targ_name = btf__name_by_offset(targ_spec->btf, name_off);
 			targ_essent_len = bpf_core_essential_name_len(targ_name);
 			if (targ_essent_len != local_essent_len)
 				continue;
 static int bpf_core_calc_field_relo(const char *prog_name,
 				    const struct bpf_core_relo *relo,
 				    const struct bpf_core_spec *spec,
-				    __u32 *val, __u32 *field_sz, __u32 *type_id,
+				    __u64 *val, __u32 *field_sz, __u32 *type_id,
 				    bool *validate)
 {
 	const struct bpf_core_accessor *acc;
 		*val = byte_sz;
 		break;
 	case BPF_CORE_FIELD_SIGNED:
-		/* enums will be assumed unsigned */
+		*val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) ||
-		*val = btf_is_enum(mt) ||
 		       (btf_int_encoding(mt) & BTF_INT_SIGNED);
 		if (validate)
 			*validate = true; /* signedness is never ambiguous */
 static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo,
 				   const struct bpf_core_spec *spec,
-				   __u32 *val, bool *validate)
+				   __u64 *val, bool *validate)
 {
 	__s64 sz;
 static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo,
 				      const struct bpf_core_spec *spec,
-				      __u32 *val)
+				      __u64 *val)
 {
 	const struct btf_type *t;
-	const struct btf_enum *e;
 	switch (relo->kind) {
 	case BPF_CORE_ENUMVAL_EXISTS:
 		if (!spec)
 			return -EUCLEAN; /* request instruction poisoning */
 		t = btf_type_by_id(spec->btf, spec->spec[0].type_id);
-		e = btf_enum(t) + spec->spec[0].idx;
+		if (btf_is_enum(t))
-		*val = e->val;
+			*val = btf_enum(t)[spec->spec[0].idx].val;
+		else
+			*val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx);
 		break;
 	default:
 		return -EOPNOTSUPP;
 			int insn_idx, const struct bpf_core_relo *relo,
 			int relo_idx, const struct bpf_core_relo_res *res)
 {
-	__u32 orig_val, new_val;
+	__u64 orig_val, new_val;
 	__u8 class;
 	class = BPF_CLASS(insn->code);
 		if (BPF_SRC(insn->code) != BPF_K)
 			return -EINVAL;
 		if (res->validate && insn->imm != orig_val) {
-			pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n",
+			pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n",
 				prog_name, relo_idx,
-				insn_idx, insn->imm, orig_val, new_val);
+				insn_idx, insn->imm, (unsigned long long)orig_val,
+				(unsigned long long)new_val);
 			return -EINVAL;
 		}
 		orig_val = insn->imm;
 		insn->imm = new_val;
-		pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n",
+		pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n",
 			 prog_name, relo_idx, insn_idx,
-			 orig_val, new_val);
+			 (unsigned long long)orig_val, (unsigned long long)new_val);
 		break;
 	case BPF_LDX:
 	case BPF_ST:
 	case BPF_STX:
 		if (res->validate && insn->off != orig_val) {
-			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n",
+			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n",
-				prog_name, relo_idx, insn_idx, insn->off, orig_val, new_val);
+				prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val,
+				(unsigned long long)new_val);
 			return -EINVAL;
 		}
 		if (new_val > SHRT_MAX) {
-			pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n",
+			pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n",
-				prog_name, relo_idx, insn_idx, new_val);
+				prog_name, relo_idx, insn_idx, (unsigned long long)new_val);
 			return -ERANGE;
 		}
 		if (res->fail_memsz_adjust) {
 		orig_val = insn->off;
 		insn->off = new_val;
-		pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n",
+		pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n",
-			 prog_name, relo_idx, insn_idx, orig_val, new_val);
+			 prog_name, relo_idx, insn_idx, (unsigned long long)orig_val,
+			 (unsigned long long)new_val);
 		if (res->new_sz != res->orig_sz) {
 			int insn_bytes_sz, insn_bpf_sz;
 			return -EINVAL;
 		}
-		imm = insn[0].imm + ((__u64)insn[1].imm << 32);
+		imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32);
 		if (res->validate && imm != orig_val) {
-			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n",
+			pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n",
 				prog_name, relo_idx,
 				insn_idx, (unsigned long long)imm,
-				orig_val, new_val);
+				(unsigned long long)orig_val, (unsigned long long)new_val);
 			return -EINVAL;
 		}
 		insn[0].imm = new_val;
-		insn[1].imm = 0; /* currently only 32-bit values are supported */
+		insn[1].imm = new_val >> 32;
-		pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n",
+		pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n",
 			 prog_name, relo_idx, insn_idx,
-			 (unsigned long long)imm, new_val);
+			 (unsigned long long)imm, (unsigned long long)new_val);
 		break;
 	}
 	default:
 int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec)
 {
 	const struct btf_type *t;
-	const struct btf_enum *e;
 	const char *s;
 	__u32 type_id;
 	int i, len = 0;
 	if (core_relo_is_enumval_based(spec->relo_kind)) {
 		t = skip_mods_and_typedefs(spec->btf, type_id, NULL);
-		e = btf_enum(t) + spec->raw_spec[0];
+		if (btf_is_enum(t)) {
-		s = btf__name_by_offset(spec->btf, e->name_off);
+			const struct btf_enum *e;
+			const char *fmt_str;
+			e = btf_enum(t) + spec->raw_spec[0];
+			s = btf__name_by_offset(spec->btf, e->name_off);
+			fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u";
+			append_buf(fmt_str, s, e->val);
+		} else {
+			const struct btf_enum64 *e;
+			const char *fmt_str;
-		append_buf("::%s = %u", s, e->val);
+			e = btf_enum64(t) + spec->raw_spec[0];
+			s = btf__name_by_offset(spec->btf, e->name_off);
+			fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu";
+			append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e));
+		}
 		return len;
 	}
  * 3. It is supported and expected that there might be multiple flavors
  *    matching the spec. As long as all the specs resolve to the same set of
  *    offsets across all candidates, there is no error. If there is any
- *    ambiguity, CO-RE relocation will fail. This is necessary to accomodate
+ *    ambiguity, CO-RE relocation will fail. This is necessary to accommodate
- *    imprefection of BTF deduplication, which can cause slight duplication of
+ *    imperfection of BTF deduplication, which can cause slight duplication of
  *    the same BTF type, if some directly or indirectly referenced (by
  *    pointer) type gets resolved to different actual types in different
- *    object files. If such situation occurs, deduplicated BTF will end up
+ *    object files. If such a situation occurs, deduplicated BTF will end up
  *    with two (or more) structurally identical types, which differ only in
  *    types they refer to through pointer. This should be OK in most cases and
  *    is not an error.
 			 * decision and value, otherwise it's dangerous to
 			 * proceed due to ambiguity
 			 */
-			pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n",
+			pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n",
 				prog_name, relo_idx,
-				cand_res.poison ? "failure" : "success", cand_res.new_val,
+				cand_res.poison ? "failure" : "success",
-				targ_res->poison ? "failure" : "success", targ_res->new_val);
+				(unsigned long long)cand_res.new_val,
+				targ_res->poison ? "failure" : "success",
+				(unsigned long long)targ_res->new_val);
 			return -EINVAL;
 		}

tools/lib/bpf/relo_core.h+2−2

@@ -46,9 +46,9 @@ struct bpf_core_spec { · 45 unmodified lines
46		46
47	struct bpf_core_relo_res {	47	struct bpf_core_relo_res {
48	/* expected value in the instruction, unless validate == false */	48	/* expected value in the instruction, unless validate == false */
49	__u32 orig_val;	49	__u64 orig_val;
50	/* new value that needs to be patched up to */	50	/* new value that needs to be patched up to */
51	__u32 new_val;	51	__u64 new_val;
52	/* relocation unsuccessful, poison instruction, but don't fail load */	52	/* relocation unsuccessful, poison instruction, but don't fail load */
53	bool poison;	53	bool poison;
54	/* some relocations can't be validated against orig_val */	54	/* some relocations can't be validated against orig_val */

tools/lib/bpf/usdt.c+63−60

 	return 0;
 }
-static int parse_lib_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt)
+static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt)
 {
 	char path[PATH_MAX], line[PATH_MAX], mode[16];
 	size_t seg_start, seg_end, seg_off;
 	return err;
 }
-static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long addr, bool relative)
+static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long virtaddr)
 {
 	struct elf_seg *seg;
 	int i;
-	if (relative) {
+	/* for ELF binaries (both executables and shared libraries), we are
-		/* for shared libraries, address is relative offset and thus
+	 * given virtual address (absolute for executables, relative for
-		 * should be fall within logical offset-based range of
+	 * libraries) which should match address range of [seg_start, seg_end)
-		 * [offset_start, offset_end)
+	 */
-		 */
+	for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
-		for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
+		if (seg->start <= virtaddr && virtaddr < seg->end)
-			if (seg->offset <= addr && addr < seg->offset + (seg->end - seg->start))
+			return seg;
-				return seg;
-		}
-	} else {
-		/* for binaries, address is absolute and thus should be within
-		 * absolute address range of [seg_start, seg_end)
-		 */
-		for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
-			if (seg->start <= addr && addr < seg->end)
-				return seg;
-		}
 	}
+	return NULL;
+}
+static struct elf_seg *find_vma_seg(struct elf_seg *segs, size_t seg_cnt, long offset)
+{
+	struct elf_seg *seg;
+	int i;
+	/* for VMA segments from /proc/<pid>/maps file, provided "address" is
+	 * actually a file offset, so should be fall within logical
+	 * offset-based range of [offset_start, offset_end)
+	 */
+	for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
+		if (seg->offset <= offset && offset < seg->offset + (seg->end - seg->start))
+			return seg;
+	}
 	return NULL;
 }
-static int parse_usdt_note(Elf *elf, const char *path, long base_addr,
+static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr,
-			   GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off,
+			   const char *data, size_t name_off, size_t desc_off,
 			   struct usdt_note *usdt_note);
 static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie);
 				const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie,
 				struct usdt_target **out_targets, size_t *out_target_cnt)
 {
-	size_t off, name_off, desc_off, seg_cnt = 0, lib_seg_cnt = 0, target_cnt = 0;
+	size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0;
-	struct elf_seg *segs = NULL, *lib_segs = NULL;
+	struct elf_seg *segs = NULL, *vma_segs = NULL;
 	struct usdt_target *targets = NULL, *target;
 	long base_addr = 0;
 	Elf_Scn *notes_scn, *base_scn;
 		struct elf_seg *seg = NULL;
 		void *tmp;
-		err = parse_usdt_note(elf, path, base_addr, &nhdr,
+		err = parse_usdt_note(elf, path, &nhdr, data->d_buf, name_off, desc_off, &note);
-				      data->d_buf, name_off, desc_off, &note);
 		if (err)
 			goto err_out;
 			usdt_rel_ip += base_addr - note.base_addr;
 		}
-		if (ehdr.e_type == ET_EXEC) {
+		/* When attaching uprobes (which is what USDTs basically are)
-			/* When attaching uprobes (which what USDTs basically
+		 * kernel expects file offset to be specified, not a relative
-			 * are) kernel expects a relative IP to be specified,
+		 * virtual address, so we need to translate virtual address to
-			 * so if we are attaching to an executable ELF binary
+		 * file offset, for both ET_EXEC and ET_DYN binaries.
-			 * (i.e., not a shared library), we need to calculate
+		 */
-			 * proper relative IP based on ELF's load address
+		seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip);
-			 */
+		if (!seg) {
-			seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip, false /* relative */);
+			err = -ESRCH;
-			if (!seg) {
+			pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n",
-				err = -ESRCH;
+				usdt_provider, usdt_name, path, usdt_abs_ip);
-				pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n",
+			goto err_out;
-					usdt_provider, usdt_name, path, usdt_abs_ip);
+		}
-				goto err_out;
+		if (!seg->is_exec) {
-			}
+			err = -ESRCH;
-			if (!seg->is_exec) {
+			pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n",
-				err = -ESRCH;
+				path, seg->start, seg->end, usdt_provider, usdt_name,
-				pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n",
+				usdt_abs_ip);
-					path, seg->start, seg->end, usdt_provider, usdt_name,
+			goto err_out;
-					usdt_abs_ip);
+		}
-				goto err_out;
+		/* translate from virtual address to file offset */
-			}
+		usdt_rel_ip = usdt_abs_ip - seg->start + seg->offset;
-			usdt_rel_ip = usdt_abs_ip - (seg->start - seg->offset);
+		if (ehdr.e_type == ET_DYN && !man->has_bpf_cookie) {
-		} else if (!man->has_bpf_cookie) { /* ehdr.e_type == ET_DYN */
 			/* If we don't have BPF cookie support but need to
 			 * attach to a shared library, we'll need to know and
 			 * record absolute addresses of attach points due to
 				goto err_out;
 			}
-			/* lib_segs are lazily initialized only if necessary */
+			/* vma_segs are lazily initialized only if necessary */
-			if (lib_seg_cnt == 0) {
+			if (vma_seg_cnt == 0) {
-				err = parse_lib_segs(pid, path, &lib_segs, &lib_seg_cnt);
+				err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt);
 				if (err) {
 					pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %d\n",
 						pid, path, err);
 				}
 			}
-			seg = find_elf_seg(lib_segs, lib_seg_cnt, usdt_rel_ip, true /* relative */);
+			seg = find_vma_seg(vma_segs, vma_seg_cnt, usdt_rel_ip);
 			if (!seg) {
 				err = -ESRCH;
 				pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n",
 				goto err_out;
 			}
-			usdt_abs_ip = seg->start + (usdt_rel_ip - seg->offset);
+			usdt_abs_ip = seg->start - seg->offset + usdt_rel_ip;
 		}
 		pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n",
 			 note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args,
 			 seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0);
-		/* Adjust semaphore address to be a relative offset */
+		/* Adjust semaphore address to be a file offset */
 		if (note.sema_addr) {
 			if (!man->has_sema_refcnt) {
 				pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n",
 				goto err_out;
 			}
-			seg = find_elf_seg(segs, seg_cnt, note.sema_addr, false /* relative */);
+			seg = find_elf_seg(segs, seg_cnt, note.sema_addr);
 			if (!seg) {
 				err = -ESRCH;
 				pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n",
 				goto err_out;
 			}
-			usdt_sema_off = note.sema_addr - (seg->start - seg->offset);
+			usdt_sema_off = note.sema_addr - seg->start + seg->offset;
 			pr_debug("usdt: sema  for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n",
 				 usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ",
 		target->rel_ip = usdt_rel_ip;
 		target->sema_off = usdt_sema_off;
-		/* notes->args references strings from Elf itself, so they can
+		/* notes.args references strings from Elf itself, so they can
 		 * be referenced safely until elf_end() call
 		 */
 		target->spec_str = note.args;
 err_out:
 	free(segs);
-	free(lib_segs);
+	free(vma_segs);
 	if (err < 0)
 		free(targets);
 	return err;
 /* Parse out USDT ELF note from '.note.stapsdt' section.
  * Logic inspired by perf's code.
  */
-static int parse_usdt_note(Elf *elf, const char *path, long base_addr,
+static int parse_usdt_note(Elf *elf, const char *path, GElf_Nhdr *nhdr,
-			   GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off,
+			   const char *data, size_t name_off, size_t desc_off,
 			   struct usdt_note *note)
 {
 	const char *provider, *name, *args;

tools/testing/selftests/bpf/.gitignore+1−0

@@ -43,3 +43,4 @@ test_cpp · 42 unmodified lines
43	*.tmp	43	*.tmp
44	xdpxceiver	44	xdpxceiver
45	xdp_redirect_multi	45	xdp_redirect_multi
		46	xdp_synproxy

tools/testing/selftests/bpf/Makefile+21−7

 TEST_GEN_PROGS_EXTENDED = test_sock_addr test_skb_cgroup_id_user \
 	flow_dissector_load test_flow_dissector test_tcp_check_syncookie_user \
 	test_lirc_mode2_user xdping test_cpp runqslower bench bpf_testmod.ko \
-	xdpxceiver xdp_redirect_multi
+	xdpxceiver xdp_redirect_multi xdp_synproxy
 TEST_CUSTOM_PROGS = $(OUTPUT)/urandom_read
 	$(call msg,BINARY,,$@)
 	$(Q)$(LINK.c) $^ $(LDLIBS) -o $@
+# LLVM's ld.lld doesn't support all the architectures, so use it only on x86
+ifeq ($(SRCARCH),x86)
+LLD := lld
+else
+LLD := ld
+endif
 # Filter out -static for liburandom_read.so and its dependent targets so that static builds
 # do not fail. Static builds leave urandom_read relying on system-wide shared libraries.
 $(OUTPUT)/liburandom_read.so: urandom_read_lib1.c urandom_read_lib2.c
 	$(call msg,LIB,,$@)
-	$(Q)$(CC) $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $^ $(LDLIBS) -fPIC -shared -o $@
+	$(Q)$(CLANG) $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $^ $(LDLIBS)   \
+		     -fuse-ld=$(LLD) -Wl,-znoseparate-code -fPIC -shared -o $@
 $(OUTPUT)/urandom_read: urandom_read.c urandom_read_aux.c $(OUTPUT)/liburandom_read.so
 	$(call msg,BINARY,,$@)
-	$(Q)$(CC) $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $(filter %.c,$^)  \
+	$(Q)$(CLANG) $(filter-out -static,$(CFLAGS) $(LDFLAGS)) $(filter %.c,$^) \
-		  liburandom_read.so $(LDLIBS)	       			       \
+		     liburandom_read.so $(LDLIBS)			       \
-		  -Wl,-rpath=. -Wl,--build-id=sha1 -o $@
+		     -fuse-ld=$(LLD) -Wl,-znoseparate-code		       \
+		     -Wl,-rpath=. -Wl,--build-id=sha1 -o $@
 $(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch])
 	$(call msg,MOD,,$@)
 			 cap_helpers.c
 TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read $(OUTPUT)/bpf_testmod.ko	\
 		       $(OUTPUT)/liburandom_read.so			\
+		       $(OUTPUT)/xdp_synproxy				\
 		       ima_setup.sh					\
 		       $(wildcard progs/btf_dump_test_case_*.c)
 TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE
 $(OUTPUT)/bench_bloom_filter_map.o: $(OUTPUT)/bloom_filter_bench.skel.h
 $(OUTPUT)/bench_bpf_loop.o: $(OUTPUT)/bpf_loop_bench.skel.h
 $(OUTPUT)/bench_strncmp.o: $(OUTPUT)/strncmp_bench.skel.h
+$(OUTPUT)/bench_bpf_hashmap_full_update.o: $(OUTPUT)/bpf_hashmap_full_update_bench.skel.h
 $(OUTPUT)/bench.o: bench.h testing_helpers.h $(BPFOBJ)
 $(OUTPUT)/bench: LDLIBS += -lm
 $(OUTPUT)/bench: $(OUTPUT)/bench.o \
 		 $(OUTPUT)/bench_ringbufs.o \
 		 $(OUTPUT)/bench_bloom_filter_map.o \
 		 $(OUTPUT)/bench_bpf_loop.o \
-		 $(OUTPUT)/bench_strncmp.o
+		 $(OUTPUT)/bench_strncmp.o \
+		 $(OUTPUT)/bench_bpf_hashmap_full_update.o
 	$(call msg,BINARY,,$@)
 	$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
 EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(SCRATCH_DIR) $(HOST_SCRATCH_DIR)	\
 	prog_tests/tests.h map_tests/tests.h verifier/tests.h		\
 	feature bpftool							\
-	$(addprefix $(OUTPUT)/,*.o *.skel.h *.lskel.h *.subskel.h no_alu32 bpf_gcc bpf_testmod.ko)
+	$(addprefix $(OUTPUT)/,*.o *.skel.h *.lskel.h *.subskel.h	\
+			       no_alu32 bpf_gcc bpf_testmod.ko		\
+			       liburandom_read.so)
 .PHONY: docs docs-clean

tools/testing/selftests/bpf/bench.c+2−0

 extern const struct bench bench_bpf_loop;
 extern const struct bench bench_strncmp_no_helper;
 extern const struct bench bench_strncmp_helper;
+extern const struct bench bench_bpf_hashmap_full_update;
 static const struct bench *benchs[] = {
 	&bench_count_global,
 	&bench_bpf_loop,
 	&bench_strncmp_no_helper,
 	&bench_strncmp_helper,
+	&bench_bpf_hashmap_full_update,
 };
 static void setup_benchmark()

tools/testing/selftests/bpf/benchs/bench_bpf_hashmap_full_update.c+96−0

@@ -0,0 +1,96 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2022 Bytedance */
	3
	4	#include <argp.h>
	5	#include "bench.h"
	6	#include "bpf_hashmap_full_update_bench.skel.h"
	7	#include "bpf_util.h"
	8
	9	/* BPF triggering benchmarks */
	10	static struct ctx {
	11	struct bpf_hashmap_full_update_bench *skel;
	12	} ctx;
	13
	14	#define MAX_LOOP_NUM 10000
	15
	16	static void validate(void)
	17	{
	18	if (env.consumer_cnt != 1) {
	19	fprintf(stderr, "benchmark doesn't support multi-consumer!\n");
	20	exit(1);
	21	}
	22	}
	23
	24	static void producer(void input)
	25	{
	26	while (true) {
	27	/* trigger the bpf program */
	28	syscall(__NR_getpgid);
	29	}
	30
	31	return NULL;
	32	}
	33
	34	static void consumer(void input)
	35	{
	36	return NULL;
	37	}
	38
	39	static void measure(struct bench_res *res)
	40	{
	41	}
	42
	43	static void setup(void)
	44	{
	45	struct bpf_link *link;
	46	int map_fd, i, max_entries;
	47
	48	setup_libbpf();
	49
	50	ctx.skel = bpf_hashmap_full_update_bench__open_and_load();
	51	if (!ctx.skel) {
	52	fprintf(stderr, "failed to open skeleton\n");
	53	exit(1);
	54	}
	55
	56	ctx.skel->bss->nr_loops = MAX_LOOP_NUM;
	57
	58	link = bpf_program__attach(ctx.skel->progs.benchmark);
	59	if (!link) {
	60	fprintf(stderr, "failed to attach program!\n");
	61	exit(1);
	62	}
	63
	64	/* fill hash_map */
	65	map_fd = bpf_map__fd(ctx.skel->maps.hash_map_bench);
	66	max_entries = bpf_map__max_entries(ctx.skel->maps.hash_map_bench);
	67	for (i = 0; i < max_entries; i++)
	68	bpf_map_update_elem(map_fd, &i, &i, BPF_ANY);
	69	}
	70
	71	void hashmap_report_final(struct bench_res res[], int res_cnt)
	72	{
	73	unsigned int nr_cpus = bpf_num_possible_cpus();
	74	int i;
	75
	76	for (i = 0; i < nr_cpus; i++) {
	77	u64 time = ctx.skel->bss->percpu_time[i];
	78
	79	if (!time)
	80	continue;
	81
	82	printf("%d:hash_map_full_perf %lld events per sec\n",
	83	i, ctx.skel->bss->nr_loops * 1000000000ll / time);
	84	}
	85	}
	86
	87	const struct bench bench_bpf_hashmap_full_update = {
	88	.name = "bpf-hashmap-ful-update",
	89	.validate = validate,
	90	.setup = setup,
	91	.producer_thread = producer,
	92	.consumer_thread = consumer,
	93	.measure = measure,
	94	.report_progress = NULL,
	95	.report_final = hashmap_report_final,
	96	};

tools/testing/selftests/bpf/benchs/run_bench_bpf_hashmap_full_update.sh+11−0

@@ -0,0 +1,11 @@
	1	#!/bin/bash
	2	# SPDX-License-Identifier: GPL-2.0
	3
	4	source ./benchs/run_common.sh
	5
	6	set -eufo pipefail
	7
	8	nr_threads=`expr $(cat /proc/cpuinfo \| grep "processor"\| wc -l) - 1`
	9	summary=$($RUN_BENCH -p $nr_threads bpf-hashmap-ful-update)
	10	printf "$summary"
	11	printf "\n"

tools/testing/selftests/bpf/btf_helpers.c+22−3

 	[BTF_KIND_FLOAT]	= "FLOAT",
 	[BTF_KIND_DECL_TAG]	= "DECL_TAG",
 	[BTF_KIND_TYPE_TAG]	= "TYPE_TAG",
+	[BTF_KIND_ENUM64]	= "ENUM64",
 };
 static const char *btf_kind_str(__u16 kind)
 {
-	if (kind > BTF_KIND_TYPE_TAG)
+	if (kind > BTF_KIND_ENUM64)
 		return "UNKNOWN";
 	return btf_kind_str_mapping[kind];
 }
 	}
 	case BTF_KIND_ENUM: {
 		const struct btf_enum *v = btf_enum(t);
+		const char *fmt_str;
-		fprintf(out, " size=%u vlen=%u", t->size, vlen);
+		fmt_str = btf_kflag(t) ? "\n\t'%s' val=%d" : "\n\t'%s' val=%u";
+		fprintf(out, " encoding=%s size=%u vlen=%u",
+			btf_kflag(t) ? "SIGNED" : "UNSIGNED", t->size, vlen);
 		for (i = 0; i < vlen; i++, v++) {
-			fprintf(out, "\n\t'%s' val=%u",
+			fprintf(out, fmt_str,
 				btf_str(btf, v->name_off), v->val);
 		}
 		break;
 	}
+	case BTF_KIND_ENUM64: {
+		const struct btf_enum64 *v = btf_enum64(t);
+		const char *fmt_str;
+		fmt_str = btf_kflag(t) ? "\n\t'%s' val=%lld" : "\n\t'%s' val=%llu";
+		fprintf(out, " encoding=%s size=%u vlen=%u",
+			btf_kflag(t) ? "SIGNED" : "UNSIGNED", t->size, vlen);
+		for (i = 0; i < vlen; i++, v++) {
+			fprintf(out, fmt_str,
+				btf_str(btf, v->name_off),
+				((__u64)v->val_hi32 << 32) | v->val_lo32);
+		}
+		break;
+	}
 	case BTF_KIND_FWD:
 		fprintf(out, " fwd_kind=%s", btf_kflag(t) ? "union" : "struct");
 		break;

tools/testing/selftests/bpf/prog_tests/attach_probe.c+48−1

@@ -17,6 +17,14 @@ static void trigger_func2(void) · 16 unmodified lines
17	asm volatile ("");	17	asm volatile ("");
18	}	18	}
19		19
		20	/* attach point for byname sleepable uprobe */
		21	static void trigger_func3(void)
		22	{
		23	asm volatile ("");
		24	}
		25
		26	static char test_data[] = "test_data";
		27
20	void test_attach_probe(void)	28	void test_attach_probe(void)
21	{	29	{
22	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts);	30	DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, uprobe_opts);
@@ -49,9 +57,17 @@ void test_attach_probe(void) · 26 unmodified lines
49	if (!ASSERT_GE(ref_ctr_offset, 0, "ref_ctr_offset"))	57	if (!ASSERT_GE(ref_ctr_offset, 0, "ref_ctr_offset"))
50	return;	58	return;
51		59
52	skel = test_attach_probe__open_and_load();	60	skel = test_attach_probe__open();
53	if (!ASSERT_OK_PTR(skel, "skel_open"))	61	if (!ASSERT_OK_PTR(skel, "skel_open"))
54	return;	62	return;
		63
		64	/* sleepable kprobe test case needs flags set before loading */
		65	if (!ASSERT_OK(bpf_program__set_flags(skel->progs.handle_kprobe_sleepable,
		66	BPF_F_SLEEPABLE), "kprobe_sleepable_flags"))
		67	goto cleanup;
		68
		69	if (!ASSERT_OK(test_attach_probe__load(skel), "skel_load"))
		70	goto cleanup;
55	if (!ASSERT_OK_PTR(skel->bss, "check_bss"))	71	if (!ASSERT_OK_PTR(skel->bss, "check_bss"))
56	goto cleanup;	72	goto cleanup;
57		73
@@ -151,6 +167,30 @@ void test_attach_probe(void) · 93 unmodified lines
151	if (!ASSERT_OK_PTR(skel->links.handle_uretprobe_byname2, "attach_uretprobe_byname2"))	167	if (!ASSERT_OK_PTR(skel->links.handle_uretprobe_byname2, "attach_uretprobe_byname2"))
152	goto cleanup;	168	goto cleanup;
153		169
		170	/* sleepable kprobes should not attach successfully */
		171	skel->links.handle_kprobe_sleepable = bpf_program__attach(skel->progs.handle_kprobe_sleepable);
		172	if (!ASSERT_ERR_PTR(skel->links.handle_kprobe_sleepable, "attach_kprobe_sleepable"))
		173	goto cleanup;
		174
		175	/* test sleepable uprobe and uretprobe variants */
		176	skel->links.handle_uprobe_byname3_sleepable = bpf_program__attach(skel->progs.handle_uprobe_byname3_sleepable);
		177	if (!ASSERT_OK_PTR(skel->links.handle_uprobe_byname3_sleepable, "attach_uprobe_byname3_sleepable"))
		178	goto cleanup;
		179
		180	skel->links.handle_uprobe_byname3 = bpf_program__attach(skel->progs.handle_uprobe_byname3);
		181	if (!ASSERT_OK_PTR(skel->links.handle_uprobe_byname3, "attach_uprobe_byname3"))
		182	goto cleanup;
		183
		184	skel->links.handle_uretprobe_byname3_sleepable = bpf_program__attach(skel->progs.handle_uretprobe_byname3_sleepable);
		185	if (!ASSERT_OK_PTR(skel->links.handle_uretprobe_byname3_sleepable, "attach_uretprobe_byname3_sleepable"))
		186	goto cleanup;
		187
		188	skel->links.handle_uretprobe_byname3 = bpf_program__attach(skel->progs.handle_uretprobe_byname3);
		189	if (!ASSERT_OK_PTR(skel->links.handle_uretprobe_byname3, "attach_uretprobe_byname3"))
		190	goto cleanup;
		191
		192	skel->bss->user_ptr = test_data;
		193
154	/* trigger & validate kprobe && kretprobe */	194	/* trigger & validate kprobe && kretprobe */
155	usleep(1);	195	usleep(1);
156		196
@@ -164,6 +204,9 @@ void test_attach_probe(void) · 7 unmodified lines
164	/* trigger & validate uprobe attached by name */	204	/* trigger & validate uprobe attached by name */
165	trigger_func2();	205	trigger_func2();
166		206
		207	/* trigger & validate sleepable uprobe attached by name */
		208	trigger_func3();
		209
167	ASSERT_EQ(skel->bss->kprobe_res, 1, "check_kprobe_res");	210	ASSERT_EQ(skel->bss->kprobe_res, 1, "check_kprobe_res");
168	ASSERT_EQ(skel->bss->kprobe2_res, 11, "check_kprobe_auto_res");	211	ASSERT_EQ(skel->bss->kprobe2_res, 11, "check_kprobe_auto_res");
169	ASSERT_EQ(skel->bss->kretprobe_res, 2, "check_kretprobe_res");	212	ASSERT_EQ(skel->bss->kretprobe_res, 2, "check_kretprobe_res");
@@ -174,6 +217,10 @@ void test_attach_probe(void) · 4 unmodified lines
174	ASSERT_EQ(skel->bss->uretprobe_byname_res, 6, "check_uretprobe_byname_res");	217	ASSERT_EQ(skel->bss->uretprobe_byname_res, 6, "check_uretprobe_byname_res");
175	ASSERT_EQ(skel->bss->uprobe_byname2_res, 7, "check_uprobe_byname2_res");	218	ASSERT_EQ(skel->bss->uprobe_byname2_res, 7, "check_uprobe_byname2_res");
176	ASSERT_EQ(skel->bss->uretprobe_byname2_res, 8, "check_uretprobe_byname2_res");	219	ASSERT_EQ(skel->bss->uretprobe_byname2_res, 8, "check_uretprobe_byname2_res");
		220	ASSERT_EQ(skel->bss->uprobe_byname3_sleepable_res, 9, "check_uprobe_byname3_sleepable_res");
		221	ASSERT_EQ(skel->bss->uprobe_byname3_res, 10, "check_uprobe_byname3_res");
		222	ASSERT_EQ(skel->bss->uretprobe_byname3_sleepable_res, 11, "check_uretprobe_byname3_sleepable_res");
		223	ASSERT_EQ(skel->bss->uretprobe_byname3_res, 12, "check_uretprobe_byname3_res");
177		224
178	cleanup:	225	cleanup:
179	test_attach_probe__destroy(skel);	226	test_attach_probe__destroy(skel);

tools/testing/selftests/bpf/prog_tests/btf.c+131−22

@@ -2896,26 +2896,6 @@ static struct btf_raw_test raw_tests[] = { · 2895 unmodified lines
2896	.err_str = "Invalid btf_info kind_flag",	2896	.err_str = "Invalid btf_info kind_flag",
2897	},	2897	},
2898		2898
2899	{
2900	.descr = "invalid enum kind_flag",
2901	.raw_types = {
2902	BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
2903	BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_ENUM, 1, 1), 4), /* [2] */
2904	BTF_ENUM_ENC(NAME_TBD, 0),
2905	BTF_END_RAW,
2906	},
2907	BTF_STR_SEC("\0A"),
2908	.map_type = BPF_MAP_TYPE_ARRAY,
2909	.map_name = "enum_type_check_btf",
2910	.key_size = sizeof(int),
2911	.value_size = sizeof(int),
2912	.key_type_id = 1,
2913	.value_type_id = 1,
2914	.max_entries = 4,
2915	.btf_load_err = true,
2916	.err_str = "Invalid btf_info kind_flag",
2917	},
2918
2919	{	2899	{
2920	.descr = "valid fwd kind_flag",	2900	.descr = "valid fwd kind_flag",
2921	.raw_types = {	2901	.raw_types = {
@@ -4072,6 +4052,42 @@ static struct btf_raw_test raw_tests[] = { · 1150 unmodified lines
4072	.btf_load_err = true,	4052	.btf_load_err = true,
4073	.err_str = "Type tags don't precede modifiers",	4053	.err_str = "Type tags don't precede modifiers",
4074	},	4054	},
		4055	{
		4056	.descr = "enum64 test #1, unsigned, size 8",
		4057	.raw_types = {
		4058	BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
		4059	BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 2), 8), /* [2] */
		4060	BTF_ENUM64_ENC(NAME_TBD, 0, 0),
		4061	BTF_ENUM64_ENC(NAME_TBD, 1, 1),
		4062	BTF_END_RAW,
		4063	},
		4064	BTF_STR_SEC("\0a\0b\0c"),
		4065	.map_type = BPF_MAP_TYPE_ARRAY,
		4066	.map_name = "tag_type_check_btf",
		4067	.key_size = sizeof(int),
		4068	.value_size = 8,
		4069	.key_type_id = 1,
		4070	.value_type_id = 2,
		4071	.max_entries = 1,
		4072	},
		4073	{
		4074	.descr = "enum64 test #2, signed, size 4",
		4075	.raw_types = {
		4076	BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
		4077	BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_ENUM64, 1, 2), 4), /* [2] */
		4078	BTF_ENUM64_ENC(NAME_TBD, -1, 0),
		4079	BTF_ENUM64_ENC(NAME_TBD, 1, 0),
		4080	BTF_END_RAW,
		4081	},
		4082	BTF_STR_SEC("\0a\0b\0c"),
		4083	.map_type = BPF_MAP_TYPE_ARRAY,
		4084	.map_name = "tag_type_check_btf",
		4085	.key_size = sizeof(int),
		4086	.value_size = 4,
		4087	.key_type_id = 1,
		4088	.value_type_id = 2,
		4089	.max_entries = 1,
		4090	},
4075		4091
4076	}; /* struct btf_raw_test raw_tests[] */	4092	}; /* struct btf_raw_test raw_tests[] */
4077		4093
@@ -7000,9 +7016,12 @@ static struct btf_dedup_test dedup_tests[] = { · 2922 unmodified lines
7000	BTF_DECL_TAG_ENC(NAME_TBD, 13, 1), /* [16] decl_tag */	7016	BTF_DECL_TAG_ENC(NAME_TBD, 13, 1), /* [16] decl_tag */
7001	BTF_DECL_TAG_ENC(NAME_TBD, 7, -1), /* [17] decl_tag */	7017	BTF_DECL_TAG_ENC(NAME_TBD, 7, -1), /* [17] decl_tag */
7002	BTF_TYPE_TAG_ENC(NAME_TBD, 8), /* [18] type_tag */	7018	BTF_TYPE_TAG_ENC(NAME_TBD, 8), /* [18] type_tag */
		7019	BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 2), 8), /* [19] enum64 */
		7020	BTF_ENUM64_ENC(NAME_TBD, 0, 0),
		7021	BTF_ENUM64_ENC(NAME_TBD, 1, 1),
7003	BTF_END_RAW,	7022	BTF_END_RAW,
7004	},	7023	},
7005	BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q\0R"),	7024	BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q\0R\0S\0T\0U"),
7006	},	7025	},
7007	.expect = {	7026	.expect = {
7008	.raw_types = {	7027	.raw_types = {
@@ -7030,9 +7049,12 @@ static struct btf_dedup_test dedup_tests[] = { · 21 unmodified lines
7030	BTF_DECL_TAG_ENC(NAME_TBD, 13, 1), /* [16] decl_tag */	7049	BTF_DECL_TAG_ENC(NAME_TBD, 13, 1), /* [16] decl_tag */
7031	BTF_DECL_TAG_ENC(NAME_TBD, 7, -1), /* [17] decl_tag */	7050	BTF_DECL_TAG_ENC(NAME_TBD, 7, -1), /* [17] decl_tag */
7032	BTF_TYPE_TAG_ENC(NAME_TBD, 8), /* [18] type_tag */	7051	BTF_TYPE_TAG_ENC(NAME_TBD, 8), /* [18] type_tag */
		7052	BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 2), 8), /* [19] enum64 */
		7053	BTF_ENUM64_ENC(NAME_TBD, 0, 0),
		7054	BTF_ENUM64_ENC(NAME_TBD, 1, 1),
7033	BTF_END_RAW,	7055	BTF_END_RAW,
7034	},	7056	},
7035	BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q\0R"),	7057	BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q\0R\0S\0T\0U"),
7036	},	7058	},
7037	},	7059	},
7038	{	7060	{
@@ -7493,6 +7515,91 @@ static struct btf_dedup_test dedup_tests[] = { · 454 unmodified lines
7493	BTF_STR_SEC("\0tag1\0t\0m"),	7515	BTF_STR_SEC("\0tag1\0t\0m"),
7494	},	7516	},
7495	},	7517	},
		7518	{
		7519	.descr = "dedup: enum64, standalone",
		7520	.input = {
		7521	.raw_types = {
		7522	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7523	BTF_ENUM64_ENC(NAME_NTH(2), 1, 123),
		7524	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7525	BTF_ENUM64_ENC(NAME_NTH(2), 1, 123),
		7526	BTF_END_RAW,
		7527	},
		7528	BTF_STR_SEC("\0e1\0e1_val"),
		7529	},
		7530	.expect = {
		7531	.raw_types = {
		7532	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7533	BTF_ENUM64_ENC(NAME_NTH(2), 1, 123),
		7534	BTF_END_RAW,
		7535	},
		7536	BTF_STR_SEC("\0e1\0e1_val"),
		7537	},
		7538	},
		7539	{
		7540	.descr = "dedup: enum64, fwd resolution",
		7541	.input = {
		7542	.raw_types = {
		7543	/* [1] fwd enum64 'e1' before full enum */
		7544	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
		7545	/* [2] full enum64 'e1' after fwd */
		7546	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7547	BTF_ENUM64_ENC(NAME_NTH(2), 1, 123),
		7548	/* [3] full enum64 'e2' before fwd */
		7549	BTF_TYPE_ENC(NAME_NTH(3), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7550	BTF_ENUM64_ENC(NAME_NTH(4), 0, 456),
		7551	/* [4] fwd enum64 'e2' after full enum */
		7552	BTF_TYPE_ENC(NAME_NTH(3), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
		7553	/* [5] incompatible full enum64 with different value */
		7554	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7555	BTF_ENUM64_ENC(NAME_NTH(2), 0, 321),
		7556	BTF_END_RAW,
		7557	},
		7558	BTF_STR_SEC("\0e1\0e1_val\0e2\0e2_val"),
		7559	},
		7560	.expect = {
		7561	.raw_types = {
		7562	/* [1] full enum64 'e1' */
		7563	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7564	BTF_ENUM64_ENC(NAME_NTH(2), 1, 123),
		7565	/* [2] full enum64 'e2' */
		7566	BTF_TYPE_ENC(NAME_NTH(3), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7567	BTF_ENUM64_ENC(NAME_NTH(4), 0, 456),
		7568	/* [3] incompatible full enum64 with different value */
		7569	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 8),
		7570	BTF_ENUM64_ENC(NAME_NTH(2), 0, 321),
		7571	BTF_END_RAW,
		7572	},
		7573	BTF_STR_SEC("\0e1\0e1_val\0e2\0e2_val"),
		7574	},
		7575	},
		7576	{
		7577	.descr = "dedup: enum and enum64, no dedup",
		7578	.input = {
		7579	.raw_types = {
		7580	/* [1] enum 'e1' */
		7581	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM, 0, 1), 4),
		7582	BTF_ENUM_ENC(NAME_NTH(2), 1),
		7583	/* [2] enum64 'e1' */
		7584	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 4),
		7585	BTF_ENUM64_ENC(NAME_NTH(2), 1, 0),
		7586	BTF_END_RAW,
		7587	},
		7588	BTF_STR_SEC("\0e1\0e1_val"),
		7589	},
		7590	.expect = {
		7591	.raw_types = {
		7592	/* [1] enum 'e1' */
		7593	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM, 0, 1), 4),
		7594	BTF_ENUM_ENC(NAME_NTH(2), 1),
		7595	/* [2] enum64 'e1' */
		7596	BTF_TYPE_ENC(NAME_NTH(1), BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 1), 4),
		7597	BTF_ENUM64_ENC(NAME_NTH(2), 1, 0),
		7598	BTF_END_RAW,
		7599	},
		7600	BTF_STR_SEC("\0e1\0e1_val"),
		7601	},
		7602	},
7496		7603
7497	};	7604	};
7498		7605
@@ -7517,6 +7624,8 @@ static int btf_type_size(const struct btf_type *t) · 18 unmodified lines
7517	return base_size + sizeof(__u32);	7624	return base_size + sizeof(__u32);
7518	case BTF_KIND_ENUM:	7625	case BTF_KIND_ENUM:
7519	return base_size + vlen * sizeof(struct btf_enum);	7626	return base_size + vlen * sizeof(struct btf_enum);
		7627	case BTF_KIND_ENUM64:
		7628	return base_size + vlen * sizeof(struct btf_enum64);
7520	case BTF_KIND_ARRAY:	7629	case BTF_KIND_ARRAY:
7521	return base_size + sizeof(struct btf_array);	7630	return base_size + sizeof(struct btf_array);
7522	case BTF_KIND_STRUCT:	7631	case BTF_KIND_STRUCT:

tools/testing/selftests/bpf/prog_tests/btf_write.c+92−34

 	const struct btf_var_secinfo *vi;
 	const struct btf_type *t;
 	const struct btf_member *m;
+	const struct btf_enum64 *v64;
 	const struct btf_enum *v;
 	const struct btf_param *p;
 	int id, err, str_off;
 	ASSERT_STREQ(btf__str_by_offset(btf, v->name_off), "v2", "v2_name");
 	ASSERT_EQ(v->val, 2, "v2_val");
 	ASSERT_STREQ(btf_type_raw_dump(btf, 9),
-		     "[9] ENUM 'e1' size=4 vlen=2\n"
+		     "[9] ENUM 'e1' encoding=UNSIGNED size=4 vlen=2\n"
 		     "\t'v1' val=1\n"
 		     "\t'v2' val=2", "raw_dump");
 	ASSERT_EQ(btf_vlen(t), 0, "enum_fwd_kind");
 	ASSERT_EQ(t->size, 4, "enum_fwd_sz");
 	ASSERT_STREQ(btf_type_raw_dump(btf, 12),
-		     "[12] ENUM 'enum_fwd' size=4 vlen=0", "raw_dump");
+		     "[12] ENUM 'enum_fwd' encoding=UNSIGNED size=4 vlen=0", "raw_dump");
 	/* TYPEDEF */
 	id = btf__add_typedef(btf, "typedef1", 1);
 	ASSERT_EQ(t->type, 1, "tag_type");
 	ASSERT_STREQ(btf_type_raw_dump(btf, 20),
 		     "[20] TYPE_TAG 'tag1' type_id=1", "raw_dump");
+	/* ENUM64 */
+	id = btf__add_enum64(btf, "e1", 8, true);
+	ASSERT_EQ(id, 21, "enum64_id");
+	err = btf__add_enum64_value(btf, "v1", -1);
+	ASSERT_OK(err, "v1_res");
+	err = btf__add_enum64_value(btf, "v2", 0x123456789); /* 4886718345 */
+	ASSERT_OK(err, "v2_res");
+	t = btf__type_by_id(btf, 21);
+	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "e1", "enum64_name");
+	ASSERT_EQ(btf_kind(t), BTF_KIND_ENUM64, "enum64_kind");
+	ASSERT_EQ(btf_vlen(t), 2, "enum64_vlen");
+	ASSERT_EQ(t->size, 8, "enum64_sz");
+	v64 = btf_enum64(t) + 0;
+	ASSERT_STREQ(btf__str_by_offset(btf, v64->name_off), "v1", "v1_name");
+	ASSERT_EQ(v64->val_hi32, 0xffffffff, "v1_val");
+	ASSERT_EQ(v64->val_lo32, 0xffffffff, "v1_val");
+	v64 = btf_enum64(t) + 1;
+	ASSERT_STREQ(btf__str_by_offset(btf, v64->name_off), "v2", "v2_name");
+	ASSERT_EQ(v64->val_hi32, 0x1, "v2_val");
+	ASSERT_EQ(v64->val_lo32, 0x23456789, "v2_val");
+	ASSERT_STREQ(btf_type_raw_dump(btf, 21),
+		     "[21] ENUM64 'e1' encoding=SIGNED size=8 vlen=2\n"
+		     "\t'v1' val=-1\n"
+		     "\t'v2' val=4886718345", "raw_dump");
+	id = btf__add_enum64(btf, "e1", 8, false);
+	ASSERT_EQ(id, 22, "enum64_id");
+	err = btf__add_enum64_value(btf, "v1", 0xffffffffFFFFFFFF); /* 18446744073709551615 */
+	ASSERT_OK(err, "v1_res");
+	t = btf__type_by_id(btf, 22);
+	ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "e1", "enum64_name");
+	ASSERT_EQ(btf_kind(t), BTF_KIND_ENUM64, "enum64_kind");
+	ASSERT_EQ(btf_vlen(t), 1, "enum64_vlen");
+	ASSERT_EQ(t->size, 8, "enum64_sz");
+	v64 = btf_enum64(t) + 0;
+	ASSERT_STREQ(btf__str_by_offset(btf, v64->name_off), "v1", "v1_name");
+	ASSERT_EQ(v64->val_hi32, 0xffffffff, "v1_val");
+	ASSERT_EQ(v64->val_lo32, 0xffffffff, "v1_val");
+	ASSERT_STREQ(btf_type_raw_dump(btf, 22),
+		     "[22] ENUM64 'e1' encoding=UNSIGNED size=8 vlen=1\n"
+		     "\t'v1' val=18446744073709551615", "raw_dump");
 }
 static void test_btf_add()
 		"\t'f2' type_id=1 bits_offset=32 bitfield_size=16",
 		"[8] UNION 'u1' size=8 vlen=1\n"
 		"\t'f1' type_id=1 bits_offset=0 bitfield_size=16",
-		"[9] ENUM 'e1' size=4 vlen=2\n"
+		"[9] ENUM 'e1' encoding=UNSIGNED size=4 vlen=2\n"
 		"\t'v1' val=1\n"
 		"\t'v2' val=2",
 		"[10] FWD 'struct_fwd' fwd_kind=struct",
 		"[11] FWD 'union_fwd' fwd_kind=union",
-		"[12] ENUM 'enum_fwd' size=4 vlen=0",
+		"[12] ENUM 'enum_fwd' encoding=UNSIGNED size=4 vlen=0",
 		"[13] TYPEDEF 'typedef1' type_id=1",
 		"[14] FUNC 'func1' type_id=15 linkage=global",
 		"[15] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n"
 		"\ttype_id=1 offset=4 size=8",
 		"[18] DECL_TAG 'tag1' type_id=16 component_idx=-1",
 		"[19] DECL_TAG 'tag2' type_id=14 component_idx=1",
-		"[20] TYPE_TAG 'tag1' type_id=1");
+		"[20] TYPE_TAG 'tag1' type_id=1",
+		"[21] ENUM64 'e1' encoding=SIGNED size=8 vlen=2\n"
+		"\t'v1' val=-1\n"
+		"\t'v2' val=4886718345",
+		"[22] ENUM64 'e1' encoding=UNSIGNED size=8 vlen=1\n"
+		"\t'v1' val=18446744073709551615");
 	btf__free(btf);
 }
 	gen_btf(btf2);
 	id = btf__add_btf(btf1, btf2);
-	if (!ASSERT_EQ(id, 21, "id"))
+	if (!ASSERT_EQ(id, 23, "id"))
 		goto cleanup;
 	VALIDATE_RAW_BTF(
 		"\t'f2' type_id=1 bits_offset=32 bitfield_size=16",
 		"[8] UNION 'u1' size=8 vlen=1\n"
 		"\t'f1' type_id=1 bits_offset=0 bitfield_size=16",
-		"[9] ENUM 'e1' size=4 vlen=2\n"
+		"[9] ENUM 'e1' encoding=UNSIGNED size=4 vlen=2\n"
 		"\t'v1' val=1\n"
 		"\t'v2' val=2",
 		"[10] FWD 'struct_fwd' fwd_kind=struct",
 		"[11] FWD 'union_fwd' fwd_kind=union",
-		"[12] ENUM 'enum_fwd' size=4 vlen=0",
+		"[12] ENUM 'enum_fwd' encoding=UNSIGNED size=4 vlen=0",
 		"[13] TYPEDEF 'typedef1' type_id=1",
 		"[14] FUNC 'func1' type_id=15 linkage=global",
 		"[15] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n"
 		"[18] DECL_TAG 'tag1' type_id=16 component_idx=-1",
 		"[19] DECL_TAG 'tag2' type_id=14 component_idx=1",
 		"[20] TYPE_TAG 'tag1' type_id=1",
+		"[21] ENUM64 'e1' encoding=SIGNED size=8 vlen=2\n"
+		"\t'v1' val=-1\n"
+		"\t'v2' val=4886718345",
+		"[22] ENUM64 'e1' encoding=UNSIGNED size=8 vlen=1\n"
+		"\t'v1' val=18446744073709551615",
 		/* types appended from the second BTF */
-		"[21] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
+		"[23] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
-		"[22] PTR '(anon)' type_id=21",
+		"[24] PTR '(anon)' type_id=23",
-		"[23] CONST '(anon)' type_id=25",
+		"[25] CONST '(anon)' type_id=27",
-		"[24] VOLATILE '(anon)' type_id=23",
+		"[26] VOLATILE '(anon)' type_id=25",
-		"[25] RESTRICT '(anon)' type_id=24",
+		"[27] RESTRICT '(anon)' type_id=26",
-		"[26] ARRAY '(anon)' type_id=22 index_type_id=21 nr_elems=10",
+		"[28] ARRAY '(anon)' type_id=24 index_type_id=23 nr_elems=10",
-		"[27] STRUCT 's1' size=8 vlen=2\n"
+		"[29] STRUCT 's1' size=8 vlen=2\n"
-		"\t'f1' type_id=21 bits_offset=0\n"
+		"\t'f1' type_id=23 bits_offset=0\n"
-		"\t'f2' type_id=21 bits_offset=32 bitfield_size=16",
+		"\t'f2' type_id=23 bits_offset=32 bitfield_size=16",
-		"[28] UNION 'u1' size=8 vlen=1\n"
+		"[30] UNION 'u1' size=8 vlen=1\n"
-		"\t'f1' type_id=21 bits_offset=0 bitfield_size=16",
+		"\t'f1' type_id=23 bits_offset=0 bitfield_size=16",
-		"[29] ENUM 'e1' size=4 vlen=2\n"
+		"[31] ENUM 'e1' encoding=UNSIGNED size=4 vlen=2\n"
 		"\t'v1' val=1\n"
 		"\t'v2' val=2",
-		"[30] FWD 'struct_fwd' fwd_kind=struct",
+		"[32] FWD 'struct_fwd' fwd_kind=struct",
-		"[31] FWD 'union_fwd' fwd_kind=union",
+		"[33] FWD 'union_fwd' fwd_kind=union",
-		"[32] ENUM 'enum_fwd' size=4 vlen=0",
+		"[34] ENUM 'enum_fwd' encoding=UNSIGNED size=4 vlen=0",
-		"[33] TYPEDEF 'typedef1' type_id=21",
+		"[35] TYPEDEF 'typedef1' type_id=23",
-		"[34] FUNC 'func1' type_id=35 linkage=global",
+		"[36] FUNC 'func1' type_id=37 linkage=global",
-		"[35] FUNC_PROTO '(anon)' ret_type_id=21 vlen=2\n"
+		"[37] FUNC_PROTO '(anon)' ret_type_id=23 vlen=2\n"
-		"\t'p1' type_id=21\n"
+		"\t'p1' type_id=23\n"
-		"\t'p2' type_id=22",
+		"\t'p2' type_id=24",
-		"[36] VAR 'var1' type_id=21, linkage=global-alloc",
+		"[38] VAR 'var1' type_id=23, linkage=global-alloc",
-		"[37] DATASEC 'datasec1' size=12 vlen=1\n"
+		"[39] DATASEC 'datasec1' size=12 vlen=1\n"
-		"\ttype_id=21 offset=4 size=8",
+		"\ttype_id=23 offset=4 size=8",
-		"[38] DECL_TAG 'tag1' type_id=36 component_idx=-1",
+		"[40] DECL_TAG 'tag1' type_id=38 component_idx=-1",
-		"[39] DECL_TAG 'tag2' type_id=34 component_idx=1",
+		"[41] DECL_TAG 'tag2' type_id=36 component_idx=1",
-		"[40] TYPE_TAG 'tag1' type_id=21");
+		"[42] TYPE_TAG 'tag1' type_id=23",
+		"[43] ENUM64 'e1' encoding=SIGNED size=8 vlen=2\n"
+		"\t'v1' val=-1\n"
+		"\t'v2' val=4886718345",
+		"[44] ENUM64 'e1' encoding=UNSIGNED size=8 vlen=1\n"
+		"\t'v1' val=18446744073709551615");
 cleanup:
 	btf__free(btf1);

tools/testing/selftests/bpf/prog_tests/core_reloc.c+64−1

@@ -84,6 +84,7 @@ static int duration = 0; · 83 unmodified lines
84	#define NESTING_ERR_CASE(name) { \	84	#define NESTING_ERR_CASE(name) { \
85	NESTING_CASE_COMMON(name), \	85	NESTING_CASE_COMMON(name), \
86	.fails = true, \	86	.fails = true, \
		87	.run_btfgen_fails = true, \
87	}	88	}
88		89
89	#define ARRAYS_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \	90	#define ARRAYS_DATA(struct_name) STRUCT_TO_CHAR_PTR(struct_name) { \
@@ -258,12 +259,14 @@ static int duration = 0; · 168 unmodified lines
258	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \	259	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o", \
259	"probed:", name), \	260	"probed:", name), \
260	.fails = true, \	261	.fails = true, \
		262	.run_btfgen_fails = true, \
261	.raw_tp_name = "sys_enter", \	263	.raw_tp_name = "sys_enter", \
262	.prog_name = "test_core_bitfields", \	264	.prog_name = "test_core_bitfields", \
263	}, { \	265	}, { \
264	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \	266	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o", \
265	"direct:", name), \	267	"direct:", name), \
266	.fails = true, \	268	.fails = true, \
		269	.run_btfgen_fails = true, \
267	.prog_name = "test_core_bitfields_direct", \	270	.prog_name = "test_core_bitfields_direct", \
268	}	271	}
269		272
@@ -304,6 +307,7 @@ static int duration = 0; · 34 unmodified lines
304	#define SIZE_ERR_CASE(name) { \	307	#define SIZE_ERR_CASE(name) { \
305	SIZE_CASE_COMMON(name), \	308	SIZE_CASE_COMMON(name), \
306	.fails = true, \	309	.fails = true, \
		310	.run_btfgen_fails = true, \
307	}	311	}
308		312
309	#define TYPE_BASED_CASE_COMMON(name) \	313	#define TYPE_BASED_CASE_COMMON(name) \
@@ -363,6 +367,25 @@ static int duration = 0; · 53 unmodified lines
363	.fails = true, \	367	.fails = true, \
364	}	368	}
365		369
		370	#define ENUM64VAL_CASE_COMMON(name) \
		371	.case_name = #name, \
		372	.bpf_obj_file = "test_core_reloc_enum64val.o", \
		373	.btf_src_file = "btf__core_reloc_" #name ".o", \
		374	.raw_tp_name = "sys_enter", \
		375	.prog_name = "test_core_enum64val"
		376
		377	#define ENUM64VAL_CASE(name, ...) { \
		378	ENUM64VAL_CASE_COMMON(name), \
		379	.output = STRUCT_TO_CHAR_PTR(core_reloc_enum64val_output) \
		380	__VA_ARGS__, \
		381	.output_len = sizeof(struct core_reloc_enum64val_output), \
		382	}
		383
		384	#define ENUM64VAL_ERR_CASE(name) { \
		385	ENUM64VAL_CASE_COMMON(name), \
		386	.fails = true, \
		387	}
		388
366	struct core_reloc_test_case;	389	struct core_reloc_test_case;
367		390
368	typedef int (setup_test_fn)(struct core_reloc_test_case test);	391	typedef int (setup_test_fn)(struct core_reloc_test_case test);
@@ -377,6 +400,7 @@ struct core_reloc_test_case { · 8 unmodified lines
377	const char *output;	400	const char *output;
378	int output_len;	401	int output_len;
379	bool fails;	402	bool fails;
		403	bool run_btfgen_fails;
380	bool needs_testmod;	404	bool needs_testmod;
381	bool relaxed_core_relocs;	405	bool relaxed_core_relocs;
382	const char *prog_name;	406	const char *prog_name;
@@ -831,6 +855,45 @@ static const struct core_reloc_test_case test_cases[] = { · 448 unmodified lines
831	.anon_val2 = 0x222,	855	.anon_val2 = 0x222,
832	}),	856	}),
833	ENUMVAL_ERR_CASE(enumval___err_missing),	857	ENUMVAL_ERR_CASE(enumval___err_missing),
		858
		859	/* 64bit enumerator value existence and value relocations */
		860	ENUM64VAL_CASE(enum64val, {
		861	.unsigned_val1_exists = true,
		862	.unsigned_val2_exists = true,
		863	.unsigned_val3_exists = true,
		864	.signed_val1_exists = true,
		865	.signed_val2_exists = true,
		866	.signed_val3_exists = true,
		867	.unsigned_val1 = 0x1ffffffffULL,
		868	.unsigned_val2 = 0x2,
		869	.signed_val1 = 0x1ffffffffLL,
		870	.signed_val2 = -2,
		871	}),
		872	ENUM64VAL_CASE(enum64val___diff, {
		873	.unsigned_val1_exists = true,
		874	.unsigned_val2_exists = true,
		875	.unsigned_val3_exists = true,
		876	.signed_val1_exists = true,
		877	.signed_val2_exists = true,
		878	.signed_val3_exists = true,
		879	.unsigned_val1 = 0x101ffffffffULL,
		880	.unsigned_val2 = 0x202ffffffffULL,
		881	.signed_val1 = -101,
		882	.signed_val2 = -202,
		883	}),
		884	ENUM64VAL_CASE(enum64val___val3_missing, {
		885	.unsigned_val1_exists = true,
		886	.unsigned_val2_exists = true,
		887	.unsigned_val3_exists = false,
		888	.signed_val1_exists = true,
		889	.signed_val2_exists = true,
		890	.signed_val3_exists = false,
		891	.unsigned_val1 = 0x111ffffffffULL,
		892	.unsigned_val2 = 0x222,
		893	.signed_val1 = 0x111ffffffffLL,
		894	.signed_val2 = -222,
		895	}),
		896	ENUM64VAL_ERR_CASE(enum64val___err_missing),
834	};	897	};
835		898
836	struct data {	899	struct data {
@@ -894,7 +957,7 @@ static void run_core_reloc_tests(bool use_btfgen) · 57 unmodified lines
894	/* generate a "minimal" BTF file and use it as source */	957	/* generate a "minimal" BTF file and use it as source */
895	if (use_btfgen) {	958	if (use_btfgen) {
896		959
897	if (!test_case->btf_src_file \|\| test_case->fails) {	960	if (!test_case->btf_src_file \|\| test_case->run_btfgen_fails) {
898	test__skip();	961	test__skip();
899	continue;	962	continue;
900	}	963	}

tools/testing/selftests/bpf/prog_tests/fexit_stress.c+4−28

 void serial_test_fexit_stress(void)
 {
-	char test_skb[128] = {};
 	int fexit_fd[CNT] = {};
 	int link_fd[CNT] = {};
-	char error[4096];
+	int err, i;
-	int err, i, filter_fd;
 	const struct bpf_insn trace_program[] = {
 		BPF_MOV64_IMM(BPF_REG_0, 0),
 	LIBBPF_OPTS(bpf_prog_load_opts, trace_opts,
 		.expected_attach_type = BPF_TRACE_FEXIT,
-		.log_buf = error,
-		.log_size = sizeof(error),
 	);
-	const struct bpf_insn skb_program[] = {
+	LIBBPF_OPTS(bpf_test_run_opts, topts);
-		BPF_MOV64_IMM(BPF_REG_0, 0),
-		BPF_EXIT_INSN(),
-	};
-	LIBBPF_OPTS(bpf_prog_load_opts, skb_opts,
-		.log_buf = error,
-		.log_size = sizeof(error),
-	);
-	LIBBPF_OPTS(bpf_test_run_opts, topts,
-		.data_in = test_skb,
-		.data_size_in = sizeof(test_skb),
-		.repeat = 1,
-	);
 	err = libbpf_find_vmlinux_btf_id("bpf_fentry_test1",
 					 trace_opts.expected_attach_type);
 			goto out;
 	}
-	filter_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL",
+	err = bpf_prog_test_run_opts(fexit_fd[0], &topts);
-				  skb_program, sizeof(skb_program) / sizeof(struct bpf_insn),
+	ASSERT_OK(err, "bpf_prog_test_run_opts");
-				  &skb_opts);
-	if (!ASSERT_GE(filter_fd, 0, "test_program_loaded"))
-		goto out;
-	err = bpf_prog_test_run_opts(filter_fd, &topts);
-	close(filter_fd);
-	CHECK_FAIL(err);
 out:
 	for (i = 0; i < CNT; i++) {
 		if (link_fd[i])

tools/testing/selftests/bpf/prog_tests/libbpf_str.c+207−0

@@ -0,0 +1,207 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
	3
	4	#include <ctype.h>
	5	#include <test_progs.h>
	6	#include <bpf/btf.h>
	7
	8	/*
	9	* Utility function uppercasing an entire string.
	10	*/
	11	static void uppercase(char *s)
	12	{
	13	for (; *s != '\0'; s++)
	14	s = toupper(s);
	15	}
	16
	17	/*
	18	* Test case to check that all bpf_attach_type variants are covered by
	19	* libbpf_bpf_attach_type_str.
	20	*/
	21	static void test_libbpf_bpf_attach_type_str(void)
	22	{
	23	struct btf *btf;
	24	const struct btf_type *t;
	25	const struct btf_enum *e;
	26	int i, n, id;
	27
	28	btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
	29	if (!ASSERT_OK_PTR(btf, "btf_parse"))
	30	return;
	31
	32	/* find enum bpf_attach_type and enumerate each value */
	33	id = btf__find_by_name_kind(btf, "bpf_attach_type", BTF_KIND_ENUM);
	34	if (!ASSERT_GT(id, 0, "bpf_attach_type_id"))
	35	goto cleanup;
	36	t = btf__type_by_id(btf, id);
	37	e = btf_enum(t);
	38	n = btf_vlen(t);
	39	for (i = 0; i < n; e++, i++) {
	40	enum bpf_attach_type attach_type = (enum bpf_attach_type)e->val;
	41	const char *attach_type_name;
	42	const char *attach_type_str;
	43	char buf[256];
	44
	45	if (attach_type == __MAX_BPF_ATTACH_TYPE)
	46	continue;
	47
	48	attach_type_name = btf__str_by_offset(btf, e->name_off);
	49	attach_type_str = libbpf_bpf_attach_type_str(attach_type);
	50	ASSERT_OK_PTR(attach_type_str, attach_type_name);
	51
	52	snprintf(buf, sizeof(buf), "BPF_%s", attach_type_str);
	53	uppercase(buf);
	54
	55	ASSERT_STREQ(buf, attach_type_name, "exp_str_value");
	56	}
	57
	58	cleanup:
	59	btf__free(btf);
	60	}
	61
	62	/*
	63	* Test case to check that all bpf_link_type variants are covered by
	64	* libbpf_bpf_link_type_str.
	65	*/
	66	static void test_libbpf_bpf_link_type_str(void)
	67	{
	68	struct btf *btf;
	69	const struct btf_type *t;
	70	const struct btf_enum *e;
	71	int i, n, id;
	72
	73	btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
	74	if (!ASSERT_OK_PTR(btf, "btf_parse"))
	75	return;
	76
	77	/* find enum bpf_link_type and enumerate each value */
	78	id = btf__find_by_name_kind(btf, "bpf_link_type", BTF_KIND_ENUM);
	79	if (!ASSERT_GT(id, 0, "bpf_link_type_id"))
	80	goto cleanup;
	81	t = btf__type_by_id(btf, id);
	82	e = btf_enum(t);
	83	n = btf_vlen(t);
	84	for (i = 0; i < n; e++, i++) {
	85	enum bpf_link_type link_type = (enum bpf_link_type)e->val;
	86	const char *link_type_name;
	87	const char *link_type_str;
	88	char buf[256];
	89
	90	if (link_type == MAX_BPF_LINK_TYPE)
	91	continue;
	92
	93	link_type_name = btf__str_by_offset(btf, e->name_off);
	94	link_type_str = libbpf_bpf_link_type_str(link_type);
	95	ASSERT_OK_PTR(link_type_str, link_type_name);
	96
	97	snprintf(buf, sizeof(buf), "BPF_LINK_TYPE_%s", link_type_str);
	98	uppercase(buf);
	99
	100	ASSERT_STREQ(buf, link_type_name, "exp_str_value");
	101	}
	102
	103	cleanup:
	104	btf__free(btf);
	105	}
	106
	107	/*
	108	* Test case to check that all bpf_map_type variants are covered by
	109	* libbpf_bpf_map_type_str.
	110	*/
	111	static void test_libbpf_bpf_map_type_str(void)
	112	{
	113	struct btf *btf;
	114	const struct btf_type *t;
	115	const struct btf_enum *e;
	116	int i, n, id;
	117
	118	btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
	119	if (!ASSERT_OK_PTR(btf, "btf_parse"))
	120	return;
	121
	122	/* find enum bpf_map_type and enumerate each value */
	123	id = btf__find_by_name_kind(btf, "bpf_map_type", BTF_KIND_ENUM);
	124	if (!ASSERT_GT(id, 0, "bpf_map_type_id"))
	125	goto cleanup;
	126	t = btf__type_by_id(btf, id);
	127	e = btf_enum(t);
	128	n = btf_vlen(t);
	129	for (i = 0; i < n; e++, i++) {
	130	enum bpf_map_type map_type = (enum bpf_map_type)e->val;
	131	const char *map_type_name;
	132	const char *map_type_str;
	133	char buf[256];
	134
	135	map_type_name = btf__str_by_offset(btf, e->name_off);
	136	map_type_str = libbpf_bpf_map_type_str(map_type);
	137	ASSERT_OK_PTR(map_type_str, map_type_name);
	138
	139	snprintf(buf, sizeof(buf), "BPF_MAP_TYPE_%s", map_type_str);
	140	uppercase(buf);
	141
	142	ASSERT_STREQ(buf, map_type_name, "exp_str_value");
	143	}
	144
	145	cleanup:
	146	btf__free(btf);
	147	}
	148
	149	/*
	150	* Test case to check that all bpf_prog_type variants are covered by
	151	* libbpf_bpf_prog_type_str.
	152	*/
	153	static void test_libbpf_bpf_prog_type_str(void)
	154	{
	155	struct btf *btf;
	156	const struct btf_type *t;
	157	const struct btf_enum *e;
	158	int i, n, id;
	159
	160	btf = btf__parse("/sys/kernel/btf/vmlinux", NULL);
	161	if (!ASSERT_OK_PTR(btf, "btf_parse"))
	162	return;
	163
	164	/* find enum bpf_prog_type and enumerate each value */
	165	id = btf__find_by_name_kind(btf, "bpf_prog_type", BTF_KIND_ENUM);
	166	if (!ASSERT_GT(id, 0, "bpf_prog_type_id"))
	167	goto cleanup;
	168	t = btf__type_by_id(btf, id);
	169	e = btf_enum(t);
	170	n = btf_vlen(t);
	171	for (i = 0; i < n; e++, i++) {
	172	enum bpf_prog_type prog_type = (enum bpf_prog_type)e->val;
	173	const char *prog_type_name;
	174	const char *prog_type_str;
	175	char buf[256];
	176
	177	prog_type_name = btf__str_by_offset(btf, e->name_off);
	178	prog_type_str = libbpf_bpf_prog_type_str(prog_type);
	179	ASSERT_OK_PTR(prog_type_str, prog_type_name);
	180
	181	snprintf(buf, sizeof(buf), "BPF_PROG_TYPE_%s", prog_type_str);
	182	uppercase(buf);
	183
	184	ASSERT_STREQ(buf, prog_type_name, "exp_str_value");
	185	}
	186
	187	cleanup:
	188	btf__free(btf);
	189	}
	190
	191	/*
	192	* Run all libbpf str conversion tests.
	193	*/
	194	void test_libbpf_str(void)
	195	{
	196	if (test__start_subtest("bpf_attach_type_str"))
	197	test_libbpf_bpf_attach_type_str();
	198
	199	if (test__start_subtest("bpf_link_type_str"))
	200	test_libbpf_bpf_link_type_str();
	201
	202	if (test__start_subtest("bpf_map_type_str"))
	203	test_libbpf_bpf_map_type_str();
	204
	205	if (test__start_subtest("bpf_prog_type_str"))
	206	test_libbpf_bpf_prog_type_str();
	207	}

tools/testing/selftests/bpf/prog_tests/tc_redirect.c+4−4

tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c+183−0

@@ -0,0 +1,183 @@
	1	// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
	2	/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
	3
	4	#define _GNU_SOURCE
	5	#include <test_progs.h>
	6	#include <network_helpers.h>
	7	#include <ctype.h>
	8
	9	#define CMD_OUT_BUF_SIZE 1023
	10
	11	#define SYS(cmd) ({ \
	12	if (!ASSERT_OK(system(cmd), (cmd))) \
	13	goto out; \
	14	})
	15
	16	#define SYS_OUT(cmd, ...) ({ \
	17	char buf[1024]; \
	18	snprintf(buf, sizeof(buf), (cmd), ##__VA_ARGS__); \
	19	FILE *f = popen(buf, "r"); \
	20	if (!ASSERT_OK_PTR(f, buf)) \
	21	goto out; \
	22	f; \
	23	})
	24
	25	/* out must be at least `size * 4 + 1` bytes long */
	26	static void escape_str(char out, const char in, size_t size)
	27	{
	28	static const char *hex = "0123456789ABCDEF";
	29	size_t i;
	30
	31	for (i = 0; i < size; i++) {
	32	if (isprint(in[i]) && in[i] != '\\' && in[i] != '\'') {
	33	*out++ = in[i];
	34	} else {
	35	*out++ = '\\';
	36	*out++ = 'x';
	37	*out++ = hex[(in[i] >> 4) & 0xf];
	38	*out++ = hex[in[i] & 0xf];
	39	}
	40	}
	41	*out++ = '\0';
	42	}
	43
	44	static bool expect_str(char buf, size_t size, const char str, const char *name)
	45	{
	46	static char escbuf_expected[CMD_OUT_BUF_SIZE * 4];
	47	static char escbuf_actual[CMD_OUT_BUF_SIZE * 4];
	48	static int duration = 0;
	49	bool ok;
	50
	51	ok = size == strlen(str) && !memcmp(buf, str, size);
	52
	53	if (!ok) {
	54	escape_str(escbuf_expected, str, strlen(str));
	55	escape_str(escbuf_actual, buf, size);
	56	}
	57	CHECK(!ok, name, "unexpected %s: actual '%s' != expected '%s'\n",
	58	name, escbuf_actual, escbuf_expected);
	59
	60	return ok;
	61	}
	62
	63	static void test_synproxy(bool xdp)
	64	{
	65	int server_fd = -1, client_fd = -1, accept_fd = -1;
	66	char prog_id, prog_id_end;
	67	struct nstoken *ns = NULL;
	68	FILE *ctrl_file = NULL;
	69	char buf[CMD_OUT_BUF_SIZE];
	70	size_t size;
	71
	72	SYS("ip netns add synproxy");
	73
	74	SYS("ip link add tmp0 type veth peer name tmp1");
	75	SYS("ip link set tmp1 netns synproxy");
	76	SYS("ip link set tmp0 up");
	77	SYS("ip addr replace 198.18.0.1/24 dev tmp0");
	78
	79	/* When checksum offload is enabled, the XDP program sees wrong
	80	* checksums and drops packets.
	81	*/
	82	SYS("ethtool -K tmp0 tx off");
	83	if (xdp)
	84	/* Workaround required for veth. */
	85	SYS("ip link set tmp0 xdp object xdp_dummy.o section xdp 2> /dev/null");
	86
	87	ns = open_netns("synproxy");
	88	if (!ASSERT_OK_PTR(ns, "setns"))
	89	goto out;
	90
	91	SYS("ip link set lo up");
	92	SYS("ip link set tmp1 up");
	93	SYS("ip addr replace 198.18.0.2/24 dev tmp1");
	94	SYS("sysctl -w net.ipv4.tcp_syncookies=2");
	95	SYS("sysctl -w net.ipv4.tcp_timestamps=1");
	96	SYS("sysctl -w net.netfilter.nf_conntrack_tcp_loose=0");
	97	SYS("iptables -t raw -I PREROUTING \
	98	-i tmp1 -p tcp -m tcp --syn --dport 8080 -j CT --notrack");
	99	SYS("iptables -t filter -A INPUT \
	100	-i tmp1 -p tcp -m tcp --dport 8080 -m state --state INVALID,UNTRACKED \
	101	-j SYNPROXY --sack-perm --timestamp --wscale 7 --mss 1460");
	102	SYS("iptables -t filter -A INPUT \
	103	-i tmp1 -m state --state INVALID -j DROP");
	104
	105	ctrl_file = SYS_OUT("./xdp_synproxy --iface tmp1 --ports 8080 \
	106	--single --mss4 1460 --mss6 1440 \
	107	--wscale 7 --ttl 64%s", xdp ? "" : " --tc");
	108	size = fread(buf, 1, sizeof(buf), ctrl_file);
	109	pclose(ctrl_file);
	110	if (!expect_str(buf, size, "Total SYNACKs generated: 0\n",
	111	"initial SYNACKs"))
	112	goto out;
	113
	114	if (!xdp) {
	115	ctrl_file = SYS_OUT("tc filter show dev tmp1 ingress");
	116	size = fread(buf, 1, sizeof(buf), ctrl_file);
	117	pclose(ctrl_file);
	118	prog_id = memmem(buf, size, " id ", 4);
	119	if (!ASSERT_OK_PTR(prog_id, "find prog id"))
	120	goto out;
	121	prog_id += 4;
	122	if (!ASSERT_LT(prog_id, buf + size, "find prog id begin"))
	123	goto out;
	124	prog_id_end = prog_id;
	125	while (prog_id_end < buf + size && *prog_id_end >= '0' &&
	126	*prog_id_end <= '9')
	127	prog_id_end++;
	128	if (!ASSERT_LT(prog_id_end, buf + size, "find prog id end"))
	129	goto out;
	130	*prog_id_end = '\0';
	131	}
	132
	133	server_fd = start_server(AF_INET, SOCK_STREAM, "198.18.0.2", 8080, 0);
	134	if (!ASSERT_GE(server_fd, 0, "start_server"))
	135	goto out;
	136
	137	close_netns(ns);
	138	ns = NULL;
	139
	140	client_fd = connect_to_fd(server_fd, 10000);
	141	if (!ASSERT_GE(client_fd, 0, "connect_to_fd"))
	142	goto out;
	143
	144	accept_fd = accept(server_fd, NULL, NULL);
	145	if (!ASSERT_GE(accept_fd, 0, "accept"))
	146	goto out;
	147
	148	ns = open_netns("synproxy");
	149	if (!ASSERT_OK_PTR(ns, "setns"))
	150	goto out;
	151
	152	if (xdp)
	153	ctrl_file = SYS_OUT("./xdp_synproxy --iface tmp1 --single");
	154	else
	155	ctrl_file = SYS_OUT("./xdp_synproxy --prog %s --single",
	156	prog_id);
	157	size = fread(buf, 1, sizeof(buf), ctrl_file);
	158	pclose(ctrl_file);
	159	if (!expect_str(buf, size, "Total SYNACKs generated: 1\n",
	160	"SYNACKs after connection"))
	161	goto out;
	162
	163	out:
	164	if (accept_fd >= 0)
	165	close(accept_fd);
	166	if (client_fd >= 0)
	167	close(client_fd);
	168	if (server_fd >= 0)
	169	close(server_fd);
	170	if (ns)
	171	close_netns(ns);
	172
	173	system("ip link del tmp0");
	174	system("ip netns del synproxy");
	175	}
	176
	177	void test_xdp_synproxy(void)
	178	{
	179	if (test__start_subtest("xdp"))
	180	test_synproxy(true);
	181	if (test__start_subtest("tc"))
	182	test_synproxy(false);
	183	}

tools/testing/selftests/bpf/progs/bpf_hashmap_full_update_bench.c+40−0

@@ -0,0 +1,40 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2022 Bytedance */
	3
	4	#include "vmlinux.h"
	5	#include <bpf/bpf_helpers.h>
	6	#include "bpf_misc.h"
	7
	8	char _license[] SEC("license") = "GPL";
	9
	10	#define MAX_ENTRIES 1000
	11
	12	struct {
	13	__uint(type, BPF_MAP_TYPE_HASH);
	14	__type(key, u32);
	15	__type(value, u64);
	16	__uint(max_entries, MAX_ENTRIES);
	17	} hash_map_bench SEC(".maps");
	18
	19	u64 __attribute__((__aligned__(256))) percpu_time[256];
	20	u64 nr_loops;
	21
	22	static int loop_update_callback(__u32 index, u32 *key)
	23	{
	24	u64 init_val = 1;
	25
	26	bpf_map_update_elem(&hash_map_bench, key, &init_val, BPF_ANY);
	27	return 0;
	28	}
	29
	30	SEC("fentry/" SYS_PREFIX "sys_getpgid")
	31	int benchmark(void *ctx)
	32	{
	33	u32 cpu = bpf_get_smp_processor_id();
	34	u32 key = cpu + MAX_ENTRIES;
	35	u64 start_time = bpf_ktime_get_ns();
	36
	37	bpf_loop(nr_loops, loop_update_callback, &key, 0);
	38	percpu_time[cpu & 255] = bpf_ktime_get_ns() - start_time;
	39	return 0;
	40	}

tools/testing/selftests/bpf/progs/btf__core_reloc_enum64val.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_enum64val x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_enum64val___diff.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_enum64val___diff x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_enum64val___err_missing.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_enum64val___err_missing x) {}

tools/testing/selftests/bpf/progs/btf__core_reloc_enum64val___val3_missing.c+3−0

@@ -0,0 +1,3 @@
	1	#include "core_reloc_types.h"
	2
	3	void f(struct core_reloc_enum64val___val3_missing x) {}

tools/testing/selftests/bpf/progs/core_reloc_types.h+78−0

@@ -1117,6 +1117,20 @@ struct core_reloc_enumval_output { · 1116 unmodified lines
1117	int anon_val2;	1117	int anon_val2;
1118	};	1118	};
1119		1119
		1120	struct core_reloc_enum64val_output {
		1121	bool unsigned_val1_exists;
		1122	bool unsigned_val2_exists;
		1123	bool unsigned_val3_exists;
		1124	bool signed_val1_exists;
		1125	bool signed_val2_exists;
		1126	bool signed_val3_exists;
		1127
		1128	long unsigned_val1;
		1129	long unsigned_val2;
		1130	long signed_val1;
		1131	long signed_val2;
		1132	};
		1133
1120	enum named_enum {	1134	enum named_enum {
1121	NAMED_ENUM_VAL1 = 1,	1135	NAMED_ENUM_VAL1 = 1,
1122	NAMED_ENUM_VAL2 = 2,	1136	NAMED_ENUM_VAL2 = 2,
@@ -1134,6 +1148,23 @@ struct core_reloc_enumval { · 11 unmodified lines
1134	anon_enum f2;	1148	anon_enum f2;
1135	};	1149	};
1136		1150
		1151	enum named_unsigned_enum64 {
		1152	UNSIGNED_ENUM64_VAL1 = 0x1ffffffffULL,
		1153	UNSIGNED_ENUM64_VAL2 = 0x2,
		1154	UNSIGNED_ENUM64_VAL3 = 0x3ffffffffULL,
		1155	};
		1156
		1157	enum named_signed_enum64 {
		1158	SIGNED_ENUM64_VAL1 = 0x1ffffffffLL,
		1159	SIGNED_ENUM64_VAL2 = -2,
		1160	SIGNED_ENUM64_VAL3 = 0x3ffffffffLL,
		1161	};
		1162
		1163	struct core_reloc_enum64val {
		1164	enum named_unsigned_enum64 f1;
		1165	enum named_signed_enum64 f2;
		1166	};
		1167
1137	/* differing enumerator values */	1168	/* differing enumerator values */
1138	enum named_enum___diff {	1169	enum named_enum___diff {
1139	NAMED_ENUM_VAL1___diff = 101,	1170	NAMED_ENUM_VAL1___diff = 101,
@@ -1152,6 +1183,23 @@ struct core_reloc_enumval___diff { · 12 unmodified lines
1152	anon_enum___diff f2;	1183	anon_enum___diff f2;
1153	};	1184	};
1154		1185
		1186	enum named_unsigned_enum64___diff {
		1187	UNSIGNED_ENUM64_VAL1___diff = 0x101ffffffffULL,
		1188	UNSIGNED_ENUM64_VAL2___diff = 0x202ffffffffULL,
		1189	UNSIGNED_ENUM64_VAL3___diff = 0x303ffffffffULL,
		1190	};
		1191
		1192	enum named_signed_enum64___diff {
		1193	SIGNED_ENUM64_VAL1___diff = -101,
		1194	SIGNED_ENUM64_VAL2___diff = -202,
		1195	SIGNED_ENUM64_VAL3___diff = -303,
		1196	};
		1197
		1198	struct core_reloc_enum64val___diff {
		1199	enum named_unsigned_enum64___diff f1;
		1200	enum named_signed_enum64___diff f2;
		1201	};
		1202
1155	/* missing (optional) third enum value */	1203	/* missing (optional) third enum value */
1156	enum named_enum___val3_missing {	1204	enum named_enum___val3_missing {
1157	NAMED_ENUM_VAL1___val3_missing = 111,	1205	NAMED_ENUM_VAL1___val3_missing = 111,
@@ -1168,6 +1216,21 @@ struct core_reloc_enumval___val3_missing { · 10 unmodified lines
1168	anon_enum___val3_missing f2;	1216	anon_enum___val3_missing f2;
1169	};	1217	};
1170		1218
		1219	enum named_unsigned_enum64___val3_missing {
		1220	UNSIGNED_ENUM64_VAL1___val3_missing = 0x111ffffffffULL,
		1221	UNSIGNED_ENUM64_VAL2___val3_missing = 0x222,
		1222	};
		1223
		1224	enum named_signed_enum64___val3_missing {
		1225	SIGNED_ENUM64_VAL1___val3_missing = 0x111ffffffffLL,
		1226	SIGNED_ENUM64_VAL2___val3_missing = -222,
		1227	};
		1228
		1229	struct core_reloc_enum64val___val3_missing {
		1230	enum named_unsigned_enum64___val3_missing f1;
		1231	enum named_signed_enum64___val3_missing f2;
		1232	};
		1233
1171	/* missing (mandatory) second enum value, should fail */	1234	/* missing (mandatory) second enum value, should fail */
1172	enum named_enum___err_missing {	1235	enum named_enum___err_missing {
1173	NAMED_ENUM_VAL1___err_missing = 1,	1236	NAMED_ENUM_VAL1___err_missing = 1,
@@ -1183,3 +1246,18 @@ struct core_reloc_enumval___err_missing { · 9 unmodified lines
1183	enum named_enum___err_missing f1;	1246	enum named_enum___err_missing f1;
1184	anon_enum___err_missing f2;	1247	anon_enum___err_missing f2;
1185	};	1248	};
		1249
		1250	enum named_unsigned_enum64___err_missing {
		1251	UNSIGNED_ENUM64_VAL1___err_missing = 0x1ffffffffULL,
		1252	UNSIGNED_ENUM64_VAL3___err_missing = 0x3ffffffffULL,
		1253	};
		1254
		1255	enum named_signed_enum64___err_missing {
		1256	SIGNED_ENUM64_VAL1___err_missing = 0x1ffffffffLL,
		1257	SIGNED_ENUM64_VAL3___err_missing = -3,
		1258	};
		1259
		1260	struct core_reloc_enum64val___err_missing {
		1261	enum named_unsigned_enum64___err_missing f1;
		1262	enum named_signed_enum64___err_missing f2;
		1263	};

tools/testing/selftests/bpf/progs/test_attach_probe.c+60−0

@@ -5,6 +5,7 @@ · 4 unmodified lines
5	#include <linux/bpf.h>	5	#include <linux/bpf.h>
6	#include <bpf/bpf_helpers.h>	6	#include <bpf/bpf_helpers.h>
7	#include <bpf/bpf_tracing.h>	7	#include <bpf/bpf_tracing.h>
		8	#include <stdbool.h>
8	#include "bpf_misc.h"	9	#include "bpf_misc.h"
9		10
10	int kprobe_res = 0;	11	int kprobe_res = 0;
@@ -17,6 +18,11 @@ int uprobe_byname_res = 0; · 6 unmodified lines
17	int uretprobe_byname_res = 0;	18	int uretprobe_byname_res = 0;
18	int uprobe_byname2_res = 0;	19	int uprobe_byname2_res = 0;
19	int uretprobe_byname2_res = 0;	20	int uretprobe_byname2_res = 0;
		21	int uprobe_byname3_sleepable_res = 0;
		22	int uprobe_byname3_res = 0;
		23	int uretprobe_byname3_sleepable_res = 0;
		24	int uretprobe_byname3_res = 0;
		25	void *user_ptr = 0;
20		26
21	SEC("kprobe")	27	SEC("kprobe")
22	int handle_kprobe(struct pt_regs *ctx)	28	int handle_kprobe(struct pt_regs *ctx)
@@ -32,6 +38,17 @@ int BPF_KPROBE(handle_kprobe_auto) · 9 unmodified lines
32	return 0;	38	return 0;
33	}	39	}
34		40
		41	/**
		42	* This program will be manually made sleepable on the userspace side
		43	* and should thus be unattachable.
		44	*/
		45	SEC("kprobe/" SYS_PREFIX "sys_nanosleep")
		46	int handle_kprobe_sleepable(struct pt_regs *ctx)
		47	{
		48	kprobe_res = 2;
		49	return 0;
		50	}
		51
35	SEC("kretprobe")	52	SEC("kretprobe")
36	int handle_kretprobe(struct pt_regs *ctx)	53	int handle_kretprobe(struct pt_regs *ctx)
37	{	54	{
@@ -93,4 +110,47 @@ int handle_uretprobe_byname2(struct pt_regs *ctx) · 55 unmodified lines
93	return 0;	110	return 0;
94	}	111	}
95		112
		113	static __always_inline bool verify_sleepable_user_copy(void)
		114	{
		115	char data[9];
		116
		117	bpf_copy_from_user(data, sizeof(data), user_ptr);
		118	return bpf_strncmp(data, sizeof(data), "test_data") == 0;
		119	}
		120
		121	SEC("uprobe.s//proc/self/exe:trigger_func3")
		122	int handle_uprobe_byname3_sleepable(struct pt_regs *ctx)
		123	{
		124	if (verify_sleepable_user_copy())
		125	uprobe_byname3_sleepable_res = 9;
		126	return 0;
		127	}
		128
		129	/**
		130	* same target as the uprobe.s above to force sleepable and non-sleepable
		131	* programs in the same bpf_prog_array
		132	*/
		133	SEC("uprobe//proc/self/exe:trigger_func3")
		134	int handle_uprobe_byname3(struct pt_regs *ctx)
		135	{
		136	uprobe_byname3_res = 10;
		137	return 0;
		138	}
		139
		140	SEC("uretprobe.s//proc/self/exe:trigger_func3")
		141	int handle_uretprobe_byname3_sleepable(struct pt_regs *ctx)
		142	{
		143	if (verify_sleepable_user_copy())
		144	uretprobe_byname3_sleepable_res = 11;
		145	return 0;
		146	}
		147
		148	SEC("uretprobe//proc/self/exe:trigger_func3")
		149	int handle_uretprobe_byname3(struct pt_regs *ctx)
		150	{
		151	uretprobe_byname3_res = 12;
		152	return 0;
		153	}
		154
		155
96	char _license[] SEC("license") = "GPL";	156	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/progs/test_core_reloc_enum64val.c+70−0

@@ -0,0 +1,70 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
	3
	4	#include <linux/bpf.h>
	5	#include <stdint.h>
	6	#include <stdbool.h>
	7	#include <bpf/bpf_helpers.h>
	8	#include <bpf/bpf_core_read.h>
	9
	10	char _license[] SEC("license") = "GPL";
	11
	12	struct {
	13	char in[256];
	14	char out[256];
	15	bool skip;
	16	} data = {};
	17
	18	enum named_unsigned_enum64 {
	19	UNSIGNED_ENUM64_VAL1 = 0x1ffffffffULL,
	20	UNSIGNED_ENUM64_VAL2 = 0x2ffffffffULL,
	21	UNSIGNED_ENUM64_VAL3 = 0x3ffffffffULL,
	22	};
	23
	24	enum named_signed_enum64 {
	25	SIGNED_ENUM64_VAL1 = 0x1ffffffffLL,
	26	SIGNED_ENUM64_VAL2 = -2,
	27	SIGNED_ENUM64_VAL3 = 0x3ffffffffLL,
	28	};
	29
	30	struct core_reloc_enum64val_output {
	31	bool unsigned_val1_exists;
	32	bool unsigned_val2_exists;
	33	bool unsigned_val3_exists;
	34	bool signed_val1_exists;
	35	bool signed_val2_exists;
	36	bool signed_val3_exists;
	37
	38	long unsigned_val1;
	39	long unsigned_val2;
	40	long signed_val1;
	41	long signed_val2;
	42	};
	43
	44	SEC("raw_tracepoint/sys_enter")
	45	int test_core_enum64val(void *ctx)
	46	{
	47	#if __clang_major__ >= 15
	48	struct core_reloc_enum64val_output out = (void )&data.out;
	49	enum named_unsigned_enum64 named_unsigned = 0;
	50	enum named_signed_enum64 named_signed = 0;
	51
	52	out->unsigned_val1_exists = bpf_core_enum_value_exists(named_unsigned, UNSIGNED_ENUM64_VAL1);
	53	out->unsigned_val2_exists = bpf_core_enum_value_exists(enum named_unsigned_enum64, UNSIGNED_ENUM64_VAL2);
	54	out->unsigned_val3_exists = bpf_core_enum_value_exists(enum named_unsigned_enum64, UNSIGNED_ENUM64_VAL3);
	55	out->signed_val1_exists = bpf_core_enum_value_exists(named_signed, SIGNED_ENUM64_VAL1);
	56	out->signed_val2_exists = bpf_core_enum_value_exists(enum named_signed_enum64, SIGNED_ENUM64_VAL2);
	57	out->signed_val3_exists = bpf_core_enum_value_exists(enum named_signed_enum64, SIGNED_ENUM64_VAL3);
	58
	59	out->unsigned_val1 = bpf_core_enum_value(named_unsigned, UNSIGNED_ENUM64_VAL1);
	60	out->unsigned_val2 = bpf_core_enum_value(named_unsigned, UNSIGNED_ENUM64_VAL2);
	61	out->signed_val1 = bpf_core_enum_value(named_signed, SIGNED_ENUM64_VAL1);
	62	out->signed_val2 = bpf_core_enum_value(named_signed, SIGNED_ENUM64_VAL2);
	63	/* NAMED_ENUM64_VAL3 value is optional */
	64
	65	#else
	66	data.skip = true;
	67	#endif
	68
	69	return 0;
	70	}

tools/testing/selftests/bpf/progs/test_tc_dtime.c+51−2

@@ -11,6 +11,8 @@ · 10 unmodified lines
11	#include <linux/in.h>	11	#include <linux/in.h>
12	#include <linux/ip.h>	12	#include <linux/ip.h>
13	#include <linux/ipv6.h>	13	#include <linux/ipv6.h>
		14	#include <linux/tcp.h>
		15	#include <linux/udp.h>
14	#include <bpf/bpf_helpers.h>	16	#include <bpf/bpf_helpers.h>
15	#include <bpf/bpf_endian.h>	17	#include <bpf/bpf_endian.h>
16	#include <sys/socket.h>	18	#include <sys/socket.h>
@@ -115,6 +117,19 @@ static bool bpf_fwd(void) · 98 unmodified lines
115	return test < TCP_IP4_RT_FWD;	117	return test < TCP_IP4_RT_FWD;
116	}	118	}
117		119
		120	static __u8 get_proto(void)
		121	{
		122	switch (test) {
		123	case UDP_IP4:
		124	case UDP_IP6:
		125	case UDP_IP4_RT_FWD:
		126	case UDP_IP6_RT_FWD:
		127	return IPPROTO_UDP;
		128	default:
		129	return IPPROTO_TCP;
		130	}
		131	}
		132
118	/* -1: parse error: TC_ACT_SHOT	133	/* -1: parse error: TC_ACT_SHOT
119	* 0: not testing traffic: TC_ACT_OK	134	* 0: not testing traffic: TC_ACT_OK
120	* >0: first byte is the inet_proto, second byte has the netns	135	* >0: first byte is the inet_proto, second byte has the netns
@@ -122,11 +137,16 @@ static bool bpf_fwd(void) · 1 unmodified line
122	*/	137	*/
123	static int skb_get_type(struct __sk_buff *skb)	138	static int skb_get_type(struct __sk_buff *skb)
124	{	139	{
		140	__u16 dst_ns_port = __bpf_htons(50000 + test);
125	void *data_end = ctx_ptr(skb->data_end);	141	void *data_end = ctx_ptr(skb->data_end);
126	void *data = ctx_ptr(skb->data);	142	void *data = ctx_ptr(skb->data);
127	__u8 inet_proto = 0, ns = 0;	143	__u8 inet_proto = 0, ns = 0;
128	struct ipv6hdr *ip6h;	144	struct ipv6hdr *ip6h;
		145	__u16 sport, dport;
129	struct iphdr *iph;	146	struct iphdr *iph;
		147	struct tcphdr *th;
		148	struct udphdr *uh;
		149	void *trans;
130		150
131	switch (skb->protocol) {	151	switch (skb->protocol) {
132	case __bpf_htons(ETH_P_IP):	152	case __bpf_htons(ETH_P_IP):
@@ -138,6 +158,7 @@ static int skb_get_type(struct __sk_buff *skb) · 5 unmodified lines
138	else if (iph->saddr == ip4_dst)	158	else if (iph->saddr == ip4_dst)
139	ns = DST_NS;	159	ns = DST_NS;
140	inet_proto = iph->protocol;	160	inet_proto = iph->protocol;
		161	trans = iph + 1;
141	break;	162	break;
142	case __bpf_htons(ETH_P_IPV6):	163	case __bpf_htons(ETH_P_IPV6):
143	ip6h = data + sizeof(struct ethhdr);	164	ip6h = data + sizeof(struct ethhdr);
@@ -148,15 +169,43 @@ static int skb_get_type(struct __sk_buff *skb) · 4 unmodified lines
148	else if (v6_equal(ip6h->saddr, (struct in6_addr)ip6_dst))	169	else if (v6_equal(ip6h->saddr, (struct in6_addr)ip6_dst))
149	ns = DST_NS;	170	ns = DST_NS;
150	inet_proto = ip6h->nexthdr;	171	inet_proto = ip6h->nexthdr;
		172	trans = ip6h + 1;
151	break;	173	break;
152	default:	174	default:
153	return 0;	175	return 0;
154	}	176	}
155		177
156	if ((inet_proto != IPPROTO_TCP && inet_proto != IPPROTO_UDP) \|\| !ns)	178	/* skb is not from src_ns or dst_ns.
		179	* skb is not the testing IPPROTO.
		180	*/
		181	if (!ns \|\| inet_proto != get_proto())
157	return 0;	182	return 0;
158		183
159	return (ns << 8 \| inet_proto);	184	switch (inet_proto) {
		185	case IPPROTO_TCP:
		186	th = trans;
		187	if (th + 1 > data_end)
		188	return -1;
		189	sport = th->source;
		190	dport = th->dest;
		191	break;
		192	case IPPROTO_UDP:
		193	uh = trans;
		194	if (uh + 1 > data_end)
		195	return -1;
		196	sport = uh->source;
		197	dport = uh->dest;
		198	break;
		199	default:
		200	return 0;
		201	}
		202
		203	/* The skb is the testing traffic */
		204	if ((ns == SRC_NS && dport == dst_ns_port) \|\|
		205	(ns == DST_NS && sport == dst_ns_port))
		206	return (ns << 8 \| inet_proto);
		207
		208	return 0;
160	}	209	}
161		210
162	/* format: direction@iface@netns	211	/* format: direction@iface@netns

tools/testing/selftests/bpf/progs/test_varlen.c+4−4

@@ -41,20 +41,20 @@ int handler64_unsigned(void *regs) · 40 unmodified lines
41	{	41	{
42	int pid = bpf_get_current_pid_tgid() >> 32;	42	int pid = bpf_get_current_pid_tgid() >> 32;
43	void *payload = payload1;	43	void *payload = payload1;
44	u64 len;	44	long len;
45		45
46	/* ignore irrelevant invocations */	46	/* ignore irrelevant invocations */
47	if (test_pid != pid \|\| !capture)	47	if (test_pid != pid \|\| !capture)
48	return 0;	48	return 0;
49		49
50	len = bpf_probe_read_kernel_str(payload, MAX_LEN, &buf_in1[0]);	50	len = bpf_probe_read_kernel_str(payload, MAX_LEN, &buf_in1[0]);
51	if (len <= MAX_LEN) {	51	if (len >= 0) {
52	payload += len;	52	payload += len;
53	payload1_len1 = len;	53	payload1_len1 = len;
54	}	54	}
55		55
56	len = bpf_probe_read_kernel_str(payload, MAX_LEN, &buf_in2[0]);	56	len = bpf_probe_read_kernel_str(payload, MAX_LEN, &buf_in2[0]);
57	if (len <= MAX_LEN) {	57	if (len >= 0) {
58	payload += len;	58	payload += len;
59	payload1_len2 = len;	59	payload1_len2 = len;
60	}	60	}
@@ -123,7 +123,7 @@ int handler32_signed(void *regs) · 62 unmodified lines
123	{	123	{
124	int pid = bpf_get_current_pid_tgid() >> 32;	124	int pid = bpf_get_current_pid_tgid() >> 32;
125	void *payload = payload4;	125	void *payload = payload4;
126	int len;	126	long len;
127		127
128	/* ignore irrelevant invocations */	128	/* ignore irrelevant invocations */
129	if (test_pid != pid \|\| !capture)	129	if (test_pid != pid \|\| !capture)

tools/testing/selftests/bpf/progs/xdp_synproxy_kern.c+833−0

@@ -0,0 +1,833 @@
	1	// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
	2	/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
	3
	4	#include "vmlinux.h"
	5
	6	#include <bpf/bpf_helpers.h>
	7	#include <bpf/bpf_endian.h>
	8	#include <asm/errno.h>
	9
	10	#define TC_ACT_OK 0
	11	#define TC_ACT_SHOT 2
	12
	13	#define NSEC_PER_SEC 1000000000L
	14
	15	#define ETH_ALEN 6
	16	#define ETH_P_IP 0x0800
	17	#define ETH_P_IPV6 0x86DD
	18
	19	#define tcp_flag_word(tp) (((union tcp_word_hdr *)(tp))->words[3])
	20
	21	#define IP_DF 0x4000
	22	#define IP_MF 0x2000
	23	#define IP_OFFSET 0x1fff
	24
	25	#define NEXTHDR_TCP 6
	26
	27	#define TCPOPT_NOP 1
	28	#define TCPOPT_EOL 0
	29	#define TCPOPT_MSS 2
	30	#define TCPOPT_WINDOW 3
	31	#define TCPOPT_SACK_PERM 4
	32	#define TCPOPT_TIMESTAMP 8
	33
	34	#define TCPOLEN_MSS 4
	35	#define TCPOLEN_WINDOW 3
	36	#define TCPOLEN_SACK_PERM 2
	37	#define TCPOLEN_TIMESTAMP 10
	38
	39	#define TCP_TS_HZ 1000
	40	#define TS_OPT_WSCALE_MASK 0xf
	41	#define TS_OPT_SACK (1 << 4)
	42	#define TS_OPT_ECN (1 << 5)
	43	#define TSBITS 6
	44	#define TSMASK (((__u32)1 << TSBITS) - 1)
	45	#define TCP_MAX_WSCALE 14U
	46
	47	#define IPV4_MAXLEN 60
	48	#define TCP_MAXLEN 60
	49
	50	#define DEFAULT_MSS4 1460
	51	#define DEFAULT_MSS6 1440
	52	#define DEFAULT_WSCALE 7
	53	#define DEFAULT_TTL 64
	54	#define MAX_ALLOWED_PORTS 8
	55
	56	#define swap(a, b) \
	57	do { typeof(a) __tmp = (a); (a) = (b); (b) = __tmp; } while (0)
	58
	59	#define __get_unaligned_t(type, ptr) ({ \
	60	const struct { type x; } __attribute__((__packed__)) *__pptr = (typeof(__pptr))(ptr); \
	61	__pptr->x; \
	62	})
	63
	64	#define get_unaligned(ptr) __get_unaligned_t(typeof(*(ptr)), (ptr))
	65
	66	struct {
	67	__uint(type, BPF_MAP_TYPE_ARRAY);
	68	__type(key, __u32);
	69	__type(value, __u64);
	70	__uint(max_entries, 2);
	71	} values SEC(".maps");
	72
	73	struct {
	74	__uint(type, BPF_MAP_TYPE_ARRAY);
	75	__type(key, __u32);
	76	__type(value, __u16);
	77	__uint(max_entries, MAX_ALLOWED_PORTS);
	78	} allowed_ports SEC(".maps");
	79
	80	extern struct nf_conn bpf_xdp_ct_lookup(struct xdp_md xdp_ctx,
	81	struct bpf_sock_tuple *bpf_tuple,
	82	__u32 len_tuple,
	83	struct bpf_ct_opts *opts,
	84	__u32 len_opts) __ksym;
	85
	86	extern struct nf_conn bpf_skb_ct_lookup(struct __sk_buff skb_ctx,
	87	struct bpf_sock_tuple *bpf_tuple,
	88	u32 len_tuple,
	89	struct bpf_ct_opts *opts,
	90	u32 len_opts) __ksym;
	91
	92	extern void bpf_ct_release(struct nf_conn *ct) __ksym;
	93
	94	static __always_inline void swap_eth_addr(__u8 a, __u8 b)
	95	{
	96	__u8 tmp[ETH_ALEN];
	97
	98	__builtin_memcpy(tmp, a, ETH_ALEN);
	99	__builtin_memcpy(a, b, ETH_ALEN);
	100	__builtin_memcpy(b, tmp, ETH_ALEN);
	101	}
	102
	103	static __always_inline __u16 csum_fold(__u32 csum)
	104	{
	105	csum = (csum & 0xffff) + (csum >> 16);
	106	csum = (csum & 0xffff) + (csum >> 16);
	107	return (__u16)~csum;
	108	}
	109
	110	static __always_inline __u16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
	111	__u32 len, __u8 proto,
	112	__u32 csum)
	113	{
	114	__u64 s = csum;
	115
	116	s += (__u32)saddr;
	117	s += (__u32)daddr;
	118	#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
	119	s += proto + len;
	120	#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
	121	s += (proto + len) << 8;
	122	#else
	123	#error Unknown endian
	124	#endif
	125	s = (s & 0xffffffff) + (s >> 32);
	126	s = (s & 0xffffffff) + (s >> 32);
	127
	128	return csum_fold((__u32)s);
	129	}
	130
	131	static __always_inline __u16 csum_ipv6_magic(const struct in6_addr *saddr,
	132	const struct in6_addr *daddr,
	133	__u32 len, __u8 proto, __u32 csum)
	134	{
	135	__u64 sum = csum;
	136	int i;
	137
	138	#pragma unroll
	139	for (i = 0; i < 4; i++)
	140	sum += (__u32)saddr->in6_u.u6_addr32[i];
	141
	142	#pragma unroll
	143	for (i = 0; i < 4; i++)
	144	sum += (__u32)daddr->in6_u.u6_addr32[i];
	145
	146	/* Don't combine additions to avoid 32-bit overflow. */
	147	sum += bpf_htonl(len);
	148	sum += bpf_htonl(proto);
	149
	150	sum = (sum & 0xffffffff) + (sum >> 32);
	151	sum = (sum & 0xffffffff) + (sum >> 32);
	152
	153	return csum_fold((__u32)sum);
	154	}
	155
	156	static __always_inline __u64 tcp_clock_ns(void)
	157	{
	158	return bpf_ktime_get_ns();
	159	}
	160
	161	static __always_inline __u32 tcp_ns_to_ts(__u64 ns)
	162	{
	163	return ns / (NSEC_PER_SEC / TCP_TS_HZ);
	164	}
	165
	166	static __always_inline __u32 tcp_time_stamp_raw(void)
	167	{
	168	return tcp_ns_to_ts(tcp_clock_ns());
	169	}
	170
	171	struct tcpopt_context {
	172	__u8 *ptr;
	173	__u8 *end;
	174	void *data_end;
	175	__be32 *tsecr;
	176	__u8 wscale;
	177	bool option_timestamp;
	178	bool option_sack;
	179	};
	180
	181	static int tscookie_tcpopt_parse(struct tcpopt_context *ctx)
	182	{
	183	__u8 opcode, opsize;
	184
	185	if (ctx->ptr >= ctx->end)
	186	return 1;
	187	if (ctx->ptr >= ctx->data_end)
	188	return 1;
	189
	190	opcode = ctx->ptr[0];
	191
	192	if (opcode == TCPOPT_EOL)
	193	return 1;
	194	if (opcode == TCPOPT_NOP) {
	195	++ctx->ptr;
	196	return 0;
	197	}
	198
	199	if (ctx->ptr + 1 >= ctx->end)
	200	return 1;
	201	if (ctx->ptr + 1 >= ctx->data_end)
	202	return 1;
	203	opsize = ctx->ptr[1];
	204	if (opsize < 2)
	205	return 1;
	206
	207	if (ctx->ptr + opsize > ctx->end)
	208	return 1;
	209
	210	switch (opcode) {
	211	case TCPOPT_WINDOW:
	212	if (opsize == TCPOLEN_WINDOW && ctx->ptr + TCPOLEN_WINDOW <= ctx->data_end)
	213	ctx->wscale = ctx->ptr[2] < TCP_MAX_WSCALE ? ctx->ptr[2] : TCP_MAX_WSCALE;
	214	break;
	215	case TCPOPT_TIMESTAMP:
	216	if (opsize == TCPOLEN_TIMESTAMP && ctx->ptr + TCPOLEN_TIMESTAMP <= ctx->data_end) {
	217	ctx->option_timestamp = true;
	218	/* Client's tsval becomes our tsecr. */
	219	ctx->tsecr = get_unaligned((__be32 )(ctx->ptr + 2));
	220	}
	221	break;
	222	case TCPOPT_SACK_PERM:
	223	if (opsize == TCPOLEN_SACK_PERM)
	224	ctx->option_sack = true;
	225	break;
	226	}
	227
	228	ctx->ptr += opsize;
	229
	230	return 0;
	231	}
	232
	233	static int tscookie_tcpopt_parse_batch(__u32 index, void *context)
	234	{
	235	int i;
	236
	237	for (i = 0; i < 7; i++)
	238	if (tscookie_tcpopt_parse(context))
	239	return 1;
	240	return 0;
	241	}
	242
	243	static __always_inline bool tscookie_init(struct tcphdr *tcp_header,
	244	__u16 tcp_len, __be32 *tsval,
	245	__be32 tsecr, void data_end)
	246	{
	247	struct tcpopt_context loop_ctx = {
	248	.ptr = (__u8 *)(tcp_header + 1),
	249	.end = (__u8 *)tcp_header + tcp_len,
	250	.data_end = data_end,
	251	.tsecr = tsecr,
	252	.wscale = TS_OPT_WSCALE_MASK,
	253	.option_timestamp = false,
	254	.option_sack = false,
	255	};
	256	u32 cookie;
	257
	258	bpf_loop(6, tscookie_tcpopt_parse_batch, &loop_ctx, 0);
	259
	260	if (!loop_ctx.option_timestamp)
	261	return false;
	262
	263	cookie = tcp_time_stamp_raw() & ~TSMASK;
	264	cookie \|= loop_ctx.wscale & TS_OPT_WSCALE_MASK;
	265	if (loop_ctx.option_sack)
	266	cookie \|= TS_OPT_SACK;
	267	if (tcp_header->ece && tcp_header->cwr)
	268	cookie \|= TS_OPT_ECN;
	269	*tsval = bpf_htonl(cookie);
	270
	271	return true;
	272	}
	273
	274	static __always_inline void values_get_tcpipopts(__u16 mss, __u8 wscale,
	275	__u8 *ttl, bool ipv6)
	276	{
	277	__u32 key = 0;
	278	__u64 *value;
	279
	280	value = bpf_map_lookup_elem(&values, &key);
	281	if (value && *value != 0) {
	282	if (ipv6)
	283	mss = (value >> 32) & 0xffff;
	284	else
	285	mss = value & 0xffff;
	286	wscale = (value >> 16) & 0xf;
	287	ttl = (value >> 24) & 0xff;
	288	return;
	289	}
	290
	291	*mss = ipv6 ? DEFAULT_MSS6 : DEFAULT_MSS4;
	292	*wscale = DEFAULT_WSCALE;
	293	*ttl = DEFAULT_TTL;
	294	}
	295
	296	static __always_inline void values_inc_synacks(void)
	297	{
	298	__u32 key = 1;
	299	__u32 *value;
	300
	301	value = bpf_map_lookup_elem(&values, &key);
	302	if (value)
	303	__sync_fetch_and_add(value, 1);
	304	}
	305
	306	static __always_inline bool check_port_allowed(__u16 port)
	307	{
	308	__u32 i;
	309
	310	for (i = 0; i < MAX_ALLOWED_PORTS; i++) {
	311	__u32 key = i;
	312	__u16 *value;
	313
	314	value = bpf_map_lookup_elem(&allowed_ports, &key);
	315
	316	if (!value)
	317	break;
	318	/* 0 is a terminator value. Check it first to avoid matching on
	319	* a forbidden port == 0 and returning true.
	320	*/
	321	if (*value == 0)
	322	break;
	323
	324	if (*value == port)
	325	return true;
	326	}
	327
	328	return false;
	329	}
	330
	331	struct header_pointers {
	332	struct ethhdr *eth;
	333	struct iphdr *ipv4;
	334	struct ipv6hdr *ipv6;
	335	struct tcphdr *tcp;
	336	__u16 tcp_len;
	337	};
	338
	339	static __always_inline int tcp_dissect(void data, void data_end,
	340	struct header_pointers *hdr)
	341	{
	342	hdr->eth = data;
	343	if (hdr->eth + 1 > data_end)
	344	return XDP_DROP;
	345
	346	switch (bpf_ntohs(hdr->eth->h_proto)) {
	347	case ETH_P_IP:
	348	hdr->ipv6 = NULL;
	349
	350	hdr->ipv4 = (void )hdr->eth + sizeof(hdr->eth);
	351	if (hdr->ipv4 + 1 > data_end)
	352	return XDP_DROP;
	353	if (hdr->ipv4->ihl * 4 < sizeof(*hdr->ipv4))
	354	return XDP_DROP;
	355	if (hdr->ipv4->version != 4)
	356	return XDP_DROP;
	357
	358	if (hdr->ipv4->protocol != IPPROTO_TCP)
	359	return XDP_PASS;
	360
	361	hdr->tcp = (void )hdr->ipv4 + hdr->ipv4->ihl 4;
	362	break;
	363	case ETH_P_IPV6:
	364	hdr->ipv4 = NULL;
	365
	366	hdr->ipv6 = (void )hdr->eth + sizeof(hdr->eth);
	367	if (hdr->ipv6 + 1 > data_end)
	368	return XDP_DROP;
	369	if (hdr->ipv6->version != 6)
	370	return XDP_DROP;
	371
	372	/* XXX: Extension headers are not supported and could circumvent
	373	* XDP SYN flood protection.
	374	*/
	375	if (hdr->ipv6->nexthdr != NEXTHDR_TCP)
	376	return XDP_PASS;
	377
	378	hdr->tcp = (void )hdr->ipv6 + sizeof(hdr->ipv6);
	379	break;
	380	default:
	381	/* XXX: VLANs will circumvent XDP SYN flood protection. */
	382	return XDP_PASS;
	383	}
	384
	385	if (hdr->tcp + 1 > data_end)
	386	return XDP_DROP;
	387	hdr->tcp_len = hdr->tcp->doff * 4;
	388	if (hdr->tcp_len < sizeof(*hdr->tcp))
	389	return XDP_DROP;
	390
	391	return XDP_TX;
	392	}
	393
	394	static __always_inline int tcp_lookup(void ctx, struct header_pointers hdr, bool xdp)
	395	{
	396	struct bpf_ct_opts ct_lookup_opts = {
	397	.netns_id = BPF_F_CURRENT_NETNS,
	398	.l4proto = IPPROTO_TCP,
	399	};
	400	struct bpf_sock_tuple tup = {};
	401	struct nf_conn *ct;
	402	__u32 tup_size;
	403
	404	if (hdr->ipv4) {
	405	/* TCP doesn't normally use fragments, and XDP can't reassemble
	406	* them.
	407	*/
	408	if ((hdr->ipv4->frag_off & bpf_htons(IP_DF \| IP_MF \| IP_OFFSET)) != bpf_htons(IP_DF))
	409	return XDP_DROP;
	410
	411	tup.ipv4.saddr = hdr->ipv4->saddr;
	412	tup.ipv4.daddr = hdr->ipv4->daddr;
	413	tup.ipv4.sport = hdr->tcp->source;
	414	tup.ipv4.dport = hdr->tcp->dest;
	415	tup_size = sizeof(tup.ipv4);
	416	} else if (hdr->ipv6) {
	417	__builtin_memcpy(tup.ipv6.saddr, &hdr->ipv6->saddr, sizeof(tup.ipv6.saddr));
	418	__builtin_memcpy(tup.ipv6.daddr, &hdr->ipv6->daddr, sizeof(tup.ipv6.daddr));
	419	tup.ipv6.sport = hdr->tcp->source;
	420	tup.ipv6.dport = hdr->tcp->dest;
	421	tup_size = sizeof(tup.ipv6);
	422	} else {
	423	/* The verifier can't track that either ipv4 or ipv6 is not
	424	* NULL.
	425	*/
	426	return XDP_ABORTED;
	427	}
	428	if (xdp)
	429	ct = bpf_xdp_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
	430	else
	431	ct = bpf_skb_ct_lookup(ctx, &tup, tup_size, &ct_lookup_opts, sizeof(ct_lookup_opts));
	432	if (ct) {
	433	unsigned long status = ct->status;
	434
	435	bpf_ct_release(ct);
	436	if (status & IPS_CONFIRMED_BIT)
	437	return XDP_PASS;
	438	} else if (ct_lookup_opts.error != -ENOENT) {
	439	return XDP_ABORTED;
	440	}
	441
	442	/* error == -ENOENT \|\| !(status & IPS_CONFIRMED_BIT) */
	443	return XDP_TX;
	444	}
	445
	446	static __always_inline __u8 tcp_mkoptions(__be32 buf, __be32 tsopt, __u16 mss,
	447	__u8 wscale)
	448	{
	449	__be32 *start = buf;
	450
	451	*buf++ = bpf_htonl((TCPOPT_MSS << 24) \| (TCPOLEN_MSS << 16) \| mss);
	452
	453	if (!tsopt)
	454	return buf - start;
	455
	456	if (tsopt[0] & bpf_htonl(1 << 4))
	457	*buf++ = bpf_htonl((TCPOPT_SACK_PERM << 24) \|
	458	(TCPOLEN_SACK_PERM << 16) \|
	459	(TCPOPT_TIMESTAMP << 8) \|
	460	TCPOLEN_TIMESTAMP);
	461	else
	462	*buf++ = bpf_htonl((TCPOPT_NOP << 24) \|
	463	(TCPOPT_NOP << 16) \|
	464	(TCPOPT_TIMESTAMP << 8) \|
	465	TCPOLEN_TIMESTAMP);
	466	*buf++ = tsopt[0];
	467	*buf++ = tsopt[1];
	468
	469	if ((tsopt[0] & bpf_htonl(0xf)) != bpf_htonl(0xf))
	470	*buf++ = bpf_htonl((TCPOPT_NOP << 24) \|
	471	(TCPOPT_WINDOW << 16) \|
	472	(TCPOLEN_WINDOW << 8) \|
	473	wscale);
	474
	475	return buf - start;
	476	}
	477
	478	static __always_inline void tcp_gen_synack(struct tcphdr *tcp_header,
	479	__u32 cookie, __be32 *tsopt,
	480	__u16 mss, __u8 wscale)
	481	{
	482	void *tcp_options;
	483
	484	tcp_flag_word(tcp_header) = TCP_FLAG_SYN \| TCP_FLAG_ACK;
	485	if (tsopt && (tsopt[0] & bpf_htonl(1 << 5)))
	486	tcp_flag_word(tcp_header) \|= TCP_FLAG_ECE;
	487	tcp_header->doff = 5; /* doff is part of tcp_flag_word. */
	488	swap(tcp_header->source, tcp_header->dest);
	489	tcp_header->ack_seq = bpf_htonl(bpf_ntohl(tcp_header->seq) + 1);
	490	tcp_header->seq = bpf_htonl(cookie);
	491	tcp_header->window = 0;
	492	tcp_header->urg_ptr = 0;
	493	tcp_header->check = 0; /* Calculate checksum later. */
	494
	495	tcp_options = (void *)(tcp_header + 1);
	496	tcp_header->doff += tcp_mkoptions(tcp_options, tsopt, mss, wscale);
	497	}
	498
	499	static __always_inline void tcpv4_gen_synack(struct header_pointers *hdr,
	500	__u32 cookie, __be32 *tsopt)
	501	{
	502	__u8 wscale;
	503	__u16 mss;
	504	__u8 ttl;
	505
	506	values_get_tcpipopts(&mss, &wscale, &ttl, false);
	507
	508	swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
	509
	510	swap(hdr->ipv4->saddr, hdr->ipv4->daddr);
	511	hdr->ipv4->check = 0; /* Calculate checksum later. */
	512	hdr->ipv4->tos = 0;
	513	hdr->ipv4->id = 0;
	514	hdr->ipv4->ttl = ttl;
	515
	516	tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
	517
	518	hdr->tcp_len = hdr->tcp->doff * 4;
	519	hdr->ipv4->tot_len = bpf_htons(sizeof(*hdr->ipv4) + hdr->tcp_len);
	520	}
	521
	522	static __always_inline void tcpv6_gen_synack(struct header_pointers *hdr,
	523	__u32 cookie, __be32 *tsopt)
	524	{
	525	__u8 wscale;
	526	__u16 mss;
	527	__u8 ttl;
	528
	529	values_get_tcpipopts(&mss, &wscale, &ttl, true);
	530
	531	swap_eth_addr(hdr->eth->h_source, hdr->eth->h_dest);
	532
	533	swap(hdr->ipv6->saddr, hdr->ipv6->daddr);
	534	(__be32 )hdr->ipv6 = bpf_htonl(0x60000000);
	535	hdr->ipv6->hop_limit = ttl;
	536
	537	tcp_gen_synack(hdr->tcp, cookie, tsopt, mss, wscale);
	538
	539	hdr->tcp_len = hdr->tcp->doff * 4;
	540	hdr->ipv6->payload_len = bpf_htons(hdr->tcp_len);
	541	}
	542
	543	static __always_inline int syncookie_handle_syn(struct header_pointers *hdr,
	544	void *ctx,
	545	void data, void data_end,
	546	bool xdp)
	547	{
	548	__u32 old_pkt_size, new_pkt_size;
	549	/* Unlike clang 10, clang 11 and 12 generate code that doesn't pass the
	550	* BPF verifier if tsopt is not volatile. Volatile forces it to store
	551	* the pointer value and use it directly, otherwise tcp_mkoptions is
	552	* (mis)compiled like this:
	553	* if (!tsopt)
	554	* return buf - start;
	555	* reg = stored_return_value_of_tscookie_init;
	556	* if (reg)
	557	* tsopt = tsopt_buf;
	558	* else
	559	* tsopt = NULL;
	560	* ...
	561	* *buf++ = tsopt[1];
	562	* It creates a dead branch where tsopt is assigned NULL, but the
	563	* verifier can't prove it's dead and blocks the program.
	564	*/
	565	__be32 * volatile tsopt = NULL;
	566	__be32 tsopt_buf[2] = {};
	567	__u16 ip_len;
	568	__u32 cookie;
	569	__s64 value;
	570
	571	/* Checksum is not yet verified, but both checksum failure and TCP
	572	* header checks return XDP_DROP, so the order doesn't matter.
	573	*/
	574	if (hdr->tcp->fin \|\| hdr->tcp->rst)
	575	return XDP_DROP;
	576
	577	/* Issue SYN cookies on allowed ports, drop SYN packets on blocked
	578	* ports.
	579	*/
	580	if (!check_port_allowed(bpf_ntohs(hdr->tcp->dest)))
	581	return XDP_DROP;
	582
	583	if (hdr->ipv4) {
	584	/* Check the IPv4 and TCP checksums before creating a SYNACK. */
	585	value = bpf_csum_diff(0, 0, (void )hdr->ipv4, hdr->ipv4->ihl 4, 0);
	586	if (value < 0)
	587	return XDP_ABORTED;
	588	if (csum_fold(value) != 0)
	589	return XDP_DROP; /* Bad IPv4 checksum. */
	590
	591	value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
	592	if (value < 0)
	593	return XDP_ABORTED;
	594	if (csum_tcpudp_magic(hdr->ipv4->saddr, hdr->ipv4->daddr,
	595	hdr->tcp_len, IPPROTO_TCP, value) != 0)
	596	return XDP_DROP; /* Bad TCP checksum. */
	597
	598	ip_len = sizeof(*hdr->ipv4);
	599
	600	value = bpf_tcp_raw_gen_syncookie_ipv4(hdr->ipv4, hdr->tcp,
	601	hdr->tcp_len);
	602	} else if (hdr->ipv6) {
	603	/* Check the TCP checksum before creating a SYNACK. */
	604	value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
	605	if (value < 0)
	606	return XDP_ABORTED;
	607	if (csum_ipv6_magic(&hdr->ipv6->saddr, &hdr->ipv6->daddr,
	608	hdr->tcp_len, IPPROTO_TCP, value) != 0)
	609	return XDP_DROP; /* Bad TCP checksum. */
	610
	611	ip_len = sizeof(*hdr->ipv6);
	612
	613	value = bpf_tcp_raw_gen_syncookie_ipv6(hdr->ipv6, hdr->tcp,
	614	hdr->tcp_len);
	615	} else {
	616	return XDP_ABORTED;
	617	}
	618
	619	if (value < 0)
	620	return XDP_ABORTED;
	621	cookie = (__u32)value;
	622
	623	if (tscookie_init((void *)hdr->tcp, hdr->tcp_len,
	624	&tsopt_buf[0], &tsopt_buf[1], data_end))
	625	tsopt = tsopt_buf;
	626
	627	/* Check that there is enough space for a SYNACK. It also covers
	628	* the check that the destination of the __builtin_memmove below
	629	* doesn't overflow.
	630	*/
	631	if (data + sizeof(*hdr->eth) + ip_len + TCP_MAXLEN > data_end)
	632	return XDP_ABORTED;
	633
	634	if (hdr->ipv4) {
	635	if (hdr->ipv4->ihl * 4 > sizeof(*hdr->ipv4)) {
	636	struct tcphdr *new_tcp_header;
	637
	638	new_tcp_header = data + sizeof(hdr->eth) + sizeof(hdr->ipv4);
	639	__builtin_memmove(new_tcp_header, hdr->tcp, sizeof(*hdr->tcp));
	640	hdr->tcp = new_tcp_header;
	641
	642	hdr->ipv4->ihl = sizeof(*hdr->ipv4) / 4;
	643	}
	644
	645	tcpv4_gen_synack(hdr, cookie, tsopt);
	646	} else if (hdr->ipv6) {
	647	tcpv6_gen_synack(hdr, cookie, tsopt);
	648	} else {
	649	return XDP_ABORTED;
	650	}
	651
	652	/* Recalculate checksums. */
	653	hdr->tcp->check = 0;
	654	value = bpf_csum_diff(0, 0, (void *)hdr->tcp, hdr->tcp_len, 0);
	655	if (value < 0)
	656	return XDP_ABORTED;
	657	if (hdr->ipv4) {
	658	hdr->tcp->check = csum_tcpudp_magic(hdr->ipv4->saddr,
	659	hdr->ipv4->daddr,
	660	hdr->tcp_len,
	661	IPPROTO_TCP,
	662	value);
	663
	664	hdr->ipv4->check = 0;
	665	value = bpf_csum_diff(0, 0, (void )hdr->ipv4, sizeof(hdr->ipv4), 0);
	666	if (value < 0)
	667	return XDP_ABORTED;
	668	hdr->ipv4->check = csum_fold(value);
	669	} else if (hdr->ipv6) {
	670	hdr->tcp->check = csum_ipv6_magic(&hdr->ipv6->saddr,
	671	&hdr->ipv6->daddr,
	672	hdr->tcp_len,
	673	IPPROTO_TCP,
	674	value);
	675	} else {
	676	return XDP_ABORTED;
	677	}
	678
	679	/* Set the new packet size. */
	680	old_pkt_size = data_end - data;
	681	new_pkt_size = sizeof(hdr->eth) + ip_len + hdr->tcp->doff 4;
	682	if (xdp) {
	683	if (bpf_xdp_adjust_tail(ctx, new_pkt_size - old_pkt_size))
	684	return XDP_ABORTED;
	685	} else {
	686	if (bpf_skb_change_tail(ctx, new_pkt_size, 0))
	687	return XDP_ABORTED;
	688	}
	689
	690	values_inc_synacks();
	691
	692	return XDP_TX;
	693	}
	694
	695	static __always_inline int syncookie_handle_ack(struct header_pointers *hdr)
	696	{
	697	int err;
	698
	699	if (hdr->tcp->rst)
	700	return XDP_DROP;
	701
	702	if (hdr->ipv4)
	703	err = bpf_tcp_raw_check_syncookie_ipv4(hdr->ipv4, hdr->tcp);
	704	else if (hdr->ipv6)
	705	err = bpf_tcp_raw_check_syncookie_ipv6(hdr->ipv6, hdr->tcp);
	706	else
	707	return XDP_ABORTED;
	708	if (err)
	709	return XDP_DROP;
	710
	711	return XDP_PASS;
	712	}
	713
	714	static __always_inline int syncookie_part1(void ctx, void data, void *data_end,
	715	struct header_pointers *hdr, bool xdp)
	716	{
	717	struct bpf_ct_opts ct_lookup_opts = {
	718	.netns_id = BPF_F_CURRENT_NETNS,
	719	.l4proto = IPPROTO_TCP,
	720	};
	721	int ret;
	722
	723	ret = tcp_dissect(data, data_end, hdr);
	724	if (ret != XDP_TX)
	725	return ret;
	726
	727	ret = tcp_lookup(ctx, hdr, xdp);
	728	if (ret != XDP_TX)
	729	return ret;
	730
	731	/* Packet is TCP and doesn't belong to an established connection. */
	732
	733	if ((hdr->tcp->syn ^ hdr->tcp->ack) != 1)
	734	return XDP_DROP;
	735
	736	/* Grow the TCP header to TCP_MAXLEN to be able to pass any hdr->tcp_len
	737	* to bpf_tcp_raw_gen_syncookie_ipv{4,6} and pass the verifier.
	738	*/
	739	if (xdp) {
	740	if (bpf_xdp_adjust_tail(ctx, TCP_MAXLEN - hdr->tcp_len))
	741	return XDP_ABORTED;
	742	} else {
	743	/* Without volatile the verifier throws this error:
	744	* R9 32-bit pointer arithmetic prohibited
	745	*/
	746	volatile u64 old_len = data_end - data;
	747
	748	if (bpf_skb_change_tail(ctx, old_len + TCP_MAXLEN - hdr->tcp_len, 0))
	749	return XDP_ABORTED;
	750	}
	751
	752	return XDP_TX;
	753	}
	754
	755	static __always_inline int syncookie_part2(void ctx, void data, void *data_end,
	756	struct header_pointers *hdr, bool xdp)
	757	{
	758	if (hdr->ipv4) {
	759	hdr->eth = data;
	760	hdr->ipv4 = (void )hdr->eth + sizeof(hdr->eth);
	761	/* IPV4_MAXLEN is needed when calculating checksum.
	762	* At least sizeof(struct iphdr) is needed here to access ihl.
	763	*/
	764	if ((void *)hdr->ipv4 + IPV4_MAXLEN > data_end)
	765	return XDP_ABORTED;
	766	hdr->tcp = (void )hdr->ipv4 + hdr->ipv4->ihl 4;
	767	} else if (hdr->ipv6) {
	768	hdr->eth = data;
	769	hdr->ipv6 = (void )hdr->eth + sizeof(hdr->eth);
	770	hdr->tcp = (void )hdr->ipv6 + sizeof(hdr->ipv6);
	771	} else {
	772	return XDP_ABORTED;
	773	}
	774
	775	if ((void *)hdr->tcp + TCP_MAXLEN > data_end)
	776	return XDP_ABORTED;
	777
	778	/* We run out of registers, tcp_len gets spilled to the stack, and the
	779	* verifier forgets its min and max values checked above in tcp_dissect.
	780	*/
	781	hdr->tcp_len = hdr->tcp->doff * 4;
	782	if (hdr->tcp_len < sizeof(*hdr->tcp))
	783	return XDP_ABORTED;
	784
	785	return hdr->tcp->syn ? syncookie_handle_syn(hdr, ctx, data, data_end, xdp) :
	786	syncookie_handle_ack(hdr);
	787	}
	788
	789	SEC("xdp")
	790	int syncookie_xdp(struct xdp_md *ctx)
	791	{
	792	void data_end = (void )(long)ctx->data_end;
	793	void data = (void )(long)ctx->data;
	794	struct header_pointers hdr;
	795	int ret;
	796
	797	ret = syncookie_part1(ctx, data, data_end, &hdr, true);
	798	if (ret != XDP_TX)
	799	return ret;
	800
	801	data_end = (void *)(long)ctx->data_end;
	802	data = (void *)(long)ctx->data;
	803
	804	return syncookie_part2(ctx, data, data_end, &hdr, true);
	805	}
	806
	807	SEC("tc")
	808	int syncookie_tc(struct __sk_buff *skb)
	809	{
	810	void data_end = (void )(long)skb->data_end;
	811	void data = (void )(long)skb->data;
	812	struct header_pointers hdr;
	813	int ret;
	814
	815	ret = syncookie_part1(skb, data, data_end, &hdr, false);
	816	if (ret != XDP_TX)
	817	return ret == XDP_PASS ? TC_ACT_OK : TC_ACT_SHOT;
	818
	819	data_end = (void *)(long)skb->data_end;
	820	data = (void *)(long)skb->data;
	821
	822	ret = syncookie_part2(skb, data, data_end, &hdr, false);
	823	switch (ret) {
	824	case XDP_PASS:
	825	return TC_ACT_OK;
	826	case XDP_TX:
	827	return bpf_redirect(skb->ifindex, 0);
	828	default:
	829	return TC_ACT_SHOT;
	830	}
	831	}
	832
	833	char _license[] SEC("license") = "GPL";

tools/testing/selftests/bpf/test_bpftool_synctypes.py+70−96

@@ -58,7 +58,7 @@ class BlockParser(object): · 57 unmodified lines
58		58
59	class ArrayParser(BlockParser):	59	class ArrayParser(BlockParser):
60	"""	60	"""
61	A parser for extracting dicionaries of values from some BPF-related arrays.	61	A parser for extracting a set of values from some BPF-related arrays.
62	@reader: a pointer to the open file to parse	62	@reader: a pointer to the open file to parse
63	@array_name: name of the array to parse	63	@array_name: name of the array to parse
64	"""	64	"""
@@ -66,7 +66,7 @@ class ArrayParser(BlockParser): · 1 unmodified line
66		66
67	def __init__(self, reader, array_name):	67	def __init__(self, reader, array_name):
68	self.array_name = array_name	68	self.array_name = array_name
69	self.start_marker = re.compile(f'(static )?const char \* const {self.array_name}\[.*\] = {{\n')	69	self.start_marker = re.compile(f'(static )?const bool {self.array_name}\[.*\] = {{\n')
70	super().__init__(reader)	70	super().__init__(reader)
71		71
72	def search_block(self):	72	def search_block(self):
@@ -80,15 +80,15 @@ class ArrayParser(BlockParser): · 7 unmodified lines
80	Parse a block and return data as a dictionary. Items to extract must be	80	Parse a block and return data as a dictionary. Items to extract must be
81	on separate lines in the file.	81	on separate lines in the file.
82	"""	82	"""
83	pattern = re.compile('\[(BPF_\w)\]\s= "(.*)",?$')	83	pattern = re.compile('\[(BPF_\w)\]\s= (true\|false),?$')
84	entries = {}	84	entries = set()
85	while True:	85	while True:
86	line = self.reader.readline()	86	line = self.reader.readline()
87	if line == '' or re.match(self.end_marker, line):	87	if line == '' or re.match(self.end_marker, line):
88	break	88	break
89	capture = pattern.search(line)	89	capture = pattern.search(line)
90	if capture:	90	if capture:
91	entries[capture.group(1)] = capture.group(2)	91	entries \|= {capture.group(1)}
92	return entries	92	return entries
93		93
94	class InlineListParser(BlockParser):	94	class InlineListParser(BlockParser):
@@ -115,7 +115,7 @@ class InlineListParser(BlockParser): · 20 unmodified lines
115	class FileExtractor(object):	115	class FileExtractor(object):
116	"""	116	"""
117	A generic reader for extracting data from a given file. This class contains	117	A generic reader for extracting data from a given file. This class contains
118	several helper methods that wrap arround parser objects to extract values	118	several helper methods that wrap around parser objects to extract values
119	from different structures.	119	from different structures.
120	This class does not offer a way to set a filename, which is expected to be	120	This class does not offer a way to set a filename, which is expected to be
121	defined in children classes.	121	defined in children classes.
@@ -139,21 +139,19 @@ class FileExtractor(object): · 17 unmodified lines
139		139
140	def get_types_from_array(self, array_name):	140	def get_types_from_array(self, array_name):
141	"""	141	"""
142	Search for and parse an array associating names to BPF_* enum members,	142	Search for and parse a list of allowed BPF_* enum members, for example:
143	for example:
144		143
145	const char * const prog_type_name[] = {	144	const bool prog_type_name[] = {
146	[BPF_PROG_TYPE_UNSPEC] = "unspec",	145	[BPF_PROG_TYPE_UNSPEC] = true,
147	[BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",	146	[BPF_PROG_TYPE_SOCKET_FILTER] = true,
148	[BPF_PROG_TYPE_KPROBE] = "kprobe",	147	[BPF_PROG_TYPE_KPROBE] = true,
149	};	148	};
150		149
151	Return a dictionary with the enum member names as keys and the	150	Return a set of the enum members, for example:
152	associated names as values, for example:
153		151
154	{'BPF_PROG_TYPE_UNSPEC': 'unspec',	152	{'BPF_PROG_TYPE_UNSPEC',
155	'BPF_PROG_TYPE_SOCKET_FILTER': 'socket_filter',	153	'BPF_PROG_TYPE_SOCKET_FILTER',
156	'BPF_PROG_TYPE_KPROBE': 'kprobe'}	154	'BPF_PROG_TYPE_KPROBE'}
157		155
158	@array_name: name of the array to parse	156	@array_name: name of the array to parse
159	"""	157	"""
@@ -186,6 +184,27 @@ class FileExtractor(object): · 26 unmodified lines
186	parser.search_block(start_marker)	184	parser.search_block(start_marker)
187	return parser.parse(pattern, end_marker)	185	return parser.parse(pattern, end_marker)
188		186
		187	def make_enum_map(self, names, enum_prefix):
		188	"""
		189	Search for and parse an enum containing BPF_* members, just as get_enum
		190	does. However, instead of just returning a set of the variant names,
		191	also generate a textual representation from them by (assuming and)
		192	removing a provided prefix and lowercasing the remainder. Then return a
		193	dict mapping from name to textual representation.
		194
		195	@enum_values: a set of enum values; e.g., as retrieved by get_enum
		196	@enum_prefix: the prefix to remove from each of the variants to infer
		197	textual representation
		198	"""
		199	mapping = {}
		200	for name in names:
		201	if not name.startswith(enum_prefix):
		202	raise Exception(f"enum variant {name} does not start with {enum_prefix}")
		203	text = name[len(enum_prefix):].lower()
		204	mapping[name] = text
		205
		206	return mapping
		207
189	def __get_description_list(self, start_marker, pattern, end_marker):	208	def __get_description_list(self, start_marker, pattern, end_marker):
190	parser = InlineListParser(self.reader)	209	parser = InlineListParser(self.reader)
191	parser.search_block(start_marker)	210	parser.search_block(start_marker)
@@ -333,11 +352,9 @@ class ProgFileExtractor(SourceFileExtractor): · 141 unmodified lines
333	"""	352	"""
334	filename = os.path.join(BPFTOOL_DIR, 'prog.c')	353	filename = os.path.join(BPFTOOL_DIR, 'prog.c')
335		354
336	def get_prog_types(self):
337	return self.get_types_from_array('prog_type_name')
338
339	def get_attach_types(self):	355	def get_attach_types(self):
340	return self.get_types_from_array('attach_type_strings')	356	types = self.get_types_from_array('attach_types')
		357	return self.make_enum_map(types, 'BPF_')
341		358
342	def get_prog_attach_help(self):	359	def get_prog_attach_help(self):
343	return self.get_help_list('ATTACH_TYPE')	360	return self.get_help_list('ATTACH_TYPE')
@@ -348,9 +365,6 @@ class MapFileExtractor(SourceFileExtractor): · 4 unmodified lines
348	"""	365	"""
349	filename = os.path.join(BPFTOOL_DIR, 'map.c')	366	filename = os.path.join(BPFTOOL_DIR, 'map.c')
350		367
351	def get_map_types(self):
352	return self.get_types_from_array('map_type_name')
353
354	def get_map_help(self):	368	def get_map_help(self):
355	return self.get_help_list('TYPE')	369	return self.get_help_list('TYPE')
356		370
@@ -363,30 +377,6 @@ class CgroupFileExtractor(SourceFileExtractor): · 6 unmodified lines
363	def get_prog_attach_help(self):	377	def get_prog_attach_help(self):
364	return self.get_help_list('ATTACH_TYPE')	378	return self.get_help_list('ATTACH_TYPE')
365		379
366	class CommonFileExtractor(SourceFileExtractor):
367	"""
368	An extractor for bpftool's common.c.
369	"""
370	filename = os.path.join(BPFTOOL_DIR, 'common.c')
371
372	def __init__(self):
373	super().__init__()
374	self.attach_types = {}
375
376	def get_attach_types(self):
377	if not self.attach_types:
378	self.attach_types = self.get_types_from_array('attach_type_name')
379	return self.attach_types
380
381	def get_cgroup_attach_types(self):
382	if not self.attach_types:
383	self.get_attach_types()
384	cgroup_types = {}
385	for (key, value) in self.attach_types.items():
386	if key.find('BPF_CGROUP') != -1:
387	cgroup_types[key] = value
388	return cgroup_types
389
390	class GenericSourceExtractor(SourceFileExtractor):	380	class GenericSourceExtractor(SourceFileExtractor):
391	"""	381	"""
392	An extractor for generic source code files.	382	An extractor for generic source code files.
@@ -403,14 +393,28 @@ class BpfHeaderExtractor(FileExtractor): · 10 unmodified lines
403	"""	393	"""
404	filename = os.path.join(INCLUDE_DIR, 'uapi/linux/bpf.h')	394	filename = os.path.join(INCLUDE_DIR, 'uapi/linux/bpf.h')
405		395
		396	def __init__(self):
		397	super().__init__()
		398	self.attach_types = {}
		399
406	def get_prog_types(self):	400	def get_prog_types(self):
407	return self.get_enum('bpf_prog_type')	401	return self.get_enum('bpf_prog_type')
408		402
409	def get_map_types(self):	403	def get_map_type_map(self):
410	return self.get_enum('bpf_map_type')	404	names = self.get_enum('bpf_map_type')
		405	return self.make_enum_map(names, 'BPF_MAP_TYPE_')
411		406
412	def get_attach_types(self):	407	def get_attach_type_map(self):
413	return self.get_enum('bpf_attach_type')	408	if not self.attach_types:
		409	names = self.get_enum('bpf_attach_type')
		410	self.attach_types = self.make_enum_map(names, 'BPF_')
		411	return self.attach_types
		412
		413	def get_cgroup_attach_type_map(self):
		414	if not self.attach_types:
		415	self.get_attach_type_map()
		416	return {name: text for name, text in self.attach_types.items()
		417	if name.startswith('BPF_CGROUP')}
414		418
415	class ManPageExtractor(FileExtractor):	419	class ManPageExtractor(FileExtractor):
416	"""	420	"""
@@ -495,21 +499,12 @@ def main(): · 78 unmodified lines
495	""")	499	""")
496	args = argParser.parse_args()	500	args = argParser.parse_args()
497		501
498	# Map types (enum)
499
500	bpf_info = BpfHeaderExtractor()	502	bpf_info = BpfHeaderExtractor()
501	ref = bpf_info.get_map_types()
502
503	map_info = MapFileExtractor()
504	source_map_items = map_info.get_map_types()
505	map_types_enum = set(source_map_items.keys())
506
507	verify(ref, map_types_enum,
508	f'Comparing BPF header (enum bpf_map_type) and {MapFileExtractor.filename} (map_type_name):')
509		503
510	# Map types (names)	504	# Map types (names)
511		505
512	source_map_types = set(source_map_items.values())	506	map_info = MapFileExtractor()
		507	source_map_types = set(bpf_info.get_map_type_map().values())
513	source_map_types.discard('unspec')	508	source_map_types.discard('unspec')
514		509
515	help_map_types = map_info.get_map_help()	510	help_map_types = map_info.get_map_help()
@@ -525,37 +520,17 @@ def main(): · 9 unmodified lines
525	bashcomp_map_types = bashcomp_info.get_map_types()	520	bashcomp_map_types = bashcomp_info.get_map_types()
526		521
527	verify(source_map_types, help_map_types,	522	verify(source_map_types, help_map_types,
528	f'Comparing {MapFileExtractor.filename} (map_type_name) and {MapFileExtractor.filename} (do_help() TYPE):')	523	f'Comparing {BpfHeaderExtractor.filename} (bpf_map_type) and {MapFileExtractor.filename} (do_help() TYPE):')
529	verify(source_map_types, man_map_types,	524	verify(source_map_types, man_map_types,
530	f'Comparing {MapFileExtractor.filename} (map_type_name) and {ManMapExtractor.filename} (TYPE):')	525	f'Comparing {BpfHeaderExtractor.filename} (bpf_map_type) and {ManMapExtractor.filename} (TYPE):')
531	verify(help_map_options, man_map_options,	526	verify(help_map_options, man_map_options,
532	f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):')	527	f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):')
533	verify(source_map_types, bashcomp_map_types,	528	verify(source_map_types, bashcomp_map_types,
534	f'Comparing {MapFileExtractor.filename} (map_type_name) and {BashcompExtractor.filename} (BPFTOOL_MAP_CREATE_TYPES):')	529	f'Comparing {BpfHeaderExtractor.filename} (bpf_map_type) and {BashcompExtractor.filename} (BPFTOOL_MAP_CREATE_TYPES):')
535
536	# Program types (enum)
537
538	ref = bpf_info.get_prog_types()
539
540	prog_info = ProgFileExtractor()
541	prog_types = set(prog_info.get_prog_types().keys())
542
543	verify(ref, prog_types,
544	f'Comparing BPF header (enum bpf_prog_type) and {ProgFileExtractor.filename} (prog_type_name):')
545
546	# Attach types (enum)
547
548	ref = bpf_info.get_attach_types()
549	bpf_info.close()
550
551	common_info = CommonFileExtractor()
552	attach_types = common_info.get_attach_types()
553
554	verify(ref, attach_types,
555	f'Comparing BPF header (enum bpf_attach_type) and {CommonFileExtractor.filename} (attach_type_name):')
556		530
557	# Attach types (names)	531	# Attach types (names)
558		532
		533	prog_info = ProgFileExtractor()
559	source_prog_attach_types = set(prog_info.get_attach_types().values())	534	source_prog_attach_types = set(prog_info.get_attach_types().values())
560		535
561	help_prog_attach_types = prog_info.get_prog_attach_help()	536	help_prog_attach_types = prog_info.get_prog_attach_help()
@@ -571,18 +546,17 @@ def main(): · 9 unmodified lines
571	bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types()	546	bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types()
572		547
573	verify(source_prog_attach_types, help_prog_attach_types,	548	verify(source_prog_attach_types, help_prog_attach_types,
574	f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):')	549	f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):')
575	verify(source_prog_attach_types, man_prog_attach_types,	550	verify(source_prog_attach_types, man_prog_attach_types,
576	f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ManProgExtractor.filename} (ATTACH_TYPE):')	551	f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {ManProgExtractor.filename} (ATTACH_TYPE):')
577	verify(help_prog_options, man_prog_options,	552	verify(help_prog_options, man_prog_options,
578	f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):')	553	f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):')
579	verify(source_prog_attach_types, bashcomp_prog_attach_types,	554	verify(source_prog_attach_types, bashcomp_prog_attach_types,
580	f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):')	555	f'Comparing {ProgFileExtractor.filename} (bpf_attach_type) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):')
581		556
582	# Cgroup attach types	557	# Cgroup attach types
583		558	source_cgroup_attach_types = set(bpf_info.get_cgroup_attach_type_map().values())
584	source_cgroup_attach_types = set(common_info.get_cgroup_attach_types().values())	559	bpf_info.close()
585	common_info.close()
586		560
587	cgroup_info = CgroupFileExtractor()	561	cgroup_info = CgroupFileExtractor()
588	help_cgroup_attach_types = cgroup_info.get_prog_attach_help()	562	help_cgroup_attach_types = cgroup_info.get_prog_attach_help()
@@ -598,13 +572,13 @@ def main(): · 9 unmodified lines
598	bashcomp_info.close()	572	bashcomp_info.close()
599		573
600	verify(source_cgroup_attach_types, help_cgroup_attach_types,	574	verify(source_cgroup_attach_types, help_cgroup_attach_types,
601	f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):')	575	f'Comparing {BpfHeaderExtractor.filename} (bpf_attach_type) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):')
602	verify(source_cgroup_attach_types, man_cgroup_attach_types,	576	verify(source_cgroup_attach_types, man_cgroup_attach_types,
603	f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {ManCgroupExtractor.filename} (ATTACH_TYPE):')	577	f'Comparing {BpfHeaderExtractor.filename} (bpf_attach_type) and {ManCgroupExtractor.filename} (ATTACH_TYPE):')
604	verify(help_cgroup_options, man_cgroup_options,	578	verify(help_cgroup_options, man_cgroup_options,
605	f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):')	579	f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):')
606	verify(source_cgroup_attach_types, bashcomp_cgroup_attach_types,	580	verify(source_cgroup_attach_types, bashcomp_cgroup_attach_types,
607	f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_CGROUP_ATTACH_TYPES):')	581	f'Comparing {BpfHeaderExtractor.filename} (bpf_attach_type) and {BashcompExtractor.filename} (BPFTOOL_CGROUP_ATTACH_TYPES):')
608		582
609	# Options for remaining commands	583	# Options for remaining commands
610		584

tools/testing/selftests/bpf/test_btf.h+1−0

@@ -39,6 +39,7 @@ · 38 unmodified lines
39	#define BTF_MEMBER_ENC(name, type, bits_offset) \	39	#define BTF_MEMBER_ENC(name, type, bits_offset) \
40	(name), (type), (bits_offset)	40	(name), (type), (bits_offset)
41	#define BTF_ENUM_ENC(name, val) (name), (val)	41	#define BTF_ENUM_ENC(name, val) (name), (val)
		42	#define BTF_ENUM64_ENC(name, val_lo32, val_hi32) (name), (val_lo32), (val_hi32)
42	#define BTF_MEMBER_OFFSET(bitfield_size, bits_offset) \	43	#define BTF_MEMBER_OFFSET(bitfield_size, bits_offset) \
43	((bitfield_size) << 24 \| (bits_offset))	44	((bitfield_size) << 24 \| (bits_offset))
44		45

tools/testing/selftests/bpf/test_xdping.sh+4−0

@@ -95,5 +95,9 @@ for server_args in "" "-I veth0 -s -S" ; do · 94 unmodified lines
95	test "$client_args" "$server_args"	95	test "$client_args" "$server_args"
96	done	96	done
97		97
		98	# Test drv mode
		99	test "-I veth1 -N" "-I veth0 -s -N"
		100	test "-I veth1 -N -c 10" "-I veth0 -s -N"
		101
98	echo "OK. All tests passed"	102	echo "OK. All tests passed"
99	exit 0	103	exit 0

tools/testing/selftests/bpf/xdp_synproxy.c+466−0

@@ -0,0 +1,466 @@
	1	// SPDX-License-Identifier: LGPL-2.1 OR BSD-2-Clause
	2	/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
	3
	4	#include <stdnoreturn.h>
	5	#include <stdlib.h>
	6	#include <stdio.h>
	7	#include <string.h>
	8	#include <errno.h>
	9	#include <unistd.h>
	10	#include <getopt.h>
	11	#include <signal.h>
	12	#include <sys/types.h>
	13	#include <bpf/bpf.h>
	14	#include <bpf/libbpf.h>
	15	#include <net/if.h>
	16	#include <linux/if_link.h>
	17	#include <linux/limits.h>
	18
	19	static unsigned int ifindex;
	20	static __u32 attached_prog_id;
	21	static bool attached_tc;
	22
	23	static void noreturn cleanup(int sig)
	24	{
	25	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
	26	int prog_fd;
	27	int err;
	28
	29	if (attached_prog_id == 0)
	30	exit(0);
	31
	32	if (attached_tc) {
	33	LIBBPF_OPTS(bpf_tc_hook, hook,
	34	.ifindex = ifindex,
	35	.attach_point = BPF_TC_INGRESS);
	36
	37	err = bpf_tc_hook_destroy(&hook);
	38	if (err < 0) {
	39	fprintf(stderr, "Error: bpf_tc_hook_destroy: %s\n", strerror(-err));
	40	fprintf(stderr, "Failed to destroy the TC hook\n");
	41	exit(1);
	42	}
	43	exit(0);
	44	}
	45
	46	prog_fd = bpf_prog_get_fd_by_id(attached_prog_id);
	47	if (prog_fd < 0) {
	48	fprintf(stderr, "Error: bpf_prog_get_fd_by_id: %s\n", strerror(-prog_fd));
	49	err = bpf_xdp_attach(ifindex, -1, 0, NULL);
	50	if (err < 0) {
	51	fprintf(stderr, "Error: bpf_set_link_xdp_fd: %s\n", strerror(-err));
	52	fprintf(stderr, "Failed to detach XDP program\n");
	53	exit(1);
	54	}
	55	} else {
	56	opts.old_prog_fd = prog_fd;
	57	err = bpf_xdp_attach(ifindex, -1, XDP_FLAGS_REPLACE, &opts);
	58	close(prog_fd);
	59	if (err < 0) {
	60	fprintf(stderr, "Error: bpf_set_link_xdp_fd_opts: %s\n", strerror(-err));
	61	/* Not an error if already replaced by someone else. */
	62	if (err != -EEXIST) {
	63	fprintf(stderr, "Failed to detach XDP program\n");
	64	exit(1);
	65	}
	66	}
	67	}
	68	exit(0);
	69	}
	70
	71	static noreturn void usage(const char *progname)
	72	{
	73	fprintf(stderr, "Usage: %s [--iface <iface>\|--prog <prog_id>] [--mss4 <mss ipv4> --mss6 <mss ipv6> --wscale <wscale> --ttl <ttl>] [--ports <port1>,<port2>,...] [--single] [--tc]\n",
	74	progname);
	75	exit(1);
	76	}
	77
	78	static unsigned long parse_arg_ul(const char progname, const char arg, unsigned long limit)
	79	{
	80	unsigned long res;
	81	char *endptr;
	82
	83	errno = 0;
	84	res = strtoul(arg, &endptr, 10);
	85	if (errno != 0 \|\| *endptr != '\0' \|\| arg[0] == '\0' \|\| res > limit)
	86	usage(progname);
	87
	88	return res;
	89	}
	90
	91	static void parse_options(int argc, char argv[], unsigned int ifindex, __u32 *prog_id,
	92	__u64 tcpipopts, char ports, bool single, bool *tc)
	93	{
	94	static struct option long_options[] = {
	95	{ "help", no_argument, NULL, 'h' },
	96	{ "iface", required_argument, NULL, 'i' },
	97	{ "prog", required_argument, NULL, 'x' },
	98	{ "mss4", required_argument, NULL, 4 },
	99	{ "mss6", required_argument, NULL, 6 },
	100	{ "wscale", required_argument, NULL, 'w' },
	101	{ "ttl", required_argument, NULL, 't' },
	102	{ "ports", required_argument, NULL, 'p' },
	103	{ "single", no_argument, NULL, 's' },
	104	{ "tc", no_argument, NULL, 'c' },
	105	{ NULL, 0, NULL, 0 },
	106	};
	107	unsigned long mss4, mss6, wscale, ttl;
	108	unsigned int tcpipopts_mask = 0;
	109
	110	if (argc < 2)
	111	usage(argv[0]);
	112
	113	*ifindex = 0;
	114	*prog_id = 0;
	115	*tcpipopts = 0;
	116	*ports = NULL;
	117	*single = false;
	118
	119	while (true) {
	120	int opt;
	121
	122	opt = getopt_long(argc, argv, "", long_options, NULL);
	123	if (opt == -1)
	124	break;
	125
	126	switch (opt) {
	127	case 'h':
	128	usage(argv[0]);
	129	break;
	130	case 'i':
	131	*ifindex = if_nametoindex(optarg);
	132	if (*ifindex == 0)
	133	usage(argv[0]);
	134	break;
	135	case 'x':
	136	*prog_id = parse_arg_ul(argv[0], optarg, UINT32_MAX);
	137	if (*prog_id == 0)
	138	usage(argv[0]);
	139	break;
	140	case 4:
	141	mss4 = parse_arg_ul(argv[0], optarg, UINT16_MAX);
	142	tcpipopts_mask \|= 1 << 0;
	143	break;
	144	case 6:
	145	mss6 = parse_arg_ul(argv[0], optarg, UINT16_MAX);
	146	tcpipopts_mask \|= 1 << 1;
	147	break;
	148	case 'w':
	149	wscale = parse_arg_ul(argv[0], optarg, 14);
	150	tcpipopts_mask \|= 1 << 2;
	151	break;
	152	case 't':
	153	ttl = parse_arg_ul(argv[0], optarg, UINT8_MAX);
	154	tcpipopts_mask \|= 1 << 3;
	155	break;
	156	case 'p':
	157	*ports = optarg;
	158	break;
	159	case 's':
	160	*single = true;
	161	break;
	162	case 'c':
	163	*tc = true;
	164	break;
	165	default:
	166	usage(argv[0]);
	167	}
	168	}
	169	if (optind < argc)
	170	usage(argv[0]);
	171
	172	if (tcpipopts_mask == 0xf) {
	173	if (mss4 == 0 \|\| mss6 == 0 \|\| wscale == 0 \|\| ttl == 0)
	174	usage(argv[0]);
	175	*tcpipopts = (mss6 << 32) \| (ttl << 24) \| (wscale << 16) \| mss4;
	176	} else if (tcpipopts_mask != 0) {
	177	usage(argv[0]);
	178	}
	179
	180	if (ifindex != 0 && prog_id != 0)
	181	usage(argv[0]);
	182	if (ifindex == 0 && prog_id == 0)
	183	usage(argv[0]);
	184	}
	185
	186	static int syncookie_attach(const char *argv0, unsigned int ifindex, bool tc)
	187	{
	188	struct bpf_prog_info info = {};
	189	__u32 info_len = sizeof(info);
	190	char xdp_filename[PATH_MAX];
	191	struct bpf_program *prog;
	192	struct bpf_object *obj;
	193	int prog_fd;
	194	int err;
	195
	196	snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv0);
	197	obj = bpf_object__open_file(xdp_filename, NULL);
	198	err = libbpf_get_error(obj);
	199	if (err < 0) {
	200	fprintf(stderr, "Error: bpf_object__open_file: %s\n", strerror(-err));
	201	return err;
	202	}
	203
	204	err = bpf_object__load(obj);
	205	if (err < 0) {
	206	fprintf(stderr, "Error: bpf_object__open_file: %s\n", strerror(-err));
	207	return err;
	208	}
	209
	210	prog = bpf_object__find_program_by_name(obj, tc ? "syncookie_tc" : "syncookie_xdp");
	211	if (!prog) {
	212	fprintf(stderr, "Error: bpf_object__find_program_by_name: program was not found\n");
	213	return -ENOENT;
	214	}
	215
	216	prog_fd = bpf_program__fd(prog);
	217
	218	err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len);
	219	if (err < 0) {
	220	fprintf(stderr, "Error: bpf_obj_get_info_by_fd: %s\n", strerror(-err));
	221	goto out;
	222	}
	223	attached_tc = tc;
	224	attached_prog_id = info.id;
	225	signal(SIGINT, cleanup);
	226	signal(SIGTERM, cleanup);
	227	if (tc) {
	228	LIBBPF_OPTS(bpf_tc_hook, hook,
	229	.ifindex = ifindex,
	230	.attach_point = BPF_TC_INGRESS);
	231	LIBBPF_OPTS(bpf_tc_opts, opts,
	232	.handle = 1,
	233	.priority = 1,
	234	.prog_fd = prog_fd);
	235
	236	err = bpf_tc_hook_create(&hook);
	237	if (err < 0) {
	238	fprintf(stderr, "Error: bpf_tc_hook_create: %s\n",
	239	strerror(-err));
	240	goto fail;
	241	}
	242	err = bpf_tc_attach(&hook, &opts);
	243	if (err < 0) {
	244	fprintf(stderr, "Error: bpf_tc_attach: %s\n",
	245	strerror(-err));
	246	goto fail;
	247	}
	248
	249	} else {
	250	err = bpf_xdp_attach(ifindex, prog_fd,
	251	XDP_FLAGS_UPDATE_IF_NOEXIST, NULL);
	252	if (err < 0) {
	253	fprintf(stderr, "Error: bpf_set_link_xdp_fd: %s\n",
	254	strerror(-err));
	255	goto fail;
	256	}
	257	}
	258	err = 0;
	259	out:
	260	bpf_object__close(obj);
	261	return err;
	262	fail:
	263	signal(SIGINT, SIG_DFL);
	264	signal(SIGTERM, SIG_DFL);
	265	attached_prog_id = 0;
	266	goto out;
	267	}
	268
	269	static int syncookie_open_bpf_maps(__u32 prog_id, int values_map_fd, int ports_map_fd)
	270	{
	271	struct bpf_prog_info prog_info;
	272	__u32 map_ids[8];
	273	__u32 info_len;
	274	int prog_fd;
	275	int err;
	276	int i;
	277
	278	*values_map_fd = -1;
	279	*ports_map_fd = -1;
	280
	281	prog_fd = bpf_prog_get_fd_by_id(prog_id);
	282	if (prog_fd < 0) {
	283	fprintf(stderr, "Error: bpf_prog_get_fd_by_id: %s\n", strerror(-prog_fd));
	284	return prog_fd;
	285	}
	286
	287	prog_info = (struct bpf_prog_info) {
	288	.nr_map_ids = 8,
	289	.map_ids = (__u64)map_ids,
	290	};
	291	info_len = sizeof(prog_info);
	292
	293	err = bpf_obj_get_info_by_fd(prog_fd, &prog_info, &info_len);
	294	if (err != 0) {
	295	fprintf(stderr, "Error: bpf_obj_get_info_by_fd: %s\n", strerror(-err));
	296	goto out;
	297	}
	298
	299	if (prog_info.nr_map_ids < 2) {
	300	fprintf(stderr, "Error: Found %u BPF maps, expected at least 2\n",
	301	prog_info.nr_map_ids);
	302	err = -ENOENT;
	303	goto out;
	304	}
	305
	306	for (i = 0; i < prog_info.nr_map_ids; i++) {
	307	struct bpf_map_info map_info = {};
	308	int map_fd;
	309
	310	err = bpf_map_get_fd_by_id(map_ids[i]);
	311	if (err < 0) {
	312	fprintf(stderr, "Error: bpf_map_get_fd_by_id: %s\n", strerror(-err));
	313	goto err_close_map_fds;
	314	}
	315	map_fd = err;
	316
	317	info_len = sizeof(map_info);
	318	err = bpf_obj_get_info_by_fd(map_fd, &map_info, &info_len);
	319	if (err != 0) {
	320	fprintf(stderr, "Error: bpf_obj_get_info_by_fd: %s\n", strerror(-err));
	321	close(map_fd);
	322	goto err_close_map_fds;
	323	}
	324	if (strcmp(map_info.name, "values") == 0) {
	325	*values_map_fd = map_fd;
	326	continue;
	327	}
	328	if (strcmp(map_info.name, "allowed_ports") == 0) {
	329	*ports_map_fd = map_fd;
	330	continue;
	331	}
	332	close(map_fd);
	333	}
	334
	335	if (values_map_fd != -1 && ports_map_fd != -1) {
	336	err = 0;
	337	goto out;
	338	}
	339
	340	err = -ENOENT;
	341
	342	err_close_map_fds:
	343	if (*values_map_fd != -1)
	344	close(*values_map_fd);
	345	if (*ports_map_fd != -1)
	346	close(*ports_map_fd);
	347	*values_map_fd = -1;
	348	*ports_map_fd = -1;
	349
	350	out:
	351	close(prog_fd);
	352	return err;
	353	}
	354
	355	int main(int argc, char *argv[])
	356	{
	357	int values_map_fd, ports_map_fd;
	358	__u64 tcpipopts;
	359	bool firstiter;
	360	__u64 prevcnt;
	361	__u32 prog_id;
	362	char *ports;
	363	bool single;
	364	int err = 0;
	365	bool tc;
	366
	367	parse_options(argc, argv, &ifindex, &prog_id, &tcpipopts, &ports,
	368	&single, &tc);
	369
	370	if (prog_id == 0) {
	371	if (!tc) {
	372	err = bpf_xdp_query_id(ifindex, 0, &prog_id);
	373	if (err < 0) {
	374	fprintf(stderr, "Error: bpf_get_link_xdp_id: %s\n",
	375	strerror(-err));
	376	goto out;
	377	}
	378	}
	379	if (prog_id == 0) {
	380	err = syncookie_attach(argv[0], ifindex, tc);
	381	if (err < 0)
	382	goto out;
	383	prog_id = attached_prog_id;
	384	}
	385	}
	386
	387	err = syncookie_open_bpf_maps(prog_id, &values_map_fd, &ports_map_fd);
	388	if (err < 0)
	389	goto out;
	390
	391	if (ports) {
	392	__u16 port_last = 0;
	393	__u32 port_idx = 0;
	394	char *p = ports;
	395
	396	fprintf(stderr, "Replacing allowed ports\n");
	397
	398	while (p && *p != '\0') {
	399	char *token = strsep(&p, ",");
	400	__u16 port;
	401
	402	port = parse_arg_ul(argv[0], token, UINT16_MAX);
	403	err = bpf_map_update_elem(ports_map_fd, &port_idx, &port, BPF_ANY);
	404	if (err != 0) {
	405	fprintf(stderr, "Error: bpf_map_update_elem: %s\n", strerror(-err));
	406	fprintf(stderr, "Failed to add port %u (index %u)\n",
	407	port, port_idx);
	408	goto out_close_maps;
	409	}
	410	fprintf(stderr, "Added port %u\n", port);
	411	port_idx++;
	412	}
	413	err = bpf_map_update_elem(ports_map_fd, &port_idx, &port_last, BPF_ANY);
	414	if (err != 0) {
	415	fprintf(stderr, "Error: bpf_map_update_elem: %s\n", strerror(-err));
	416	fprintf(stderr, "Failed to add the terminator value 0 (index %u)\n",
	417	port_idx);
	418	goto out_close_maps;
	419	}
	420	}
	421
	422	if (tcpipopts) {
	423	__u32 key = 0;
	424
	425	fprintf(stderr, "Replacing TCP/IP options\n");
	426
	427	err = bpf_map_update_elem(values_map_fd, &key, &tcpipopts, BPF_ANY);
	428	if (err != 0) {
	429	fprintf(stderr, "Error: bpf_map_update_elem: %s\n", strerror(-err));
	430	goto out_close_maps;
	431	}
	432	}
	433
	434	if ((ports \|\| tcpipopts) && attached_prog_id == 0 && !single)
	435	goto out_close_maps;
	436
	437	prevcnt = 0;
	438	firstiter = true;
	439	while (true) {
	440	__u32 key = 1;
	441	__u64 value;
	442
	443	err = bpf_map_lookup_elem(values_map_fd, &key, &value);
	444	if (err != 0) {
	445	fprintf(stderr, "Error: bpf_map_lookup_elem: %s\n", strerror(-err));
	446	goto out_close_maps;
	447	}
	448	if (firstiter) {
	449	prevcnt = value;
	450	firstiter = false;
	451	}
	452	if (single) {
	453	printf("Total SYNACKs generated: %llu\n", value);
	454	break;
	455	}
	456	printf("SYNACKs generated: %llu (total %llu)\n", value - prevcnt, value);
	457	prevcnt = value;
	458	sleep(1);
	459	}
	460
	461	out_close_maps:
	462	close(values_map_fd);
	463	close(ports_map_fd);
	464	out:
	465	return err == 0 ? 0 : 1;
	466	}

Merge https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code

@@ -646,7 +646,7 @@ static void test_tcp_clear_dtime(struct test_tc_dtime *skel) · 645 unmodified lines
646	__u32 *errs = skel->bss->errs[t];	646	__u32 *errs = skel->bss->errs[t];
647		647
648	skel->bss->test = t;	648	skel->bss->test = t;
649	test_inet_dtime(AF_INET6, SOCK_STREAM, IP6_DST, 0);	649	test_inet_dtime(AF_INET6, SOCK_STREAM, IP6_DST, 50000 + t);
650		650
651	ASSERT_EQ(dtimes[INGRESS_FWDNS_P100], 0,	651	ASSERT_EQ(dtimes[INGRESS_FWDNS_P100], 0,
652	dtime_cnt_str(t, INGRESS_FWDNS_P100));	652	dtime_cnt_str(t, INGRESS_FWDNS_P100));
@@ -683,7 +683,7 @@ static void test_tcp_dtime(struct test_tc_dtime *skel, int family, bool bpf_fwd) · 30 unmodified lines
683	errs = skel->bss->errs[t];	683	errs = skel->bss->errs[t];
684		684
685	skel->bss->test = t;	685	skel->bss->test = t;
686	test_inet_dtime(family, SOCK_STREAM, addr, 0);	686	test_inet_dtime(family, SOCK_STREAM, addr, 50000 + t);
687		687
688	/* fwdns_prio100 prog does not read delivery_time_type, so	688	/* fwdns_prio100 prog does not read delivery_time_type, so
689	* kernel puts the (rcv) timetamp in __sk_buff->tstamp	689	* kernel puts the (rcv) timetamp in __sk_buff->tstamp
@@ -715,13 +715,13 @@ static void test_udp_dtime(struct test_tc_dtime *skel, int family, bool bpf_fwd) · 25 unmodified lines
715	errs = skel->bss->errs[t];	715	errs = skel->bss->errs[t];
716		716
717	skel->bss->test = t;	717	skel->bss->test = t;
718	test_inet_dtime(family, SOCK_DGRAM, addr, 0);	718	test_inet_dtime(family, SOCK_DGRAM, addr, 50000 + t);
719		719
720	ASSERT_EQ(dtimes[INGRESS_FWDNS_P100], 0,	720	ASSERT_EQ(dtimes[INGRESS_FWDNS_P100], 0,
721	dtime_cnt_str(t, INGRESS_FWDNS_P100));	721	dtime_cnt_str(t, INGRESS_FWDNS_P100));
722	/* non mono delivery time is not forwarded */	722	/* non mono delivery time is not forwarded */
723	ASSERT_EQ(dtimes[INGRESS_FWDNS_P101], 0,	723	ASSERT_EQ(dtimes[INGRESS_FWDNS_P101], 0,
724	dtime_cnt_str(t, INGRESS_FWDNS_P100));	724	dtime_cnt_str(t, INGRESS_FWDNS_P101));
725	for (i = EGRESS_FWDNS_P100; i < SET_DTIME; i++)	725	for (i = EGRESS_FWDNS_P100; i < SET_DTIME; i++)
726	ASSERT_GT(dtimes[i], 0, dtime_cnt_str(t, i));	726	ASSERT_GT(dtimes[i], 0, dtime_cnt_str(t, i));
727		727