lael-labs/linux

arch/x86/net/bpf_jit_comp.c+8−10

 	return proglen;
 }
-static void save_regs(struct btf_func_model *m, u8 **prog, int nr_args,
+static void save_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
 		      int stack_size)
 {
 	int i;
 			 -(stack_size - i * 8));
 }
-static void restore_regs(struct btf_func_model *m, u8 **prog, int nr_args,
+static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
 			 int stack_size)
 {
 	int i;
 			 -(stack_size - i * 8));
 }
-static int invoke_bpf(struct btf_func_model *m, u8 **pprog,
+static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
 		      struct bpf_prog **progs, int prog_cnt, int stack_size)
 {
 	u8 *prog = *pprog;
  * add rsp, 8                      // skip eth_type_trans's frame
  * ret                             // return to its caller
  */
-int arch_prepare_bpf_trampoline(void *image, struct btf_func_model *m, u32 flags,
+int arch_prepare_bpf_trampoline(void *image, void *image_end,
+				const struct btf_func_model *m, u32 flags,
 				struct bpf_prog **fentry_progs, int fentry_cnt,
 				struct bpf_prog **fexit_progs, int fexit_cnt,
 				void *orig_call)
 		/* skip our return address and return to parent */
 		EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */
 	EMIT1(0xC3); /* ret */
-	/* One half of the page has active running trampoline.
+	/* Make sure the trampoline generation logic doesn't overflow */
-	 * Another half is an area for next trampoline.
+	if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY))
-	 * Make sure the trampoline generation logic doesn't overflow.
-	 */
-	if (WARN_ON_ONCE(prog - (u8 *)image > PAGE_SIZE / 2 - BPF_INSN_SAFETY))
 		return -EFAULT;
-	return 0;
+	return prog - (u8 *)image;
 }
 static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)

include/linux/bpf.h+77−2

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	#include <linux/u64_stats_sync.h>	17	#include <linux/u64_stats_sync.h>
18	#include <linux/refcount.h>	18	#include <linux/refcount.h>
19	#include <linux/mutex.h>	19	#include <linux/mutex.h>
		20	#include <linux/module.h>
20		21
21	struct bpf_verifier_env;	22	struct bpf_verifier_env;
22	struct bpf_verifier_log;	23	struct bpf_verifier_log;
@@ -106,6 +107,7 @@ struct bpf_map { · 83 unmodified lines
106	struct btf *btf;	107	struct btf *btf;
107	struct bpf_map_memory memory;	108	struct bpf_map_memory memory;
108	char name[BPF_OBJ_NAME_LEN];	109	char name[BPF_OBJ_NAME_LEN];
		110	u32 btf_vmlinux_value_type_id;
109	bool unpriv_array;	111	bool unpriv_array;
110	bool frozen; /* write-once; write-protected by freeze_mutex */	112	bool frozen; /* write-once; write-protected by freeze_mutex */
111	/* 22 bytes hole */	113	/* 22 bytes hole */
@@ -183,7 +185,8 @@ static inline bool bpf_map_offload_neutral(const struct bpf_map *map) · 71 unmodified lines
183		185
184	static inline bool bpf_map_support_seq_show(const struct bpf_map *map)	186	static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
185	{	187	{
186	return map->btf && map->ops->map_seq_show_elem;	188	return (map->btf_value_type_id \|\| map->btf_vmlinux_value_type_id) &&
		189	map->ops->map_seq_show_elem;
187	}	190	}
188		191
189	int map_check_no_btf(const struct bpf_map *map,	192	int map_check_no_btf(const struct bpf_map *map,
@@ -349,6 +352,10 @@ struct bpf_verifier_ops { · 159 unmodified lines
349	const struct bpf_insn *src,	352	const struct bpf_insn *src,
350	struct bpf_insn *dst,	353	struct bpf_insn *dst,
351	struct bpf_prog prog, u32 target_size);	354	struct bpf_prog prog, u32 target_size);
		355	int (btf_struct_access)(struct bpf_verifier_log log,
		356	const struct btf_type *t, int off, int size,
		357	enum bpf_access_type atype,
		358	u32 *next_btf_id);
352	};	359	};
353		360
354	struct bpf_prog_offload_ops {	361	struct bpf_prog_offload_ops {
@@ -437,7 +444,8 @@ struct btf_func_model { · 82 unmodified lines
437	* fentry = a set of program to run before calling original function	444	* fentry = a set of program to run before calling original function
438	* fexit = a set of program to run after original function	445	* fexit = a set of program to run after original function
439	*/	446	*/
440	int arch_prepare_bpf_trampoline(void image, struct btf_func_model m, u32 flags,	447	int arch_prepare_bpf_trampoline(void image, void image_end,
		448	const struct btf_func_model *m, u32 flags,
441	struct bpf_prog **fentry_progs, int fentry_cnt,	449	struct bpf_prog **fentry_progs, int fentry_cnt,
442	struct bpf_prog **fexit_progs, int fexit_cnt,	450	struct bpf_prog **fexit_progs, int fexit_cnt,
443	void *orig_call);	451	void *orig_call);
@@ -668,6 +676,73 @@ struct bpf_array_aux { · 224 unmodified lines
668	struct work_struct work;	676	struct work_struct work;
669	};	677	};
670		678
		679	struct bpf_struct_ops_value;
		680	struct btf_type;
		681	struct btf_member;
		682
		683	#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
		684	struct bpf_struct_ops {
		685	const struct bpf_verifier_ops *verifier_ops;
		686	int (init)(struct btf btf);
		687	int (check_member)(const struct btf_type t,
		688	const struct btf_member *member);
		689	int (init_member)(const struct btf_type t,
		690	const struct btf_member *member,
		691	void kdata, const void udata);
		692	int (reg)(void kdata);
		693	void (unreg)(void kdata);
		694	const struct btf_type *type;
		695	const struct btf_type *value_type;
		696	const char *name;
		697	struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
		698	u32 type_id;
		699	u32 value_id;
		700	};
		701
		702	#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
		703	#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
		704	const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
		705	void bpf_struct_ops_init(struct btf *btf);
		706	bool bpf_struct_ops_get(const void *kdata);
		707	void bpf_struct_ops_put(const void *kdata);
		708	int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map map, void key,
		709	void *value);
		710	static inline bool bpf_try_module_get(const void data, struct module owner)
		711	{
		712	if (owner == BPF_MODULE_OWNER)
		713	return bpf_struct_ops_get(data);
		714	else
		715	return try_module_get(owner);
		716	}
		717	static inline void bpf_module_put(const void data, struct module owner)
		718	{
		719	if (owner == BPF_MODULE_OWNER)
		720	bpf_struct_ops_put(data);
		721	else
		722	module_put(owner);
		723	}
		724	#else
		725	static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
		726	{
		727	return NULL;
		728	}
		729	static inline void bpf_struct_ops_init(struct btf *btf) { }
		730	static inline bool bpf_try_module_get(const void data, struct module owner)
		731	{
		732	return try_module_get(owner);
		733	}
		734	static inline void bpf_module_put(const void data, struct module owner)
		735	{
		736	module_put(owner);
		737	}
		738	static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
		739	void *key,
		740	void *value)
		741	{
		742	return -EINVAL;
		743	}
		744	#endif
		745
671	struct bpf_array {	746	struct bpf_array {
672	struct bpf_map map;	747	struct bpf_map map;
673	u32 elem_size;	748	u32 elem_size;

include/linux/bpf_types.h+7−0

 BPF_PROG_TYPE(BPF_PROG_TYPE_SK_REUSEPORT, sk_reuseport,
 	      struct sk_reuseport_md, struct sk_reuseport_kern)
 #endif
+#if defined(CONFIG_BPF_JIT)
+BPF_PROG_TYPE(BPF_PROG_TYPE_STRUCT_OPS, bpf_struct_ops,
+	      void *, void *)
+#endif
 BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops)
 BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops)
 #endif
 BPF_MAP_TYPE(BPF_MAP_TYPE_QUEUE, queue_map_ops)
 BPF_MAP_TYPE(BPF_MAP_TYPE_STACK, stack_map_ops)
+#if defined(CONFIG_BPF_JIT)
+BPF_MAP_TYPE(BPF_MAP_TYPE_STRUCT_OPS, bpf_struct_ops_map_ops)
+#endif

include/linux/btf.h+47−0

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	#include <linux/types.h>	7	#include <linux/types.h>
8	#include <uapi/linux/btf.h>	8	#include <uapi/linux/btf.h>
9		9
		10	#define BTF_TYPE_EMIT(type) ((void)(type *)0)
		11
10	struct btf;	12	struct btf;
11	struct btf_member;	13	struct btf_member;
12	struct btf_type;	14	struct btf_type;
@@ -53,6 +55,22 @@ bool btf_member_is_reg_int(const struct btf btf, const struct btf_type s, · 40 unmodified lines
53	u32 expected_offset, u32 expected_size);	55	u32 expected_offset, u32 expected_size);
54	int btf_find_spin_lock(const struct btf btf, const struct btf_type t);	56	int btf_find_spin_lock(const struct btf btf, const struct btf_type t);
55	bool btf_type_is_void(const struct btf_type *t);	57	bool btf_type_is_void(const struct btf_type *t);
		58	s32 btf_find_by_name_kind(const struct btf btf, const char name, u8 kind);
		59	const struct btf_type btf_type_skip_modifiers(const struct btf btf,
		60	u32 id, u32 *res_id);
		61	const struct btf_type btf_type_resolve_ptr(const struct btf btf,
		62	u32 id, u32 *res_id);
		63	const struct btf_type btf_type_resolve_func_ptr(const struct btf btf,
		64	u32 id, u32 *res_id);
		65	const struct btf_type *
		66	btf_resolve_size(const struct btf btf, const struct btf_type type,
		67	u32 type_size, const struct btf_type *elem_type,
		68	u32 *total_nelems);
		69
		70	#define for_each_member(i, struct_type, member) \
		71	for (i = 0, member = btf_type_member(struct_type); \
		72	i < btf_type_vlen(struct_type); \
		73	i++, member++)
56		74
57	static inline bool btf_type_is_ptr(const struct btf_type *t)	75	static inline bool btf_type_is_ptr(const struct btf_type *t)
58	{	76	{
@@ -84,6 +102,35 @@ static inline bool btf_type_is_func_proto(const struct btf_type *t) · 25 unmodified lines
84	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;	102	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
85	}	103	}
86		104
		105	static inline u16 btf_type_vlen(const struct btf_type *t)
		106	{
		107	return BTF_INFO_VLEN(t->info);
		108	}
		109
		110	static inline bool btf_type_kflag(const struct btf_type *t)
		111	{
		112	return BTF_INFO_KFLAG(t->info);
		113	}
		114
		115	static inline u32 btf_member_bit_offset(const struct btf_type *struct_type,
		116	const struct btf_member *member)
		117	{
		118	return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
		119	: member->offset;
		120	}
		121
		122	static inline u32 btf_member_bitfield_size(const struct btf_type *struct_type,
		123	const struct btf_member *member)
		124	{
		125	return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
		126	: 0;
		127	}
		128
		129	static inline const struct btf_member btf_type_member(const struct btf_type t)
		130	{
		131	return (const struct btf_member *)(t + 1);
		132	}
		133
87	#ifdef CONFIG_BPF_SYSCALL	134	#ifdef CONFIG_BPF_SYSCALL
88	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);	135	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);
89	const char btf_name_by_offset(const struct btf btf, u32 offset);	136	const char btf_name_by_offset(const struct btf btf, u32 offset);

include/linux/filter.h+2−0

@@ -843,6 +843,8 @@ int bpf_prog_create(struct bpf_prog *pfp, struct sock_fprog_kern fprog); · 842 unmodified lines
843	int bpf_prog_create_from_user(struct bpf_prog *pfp, struct sock_fprog fprog,	843	int bpf_prog_create_from_user(struct bpf_prog *pfp, struct sock_fprog fprog,
844	bpf_aux_classic_check_t trans, bool save_orig);	844	bpf_aux_classic_check_t trans, bool save_orig);
845	void bpf_prog_destroy(struct bpf_prog *fp);	845	void bpf_prog_destroy(struct bpf_prog *fp);
		846	const struct bpf_func_proto *
		847	bpf_base_func_proto(enum bpf_func_id func_id);
846		848
847	int sk_attach_filter(struct sock_fprog fprog, struct sock sk);	849	int sk_attach_filter(struct sock_fprog fprog, struct sock sk);
848	int sk_attach_bpf(u32 ufd, struct sock *sk);	850	int sk_attach_bpf(u32 ufd, struct sock *sk);

include/net/tcp.h+2−0

 #define TCP_CONG_NON_RESTRICTED 0x1
 /* Requires ECN/ECT set on all packets */
 #define TCP_CONG_NEEDS_ECN	0x2
+#define TCP_CONG_MASK	(TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
 union tcp_cc_info;
 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
 extern struct tcp_congestion_ops tcp_reno;
+struct tcp_congestion_ops *tcp_ca_find(const char *name);
 struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
 u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
 #ifdef CONFIG_INET

include/uapi/linux/bpf.h+17−2

 	BPF_MAP_TYPE_STACK,
 	BPF_MAP_TYPE_SK_STORAGE,
 	BPF_MAP_TYPE_DEVMAP_HASH,
+	BPF_MAP_TYPE_STRUCT_OPS,
 };
 /* Note that tracing related programs such as
 	BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE,
 	BPF_PROG_TYPE_CGROUP_SOCKOPT,
 	BPF_PROG_TYPE_TRACING,
+	BPF_PROG_TYPE_STRUCT_OPS,
 };
 enum bpf_attach_type {
 		__u32	btf_fd;		/* fd pointing to a BTF type data */
 		__u32	btf_key_type_id;	/* BTF type_id of the key */
 		__u32	btf_value_type_id;	/* BTF type_id of the value */
+		__u32	btf_vmlinux_value_type_id;/* BTF type_id of a kernel-
+						   * struct stored as the
+						   * map value
+						   */
 	};
 	struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */
  * 	Return
  * 		On success, the strictly positive length of the string,	including
  * 		the trailing NUL character. On error, a negative value.
+ *
+ * int bpf_tcp_send_ack(void *tp, u32 rcv_nxt)
+ *	Description
+ *		Send out a tcp-ack. *tp* is the in-kernel struct tcp_sock.
+ *		*rcv_nxt* is the ack_seq to be sent out.
+ *	Return
+ *		0 on success, or a negative error in case of failure.
+ *
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
 	FN(probe_read_user),		\
 	FN(probe_read_kernel),		\
 	FN(probe_read_user_str),	\
-	FN(probe_read_kernel_str),
+	FN(probe_read_kernel_str),	\
+	FN(tcp_send_ack),
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
  * function eBPF program intends to call
 	__u32 map_flags;
 	char  name[BPF_OBJ_NAME_LEN];
 	__u32 ifindex;
-	__u32 :32;
+	__u32 btf_vmlinux_value_type_id;
 	__u64 netns_dev;
 	__u64 netns_ino;
 	__u32 btf_id;

kernel/bpf/Makefile+3−0

@@ -27,3 +27,6 @@ endif · 26 unmodified lines
27	ifeq ($(CONFIG_SYSFS),y)	27	ifeq ($(CONFIG_SYSFS),y)
28	obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o	28	obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o
29	endif	29	endif
		30	ifeq ($(CONFIG_BPF_JIT),y)
		31	obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o
		32	endif

kernel/bpf/bpf_struct_ops.c+628−0

@@ -0,0 +1,628 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (c) 2019 Facebook */
	3
	4	#include <linux/bpf.h>
	5	#include <linux/bpf_verifier.h>
	6	#include <linux/btf.h>
	7	#include <linux/filter.h>
	8	#include <linux/slab.h>
	9	#include <linux/numa.h>
	10	#include <linux/seq_file.h>
	11	#include <linux/refcount.h>
	12	#include <linux/mutex.h>
	13
	14	enum bpf_struct_ops_state {
	15	BPF_STRUCT_OPS_STATE_INIT,
	16	BPF_STRUCT_OPS_STATE_INUSE,
	17	BPF_STRUCT_OPS_STATE_TOBEFREE,
	18	};
	19
	20	#define BPF_STRUCT_OPS_COMMON_VALUE \
	21	refcount_t refcnt; \
	22	enum bpf_struct_ops_state state
	23
	24	struct bpf_struct_ops_value {
	25	BPF_STRUCT_OPS_COMMON_VALUE;
	26	char data[0] ____cacheline_aligned_in_smp;
	27	};
	28
	29	struct bpf_struct_ops_map {
	30	struct bpf_map map;
	31	const struct bpf_struct_ops *st_ops;
	32	/* protect map_update */
	33	struct mutex lock;
	34	/* progs has all the bpf_prog that is populated
	35	* to the func ptr of the kernel's struct
	36	* (in kvalue.data).
	37	*/
	38	struct bpf_prog **progs;
	39	/* image is a page that has all the trampolines
	40	* that stores the func args before calling the bpf_prog.
	41	* A PAGE_SIZE "image" is enough to store all trampoline for
	42	* "progs[]".
	43	*/
	44	void *image;
	45	/* uvalue->data stores the kernel struct
	46	* (e.g. tcp_congestion_ops) that is more useful
	47	* to userspace than the kvalue. For example,
	48	* the bpf_prog's id is stored instead of the kernel
	49	* address of a func ptr.
	50	*/
	51	struct bpf_struct_ops_value *uvalue;
	52	/* kvalue.data stores the actual kernel's struct
	53	* (e.g. tcp_congestion_ops) that will be
	54	* registered to the kernel subsystem.
	55	*/
	56	struct bpf_struct_ops_value kvalue;
	57	};
	58
	59	#define VALUE_PREFIX "bpf_struct_ops_"
	60	#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
	61
	62	/* bpf_struct_ops_##_name (e.g. bpf_struct_ops_tcp_congestion_ops) is
	63	* the map's value exposed to the userspace and its btf-type-id is
	64	* stored at the map->btf_vmlinux_value_type_id.
	65	*
	66	*/
	67	#define BPF_STRUCT_OPS_TYPE(_name) \
	68	extern struct bpf_struct_ops bpf_##_name; \
	69	\
	70	struct bpf_struct_ops_##_name { \
	71	BPF_STRUCT_OPS_COMMON_VALUE; \
	72	struct _name data ____cacheline_aligned_in_smp; \
	73	};
	74	#include "bpf_struct_ops_types.h"
	75	#undef BPF_STRUCT_OPS_TYPE
	76
	77	enum {
	78	#define BPF_STRUCT_OPS_TYPE(_name) BPF_STRUCT_OPS_TYPE_##_name,
	79	#include "bpf_struct_ops_types.h"
	80	#undef BPF_STRUCT_OPS_TYPE
	81	__NR_BPF_STRUCT_OPS_TYPE,
	82	};
	83
	84	static struct bpf_struct_ops * const bpf_struct_ops[] = {
	85	#define BPF_STRUCT_OPS_TYPE(_name) \
	86	[BPF_STRUCT_OPS_TYPE_##_name] = &bpf_##_name,
	87	#include "bpf_struct_ops_types.h"
	88	#undef BPF_STRUCT_OPS_TYPE
	89	};
	90
	91	const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
	92	};
	93
	94	const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
	95	};
	96
	97	static const struct btf_type *module_type;
	98
	99	void bpf_struct_ops_init(struct btf *btf)
	100	{
	101	s32 type_id, value_id, module_id;
	102	const struct btf_member *member;
	103	struct bpf_struct_ops *st_ops;
	104	struct bpf_verifier_log log = {};
	105	const struct btf_type *t;
	106	char value_name[128];
	107	const char *mname;
	108	u32 i, j;
	109
	110	/* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */
	111	#define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name);
	112	#include "bpf_struct_ops_types.h"
	113	#undef BPF_STRUCT_OPS_TYPE
	114
	115	module_id = btf_find_by_name_kind(btf, "module", BTF_KIND_STRUCT);
	116	if (module_id < 0) {
	117	pr_warn("Cannot find struct module in btf_vmlinux\n");
	118	return;
	119	}
	120	module_type = btf_type_by_id(btf, module_id);
	121
	122	for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
	123	st_ops = bpf_struct_ops[i];
	124
	125	if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
	126	sizeof(value_name)) {
	127	pr_warn("struct_ops name %s is too long\n",
	128	st_ops->name);
	129	continue;
	130	}
	131	sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
	132
	133	value_id = btf_find_by_name_kind(btf, value_name,
	134	BTF_KIND_STRUCT);
	135	if (value_id < 0) {
	136	pr_warn("Cannot find struct %s in btf_vmlinux\n",
	137	value_name);
	138	continue;
	139	}
	140
	141	type_id = btf_find_by_name_kind(btf, st_ops->name,
	142	BTF_KIND_STRUCT);
	143	if (type_id < 0) {
	144	pr_warn("Cannot find struct %s in btf_vmlinux\n",
	145	st_ops->name);
	146	continue;
	147	}
	148	t = btf_type_by_id(btf, type_id);
	149	if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
	150	pr_warn("Cannot support #%u members in struct %s\n",
	151	btf_type_vlen(t), st_ops->name);
	152	continue;
	153	}
	154
	155	for_each_member(j, t, member) {
	156	const struct btf_type *func_proto;
	157
	158	mname = btf_name_by_offset(btf, member->name_off);
	159	if (!*mname) {
	160	pr_warn("anon member in struct %s is not supported\n",
	161	st_ops->name);
	162	break;
	163	}
	164
	165	if (btf_member_bitfield_size(t, member)) {
	166	pr_warn("bit field member %s in struct %s is not supported\n",
	167	mname, st_ops->name);
	168	break;
	169	}
	170
	171	func_proto = btf_type_resolve_func_ptr(btf,
	172	member->type,
	173	NULL);
	174	if (func_proto &&
	175	btf_distill_func_proto(&log, btf,
	176	func_proto, mname,
	177	&st_ops->func_models[j])) {
	178	pr_warn("Error in parsing func ptr %s in struct %s\n",
	179	mname, st_ops->name);
	180	break;
	181	}
	182	}
	183
	184	if (j == btf_type_vlen(t)) {
	185	if (st_ops->init(btf)) {
	186	pr_warn("Error in init bpf_struct_ops %s\n",
	187	st_ops->name);
	188	} else {
	189	st_ops->type_id = type_id;
	190	st_ops->type = t;
	191	st_ops->value_id = value_id;
	192	st_ops->value_type = btf_type_by_id(btf,
	193	value_id);
	194	}
	195	}
	196	}
	197	}
	198
	199	extern struct btf *btf_vmlinux;
	200
	201	static const struct bpf_struct_ops *
	202	bpf_struct_ops_find_value(u32 value_id)
	203	{
	204	unsigned int i;
	205
	206	if (!value_id \|\| !btf_vmlinux)
	207	return NULL;
	208
	209	for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
	210	if (bpf_struct_ops[i]->value_id == value_id)
	211	return bpf_struct_ops[i];
	212	}
	213
	214	return NULL;
	215	}
	216
	217	const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
	218	{
	219	unsigned int i;
	220
	221	if (!type_id \|\| !btf_vmlinux)
	222	return NULL;
	223
	224	for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
	225	if (bpf_struct_ops[i]->type_id == type_id)
	226	return bpf_struct_ops[i];
	227	}
	228
	229	return NULL;
	230	}
	231
	232	static int bpf_struct_ops_map_get_next_key(struct bpf_map map, void key,
	233	void *next_key)
	234	{
	235	if (key && (u32 )key == 0)
	236	return -ENOENT;
	237
	238	(u32 )next_key = 0;
	239	return 0;
	240	}
	241
	242	int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map map, void key,
	243	void *value)
	244	{
	245	struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;
	246	struct bpf_struct_ops_value uvalue, kvalue;
	247	enum bpf_struct_ops_state state;
	248
	249	if (unlikely((u32 )key != 0))
	250	return -ENOENT;
	251
	252	kvalue = &st_map->kvalue;
	253	/* Pair with smp_store_release() during map_update */
	254	state = smp_load_acquire(&kvalue->state);
	255	if (state == BPF_STRUCT_OPS_STATE_INIT) {
	256	memset(value, 0, map->value_size);
	257	return 0;
	258	}
	259
	260	/* No lock is needed. state and refcnt do not need
	261	* to be updated together under atomic context.
	262	*/
	263	uvalue = (struct bpf_struct_ops_value *)value;
	264	memcpy(uvalue, st_map->uvalue, map->value_size);
	265	uvalue->state = state;
	266	refcount_set(&uvalue->refcnt, refcount_read(&kvalue->refcnt));
	267
	268	return 0;
	269	}
	270
	271	static void bpf_struct_ops_map_lookup_elem(struct bpf_map map, void *key)
	272	{
	273	return ERR_PTR(-EINVAL);
	274	}
	275
	276	static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
	277	{
	278	const struct btf_type *t = st_map->st_ops->type;
	279	u32 i;
	280
	281	for (i = 0; i < btf_type_vlen(t); i++) {
	282	if (st_map->progs[i]) {
	283	bpf_prog_put(st_map->progs[i]);
	284	st_map->progs[i] = NULL;
	285	}
	286	}
	287	}
	288
	289	static int check_zero_holes(const struct btf_type t, void data)
	290	{
	291	const struct btf_member *member;
	292	u32 i, moff, msize, prev_mend = 0;
	293	const struct btf_type *mtype;
	294
	295	for_each_member(i, t, member) {
	296	moff = btf_member_bit_offset(t, member) / 8;
	297	if (moff > prev_mend &&
	298	memchr_inv(data + prev_mend, 0, moff - prev_mend))
	299	return -EINVAL;
	300
	301	mtype = btf_type_by_id(btf_vmlinux, member->type);
	302	mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
	303	NULL, NULL);
	304	if (IS_ERR(mtype))
	305	return PTR_ERR(mtype);
	306	prev_mend = moff + msize;
	307	}
	308
	309	if (t->size > prev_mend &&
	310	memchr_inv(data + prev_mend, 0, t->size - prev_mend))
	311	return -EINVAL;
	312
	313	return 0;
	314	}
	315
	316	static int bpf_struct_ops_map_update_elem(struct bpf_map map, void key,
	317	void *value, u64 flags)
	318	{
	319	struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;
	320	const struct bpf_struct_ops *st_ops = st_map->st_ops;
	321	struct bpf_struct_ops_value uvalue, kvalue;
	322	const struct btf_member *member;
	323	const struct btf_type *t = st_ops->type;
	324	void udata, kdata;
	325	int prog_fd, err = 0;
	326	void *image;
	327	u32 i;
	328
	329	if (flags)
	330	return -EINVAL;
	331
	332	if ((u32 )key != 0)
	333	return -E2BIG;
	334
	335	err = check_zero_holes(st_ops->value_type, value);
	336	if (err)
	337	return err;
	338
	339	uvalue = (struct bpf_struct_ops_value *)value;
	340	err = check_zero_holes(t, uvalue->data);
	341	if (err)
	342	return err;
	343
	344	if (uvalue->state \|\| refcount_read(&uvalue->refcnt))
	345	return -EINVAL;
	346
	347	uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
	348	kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
	349
	350	mutex_lock(&st_map->lock);
	351
	352	if (kvalue->state != BPF_STRUCT_OPS_STATE_INIT) {
	353	err = -EBUSY;
	354	goto unlock;
	355	}
	356
	357	memcpy(uvalue, value, map->value_size);
	358
	359	udata = &uvalue->data;
	360	kdata = &kvalue->data;
	361	image = st_map->image;
	362
	363	for_each_member(i, t, member) {
	364	const struct btf_type mtype, ptype;
	365	struct bpf_prog *prog;
	366	u32 moff;
	367
	368	moff = btf_member_bit_offset(t, member) / 8;
	369	ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL);
	370	if (ptype == module_type) {
	371	if ((void *)(udata + moff))
	372	goto reset_unlock;
	373	(void *)(kdata + moff) = BPF_MODULE_OWNER;
	374	continue;
	375	}
	376
	377	err = st_ops->init_member(t, member, kdata, udata);
	378	if (err < 0)
	379	goto reset_unlock;
	380
	381	/* The ->init_member() has handled this member */
	382	if (err > 0)
	383	continue;
	384
	385	/* If st_ops->init_member does not handle it,
	386	* we will only handle func ptrs and zero-ed members
	387	* here. Reject everything else.
	388	*/
	389
	390	/* All non func ptr member must be 0 */
	391	if (!ptype \|\| !btf_type_is_func_proto(ptype)) {
	392	u32 msize;
	393
	394	mtype = btf_type_by_id(btf_vmlinux, member->type);
	395	mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
	396	NULL, NULL);
	397	if (IS_ERR(mtype)) {
	398	err = PTR_ERR(mtype);
	399	goto reset_unlock;
	400	}
	401
	402	if (memchr_inv(udata + moff, 0, msize)) {
	403	err = -EINVAL;
	404	goto reset_unlock;
	405	}
	406
	407	continue;
	408	}
	409
	410	prog_fd = (int)((unsigned long )(udata + moff));
	411	/* Similar check as the attr->attach_prog_fd */
	412	if (!prog_fd)
	413	continue;
	414
	415	prog = bpf_prog_get(prog_fd);
	416	if (IS_ERR(prog)) {
	417	err = PTR_ERR(prog);
	418	goto reset_unlock;
	419	}
	420	st_map->progs[i] = prog;
	421
	422	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS \|\|
	423	prog->aux->attach_btf_id != st_ops->type_id \|\|
	424	prog->expected_attach_type != i) {
	425	err = -EINVAL;
	426	goto reset_unlock;
	427	}
	428
	429	err = arch_prepare_bpf_trampoline(image,
	430	st_map->image + PAGE_SIZE,
	431	&st_ops->func_models[i], 0,
	432	&prog, 1, NULL, 0, NULL);
	433	if (err < 0)
	434	goto reset_unlock;
	435
	436	(void *)(kdata + moff) = image;
	437	image += err;
	438
	439	/* put prog_id to udata */
	440	(unsigned long )(udata + moff) = prog->aux->id;
	441	}
	442
	443	refcount_set(&kvalue->refcnt, 1);
	444	bpf_map_inc(map);
	445
	446	set_memory_ro((long)st_map->image, 1);
	447	set_memory_x((long)st_map->image, 1);
	448	err = st_ops->reg(kdata);
	449	if (likely(!err)) {
	450	/* Pair with smp_load_acquire() during lookup_elem().
	451	* It ensures the above udata updates (e.g. prog->aux->id)
	452	* can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
	453	*/
	454	smp_store_release(&kvalue->state, BPF_STRUCT_OPS_STATE_INUSE);
	455	goto unlock;
	456	}
	457
	458	/* Error during st_ops->reg(). It is very unlikely since
	459	* the above init_member() should have caught it earlier
	460	* before reg(). The only possibility is if there was a race
	461	* in registering the struct_ops (under the same name) to
	462	* a sub-system through different struct_ops's maps.
	463	*/
	464	set_memory_nx((long)st_map->image, 1);
	465	set_memory_rw((long)st_map->image, 1);
	466	bpf_map_put(map);
	467
	468	reset_unlock:
	469	bpf_struct_ops_map_put_progs(st_map);
	470	memset(uvalue, 0, map->value_size);
	471	memset(kvalue, 0, map->value_size);
	472	unlock:
	473	mutex_unlock(&st_map->lock);
	474	return err;
	475	}
	476
	477	static int bpf_struct_ops_map_delete_elem(struct bpf_map map, void key)
	478	{
	479	enum bpf_struct_ops_state prev_state;
	480	struct bpf_struct_ops_map *st_map;
	481
	482	st_map = (struct bpf_struct_ops_map *)map;
	483	prev_state = cmpxchg(&st_map->kvalue.state,
	484	BPF_STRUCT_OPS_STATE_INUSE,
	485	BPF_STRUCT_OPS_STATE_TOBEFREE);
	486	if (prev_state == BPF_STRUCT_OPS_STATE_INUSE) {
	487	st_map->st_ops->unreg(&st_map->kvalue.data);
	488	if (refcount_dec_and_test(&st_map->kvalue.refcnt))
	489	bpf_map_put(map);
	490	}
	491
	492	return 0;
	493	}
	494
	495	static void bpf_struct_ops_map_seq_show_elem(struct bpf_map map, void key,
	496	struct seq_file *m)
	497	{
	498	void *value;
	499
	500	value = bpf_struct_ops_map_lookup_elem(map, key);
	501	if (!value)
	502	return;
	503
	504	btf_type_seq_show(btf_vmlinux, map->btf_vmlinux_value_type_id,
	505	value, m);
	506	seq_puts(m, "\n");
	507	}
	508
	509	static void bpf_struct_ops_map_free(struct bpf_map *map)
	510	{
	511	struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;
	512
	513	if (st_map->progs)
	514	bpf_struct_ops_map_put_progs(st_map);
	515	bpf_map_area_free(st_map->progs);
	516	bpf_jit_free_exec(st_map->image);
	517	bpf_map_area_free(st_map->uvalue);
	518	bpf_map_area_free(st_map);
	519	}
	520
	521	static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
	522	{
	523	if (attr->key_size != sizeof(unsigned int) \|\| attr->max_entries != 1 \|\|
	524	attr->map_flags \|\| !attr->btf_vmlinux_value_type_id)
	525	return -EINVAL;
	526	return 0;
	527	}
	528
	529	static struct bpf_map bpf_struct_ops_map_alloc(union bpf_attr attr)
	530	{
	531	const struct bpf_struct_ops *st_ops;
	532	size_t map_total_size, st_map_size;
	533	struct bpf_struct_ops_map *st_map;
	534	const struct btf_type t, vt;
	535	struct bpf_map_memory mem;
	536	struct bpf_map *map;
	537	int err;
	538
	539	if (!capable(CAP_SYS_ADMIN))
	540	return ERR_PTR(-EPERM);
	541
	542	st_ops = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id);
	543	if (!st_ops)
	544	return ERR_PTR(-ENOTSUPP);
	545
	546	vt = st_ops->value_type;
	547	if (attr->value_size != vt->size)
	548	return ERR_PTR(-EINVAL);
	549
	550	t = st_ops->type;
	551
	552	st_map_size = sizeof(*st_map) +
	553	/* kvalue stores the
	554	* struct bpf_struct_ops_tcp_congestions_ops
	555	*/
	556	(vt->size - sizeof(struct bpf_struct_ops_value));
	557	map_total_size = st_map_size +
	558	/* uvalue */
	559	sizeof(vt->size) +
	560	/* struct bpf_progs *progs /
	561	btf_type_vlen(t) * sizeof(struct bpf_prog *);
	562	err = bpf_map_charge_init(&mem, map_total_size);
	563	if (err < 0)
	564	return ERR_PTR(err);
	565
	566	st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
	567	if (!st_map) {
	568	bpf_map_charge_finish(&mem);
	569	return ERR_PTR(-ENOMEM);
	570	}
	571	st_map->st_ops = st_ops;
	572	map = &st_map->map;
	573
	574	st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
	575	st_map->progs =
	576	bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_prog *),
	577	NUMA_NO_NODE);
	578	st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);
	579	if (!st_map->uvalue \|\| !st_map->progs \|\| !st_map->image) {
	580	bpf_struct_ops_map_free(map);
	581	bpf_map_charge_finish(&mem);
	582	return ERR_PTR(-ENOMEM);
	583	}
	584
	585	mutex_init(&st_map->lock);
	586	set_vm_flush_reset_perms(st_map->image);
	587	bpf_map_init_from_attr(map, attr);
	588	bpf_map_charge_move(&map->memory, &mem);
	589
	590	return map;
	591	}
	592
	593	const struct bpf_map_ops bpf_struct_ops_map_ops = {
	594	.map_alloc_check = bpf_struct_ops_map_alloc_check,
	595	.map_alloc = bpf_struct_ops_map_alloc,
	596	.map_free = bpf_struct_ops_map_free,
	597	.map_get_next_key = bpf_struct_ops_map_get_next_key,
	598	.map_lookup_elem = bpf_struct_ops_map_lookup_elem,
	599	.map_delete_elem = bpf_struct_ops_map_delete_elem,
	600	.map_update_elem = bpf_struct_ops_map_update_elem,
	601	.map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
	602	};
	603
	604	/* "const void *" because some subsystem is
	605	* passing a const (e.g. const struct tcp_congestion_ops *)
	606	*/
	607	bool bpf_struct_ops_get(const void *kdata)
	608	{
	609	struct bpf_struct_ops_value *kvalue;
	610
	611	kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
	612
	613	return refcount_inc_not_zero(&kvalue->refcnt);
	614	}
	615
	616	void bpf_struct_ops_put(const void *kdata)
	617	{
	618	struct bpf_struct_ops_value *kvalue;
	619
	620	kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
	621	if (refcount_dec_and_test(&kvalue->refcnt)) {
	622	struct bpf_struct_ops_map *st_map;
	623
	624	st_map = container_of(kvalue, struct bpf_struct_ops_map,
	625	kvalue);
	626	bpf_map_put(&st_map->map);
	627	}
	628	}

kernel/bpf/bpf_struct_ops_types.h+9−0

@@ -0,0 +1,9 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	/* internal file - do not include directly */
	3
	4	#ifdef CONFIG_BPF_JIT
	5	#ifdef CONFIG_INET
	6	#include <net/tcp.h>
	7	BPF_STRUCT_OPS_TYPE(tcp_congestion_ops)
	8	#endif
	9	#endif

kernel/bpf/btf.c+116−47

  */
 #define BTF_MAX_SIZE (16 * 1024 * 1024)
-#define for_each_member(i, struct_type, member)			\
-	for (i = 0, member = btf_type_member(struct_type);	\
-	     i < btf_type_vlen(struct_type);			\
-	     i++, member++)
 #define for_each_member_from(i, from, struct_type, member)		\
 	for (i = from, member = btf_type_member(struct_type) + from;	\
 	     i < btf_type_vlen(struct_type);				\
 	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
 }
+s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind)
+{
+	const struct btf_type *t;
+	const char *tname;
+	u32 i;
+	for (i = 1; i <= btf->nr_types; i++) {
+		t = btf->types[i];
+		if (BTF_INFO_KIND(t->info) != kind)
+			continue;
+		tname = btf_name_by_offset(btf, t->name_off);
+		if (!strcmp(tname, name))
+			return i;
+	}
+	return -ENOENT;
+}
+const struct btf_type *btf_type_skip_modifiers(const struct btf *btf,
+					       u32 id, u32 *res_id)
+{
+	const struct btf_type *t = btf_type_by_id(btf, id);
+	while (btf_type_is_modifier(t)) {
+		id = t->type;
+		t = btf_type_by_id(btf, t->type);
+	}
+	if (res_id)
+		*res_id = id;
+	return t;
+}
+const struct btf_type *btf_type_resolve_ptr(const struct btf *btf,
+					    u32 id, u32 *res_id)
+{
+	const struct btf_type *t;
+	t = btf_type_skip_modifiers(btf, id, NULL);
+	if (!btf_type_is_ptr(t))
+		return NULL;
+	return btf_type_skip_modifiers(btf, t->type, res_id);
+}
+const struct btf_type *btf_type_resolve_func_ptr(const struct btf *btf,
+						 u32 id, u32 *res_id)
+{
+	const struct btf_type *ptype;
+	ptype = btf_type_resolve_ptr(btf, id, res_id);
+	if (ptype && btf_type_is_func_proto(ptype))
+		return ptype;
+	return NULL;
+}
 /* Types that act only as a source, not sink or intermediate
  * type when resolving.
  */
 		return "UNKN";
 }
-static u16 btf_type_vlen(const struct btf_type *t)
-{
-	return BTF_INFO_VLEN(t->info);
-}
-static bool btf_type_kflag(const struct btf_type *t)
-{
-	return BTF_INFO_KFLAG(t->info);
-}
-static u32 btf_member_bit_offset(const struct btf_type *struct_type,
-			     const struct btf_member *member)
-{
-	return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
-					   : member->offset;
-}
-static u32 btf_member_bitfield_size(const struct btf_type *struct_type,
-				    const struct btf_member *member)
-{
-	return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
-					   : 0;
-}
 static u32 btf_type_int(const struct btf_type *t)
 {
 	return *(u32 *)(t + 1);
 	return (const struct btf_array *)(t + 1);
 }
-static const struct btf_member *btf_type_member(const struct btf_type *t)
-{
-	return (const struct btf_member *)(t + 1);
-}
 static const struct btf_enum *btf_type_enum(const struct btf_type *t)
 {
 	return (const struct btf_enum *)(t + 1);
  * *elem_type: same as return type ("struct X")
  * *total_nelems: 1
  */
-static const struct btf_type *
+const struct btf_type *
 btf_resolve_size(const struct btf *btf, const struct btf_type *type,
 		 u32 *type_size, const struct btf_type **elem_type,
 		 u32 *total_nelems)
 		return ERR_PTR(-EINVAL);
 	*type_size = nelems * size;
-	*total_nelems = nelems;
+	if (total_nelems)
-	*elem_type = type;
+		*total_nelems = nelems;
+	if (elem_type)
+		*elem_type = type;
 	return array_type ? : type;
 }
 				  u32 type_id, void *data,
 				  u8 bits_offset, struct seq_file *m)
 {
-	t = btf_type_id_resolve(btf, &type_id);
+	if (btf->resolved_ids)
+		t = btf_type_id_resolve(btf, &type_id);
+	else
+		t = btf_type_skip_modifiers(btf, type_id, NULL);
 	btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
 }
 		goto errout;
 	}
+	bpf_struct_ops_init(btf);
 	btf_verifier_env_free(env);
 	refcount_set(&btf->refcnt, 1);
 	return btf;
 	/* skip modifiers */
 	while (btf_type_is_modifier(t))
 		t = btf_type_by_id(btf, t->type);
-	if (btf_type_is_int(t))
+	if (btf_type_is_int(t) || btf_type_is_enum(t))
 		/* accessing a scalar */
 		return true;
 	if (!btf_type_is_ptr(t)) {
 	/* this is a pointer to another type */
 	info->reg_type = PTR_TO_BTF_ID;
-	info->btf_id = t->type;
 	if (tgt_prog) {
 		ret = btf_translate_to_vmlinux(log, btf, t, tgt_prog->type);
 			return false;
 		}
 	}
+	info->btf_id = t->type;
 	t = btf_type_by_id(btf, t->type);
 	/* skip modifiers */
-	while (btf_type_is_modifier(t))
+	while (btf_type_is_modifier(t)) {
+		info->btf_id = t->type;
 		t = btf_type_by_id(btf, t->type);
+	}
 	if (!btf_type_is_struct(t)) {
 		bpf_log(log,
 			"func '%s' arg%d type %s is not a struct\n",
 again:
 	tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
 	if (!btf_type_is_struct(t)) {
-		bpf_log(log, "Type '%s' is not a struct", tname);
+		bpf_log(log, "Type '%s' is not a struct\n", tname);
 		return -EINVAL;
 	}
-	for_each_member(i, t, member) {
+	if (off + size > t->size) {
-		if (btf_member_bitfield_size(t, member))
+		bpf_log(log, "access beyond struct %s at off %u size %u\n",
-			/* bitfields are not supported yet */
+			tname, off, size);
-			continue;
+		return -EACCES;
+	}
+	for_each_member(i, t, member) {
 		/* offset of the field in bytes */
 		moff = btf_member_bit_offset(t, member) / 8;
 		if (off + size <= moff)
 			/* won't find anything, field is already too far */
 			break;
+		if (btf_member_bitfield_size(t, member)) {
+			u32 end_bit = btf_member_bit_offset(t, member) +
+				btf_member_bitfield_size(t, member);
+			/* off <= moff instead of off == moff because clang
+			 * does not generate a BTF member for anonymous
+			 * bitfield like the ":16" here:
+			 * struct {
+			 *	int :16;
+			 *	int x:8;
+			 * };
+			 */
+			if (off <= moff &&
+			    BITS_ROUNDUP_BYTES(end_bit) <= off + size)
+				return SCALAR_VALUE;
+			/* off may be accessing a following member
+			 *
+			 * or
+			 *
+			 * Doing partial access at either end of this
+			 * bitfield.  Continue on this case also to
+			 * treat it as not accessing this bitfield
+			 * and eventually error out as field not
+			 * found to keep it simple.
+			 * It could be relaxed if there was a legit
+			 * partial access case later.
+			 */
+			continue;
+		}
 		/* In case of "off" is pointing to holes of a struct */
 		if (off < moff)
-			continue;
+			break;
 		/* type of the field */
 		mtype = btf_type_by_id(btf_vmlinux, member->type);

kernel/bpf/map_in_map.c+2−1

 	 */
 	if (inner_map->map_type == BPF_MAP_TYPE_PROG_ARRAY ||
 	    inner_map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
-	    inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
+	    inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE ||
+	    inner_map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
 		fdput(f);
 		return ERR_PTR(-ENOTSUPP);
 	}

kernel/bpf/syscall.c+46−23

@@ -628,7 +628,7 @@ static int map_check_btf(struct bpf_map map, const struct btf btf, · 627 unmodified lines
628	return ret;	628	return ret;
629	}	629	}
630		630
631	#define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id	631	#define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
632	/* called via syscall */	632	/* called via syscall */
633	static int map_create(union bpf_attr *attr)	633	static int map_create(union bpf_attr *attr)
634	{	634	{
@@ -642,6 +642,14 @@ static int map_create(union bpf_attr *attr) · 7 unmodified lines
642	if (err)	642	if (err)
643	return -EINVAL;	643	return -EINVAL;
644		644
		645	if (attr->btf_vmlinux_value_type_id) {
		646	if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS \|\|
		647	attr->btf_key_type_id \|\| attr->btf_value_type_id)
		648	return -EINVAL;
		649	} else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
		650	return -EINVAL;
		651	}
		652
645	f_flags = bpf_get_file_flag(attr->map_flags);	653	f_flags = bpf_get_file_flag(attr->map_flags);
646	if (f_flags < 0)	654	if (f_flags < 0)
647	return f_flags;	655	return f_flags;
@@ -664,32 +672,35 @@ static int map_create(union bpf_attr *attr) · 16 unmodified lines
664	atomic64_set(&map->usercnt, 1);	672	atomic64_set(&map->usercnt, 1);
665	mutex_init(&map->freeze_mutex);	673	mutex_init(&map->freeze_mutex);
666		674
667	if (attr->btf_key_type_id \|\| attr->btf_value_type_id) {	675	map->spin_lock_off = -EINVAL;
		676	if (attr->btf_key_type_id \|\| attr->btf_value_type_id \|\|
		677	/* Even the map's value is a kernel's struct,
		678	* the bpf_prog.o must have BTF to begin with
		679	* to figure out the corresponding kernel's
		680	* counter part. Thus, attr->btf_fd has
		681	* to be valid also.
		682	*/
		683	attr->btf_vmlinux_value_type_id) {
668	struct btf *btf;	684	struct btf *btf;
669		685
670	if (!attr->btf_value_type_id) {
671	err = -EINVAL;
672	goto free_map;
673	}
674
675	btf = btf_get_by_fd(attr->btf_fd);	686	btf = btf_get_by_fd(attr->btf_fd);
676	if (IS_ERR(btf)) {	687	if (IS_ERR(btf)) {
677	err = PTR_ERR(btf);	688	err = PTR_ERR(btf);
678	goto free_map;	689	goto free_map;
679	}	690	}
		691	map->btf = btf;
680		692
681	err = map_check_btf(map, btf, attr->btf_key_type_id,	693	if (attr->btf_value_type_id) {
682	attr->btf_value_type_id);	694	err = map_check_btf(map, btf, attr->btf_key_type_id,
683	if (err) {	695	attr->btf_value_type_id);
684	btf_put(btf);	696	if (err)
685	goto free_map;	697	goto free_map;
686	}	698	}
687		699
688	map->btf = btf;
689	map->btf_key_type_id = attr->btf_key_type_id;	700	map->btf_key_type_id = attr->btf_key_type_id;
690	map->btf_value_type_id = attr->btf_value_type_id;	701	map->btf_value_type_id = attr->btf_value_type_id;
691	} else {	702	map->btf_vmlinux_value_type_id =
692	map->spin_lock_off = -EINVAL;	703	attr->btf_vmlinux_value_type_id;
693	}	704	}
694		705
695	err = security_bpf_map_alloc(map);	706	err = security_bpf_map_alloc(map);
@@ -888,6 +899,9 @@ static int map_lookup_elem(union bpf_attr *attr) · 192 unmodified lines
888	} else if (map->map_type == BPF_MAP_TYPE_QUEUE \|\|	899	} else if (map->map_type == BPF_MAP_TYPE_QUEUE \|\|
889	map->map_type == BPF_MAP_TYPE_STACK) {	900	map->map_type == BPF_MAP_TYPE_STACK) {
890	err = map->ops->map_peek_elem(map, value);	901	err = map->ops->map_peek_elem(map, value);
		902	} else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
		903	/* struct_ops map requires directly updating "value" */
		904	err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
891	} else {	905	} else {
892	rcu_read_lock();	906	rcu_read_lock();
893	if (map->ops->map_lookup_elem_sys_only)	907	if (map->ops->map_lookup_elem_sys_only)
@@ -1003,7 +1017,8 @@ static int map_update_elem(union bpf_attr *attr) · 109 unmodified lines
1003	goto out;	1017	goto out;
1004	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP \|\|	1018	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP \|\|
1005	map->map_type == BPF_MAP_TYPE_SOCKHASH \|\|	1019	map->map_type == BPF_MAP_TYPE_SOCKHASH \|\|
1006	map->map_type == BPF_MAP_TYPE_SOCKMAP) {	1020	map->map_type == BPF_MAP_TYPE_SOCKMAP \|\|
		1021	map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1007	err = map->ops->map_update_elem(map, key, value, attr->flags);	1022	err = map->ops->map_update_elem(map, key, value, attr->flags);
1008	goto out;	1023	goto out;
1009	} else if (IS_FD_PROG_ARRAY(map)) {	1024	} else if (IS_FD_PROG_ARRAY(map)) {
@@ -1092,7 +1107,9 @@ static int map_delete_elem(union bpf_attr *attr) · 82 unmodified lines
1092	if (bpf_map_is_dev_bound(map)) {	1107	if (bpf_map_is_dev_bound(map)) {
1093	err = bpf_map_offload_delete_elem(map, key);	1108	err = bpf_map_offload_delete_elem(map, key);
1094	goto out;	1109	goto out;
1095	} else if (IS_FD_PROG_ARRAY(map)) {	1110	} else if (IS_FD_PROG_ARRAY(map) \|\|
		1111	map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
		1112	/* These maps require sleepable context */
1096	err = map->ops->map_delete_elem(map, key);	1113	err = map->ops->map_delete_elem(map, key);
1097	goto out;	1114	goto out;
1098	}	1115	}
@@ -1672,17 +1689,22 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, · 573 unmodified lines
1672	enum bpf_attach_type expected_attach_type,	1689	enum bpf_attach_type expected_attach_type,
1673	u32 btf_id, u32 prog_fd)	1690	u32 btf_id, u32 prog_fd)
1674	{	1691	{
1675	switch (prog_type) {	1692	if (btf_id) {
1676	case BPF_PROG_TYPE_TRACING:
1677	if (btf_id > BTF_MAX_TYPE)	1693	if (btf_id > BTF_MAX_TYPE)
1678	return -EINVAL;	1694	return -EINVAL;
1679	break;	1695
1680	default:	1696	switch (prog_type) {
1681	if (btf_id \|\| prog_fd)	1697	case BPF_PROG_TYPE_TRACING:
		1698	case BPF_PROG_TYPE_STRUCT_OPS:
		1699	break;
		1700	default:
1682	return -EINVAL;	1701	return -EINVAL;
1683	break;	1702	}
1684	}	1703	}
1685		1704
		1705	if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING)
		1706	return -EINVAL;
		1707
1686	switch (prog_type) {	1708	switch (prog_type) {
1687	case BPF_PROG_TYPE_CGROUP_SOCK:	1709	case BPF_PROG_TYPE_CGROUP_SOCK:
1688	switch (expected_attach_type) {	1710	switch (expected_attach_type) {
@@ -2817,6 +2839,7 @@ static int bpf_map_get_info_by_fd(struct bpf_map *map, · 1128 unmodified lines
2817	info.btf_key_type_id = map->btf_key_type_id;	2839	info.btf_key_type_id = map->btf_key_type_id;
2818	info.btf_value_type_id = map->btf_value_type_id;	2840	info.btf_value_type_id = map->btf_value_type_id;
2819	}	2841	}
		2842	info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
2820		2843
2821	if (bpf_map_is_dev_bound(map)) {	2844	if (bpf_map_is_dev_bound(map)) {
2822	err = bpf_map_offload_info_fill(&info, map);	2845	err = bpf_map_offload_info_fill(&info, map);

kernel/bpf/trampoline.c+5−3

 	if (fexit_cnt)
 		flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME;
-	err = arch_prepare_bpf_trampoline(new_image, &tr->func.model, flags,
+	err = arch_prepare_bpf_trampoline(new_image, new_image + PAGE_SIZE / 2,
+					  &tr->func.model, flags,
 					  fentry, fentry_cnt,
 					  fexit, fexit_cnt,
 					  tr->func.addr);
-	if (err)
+	if (err < 0)
 		goto out;
 	if (tr->selector)
 }
 int __weak
-arch_prepare_bpf_trampoline(void *image, struct btf_func_model *m, u32 flags,
+arch_prepare_bpf_trampoline(void *image, void *image_end,
+			    const struct btf_func_model *m, u32 flags,
 			    struct bpf_prog **fentry_progs, int fentry_cnt,
 			    struct bpf_prog **fexit_progs, int fexit_cnt,
 			    void *orig_call)

kernel/bpf/verifier.c+109−30

@@ -2859,11 +2859,6 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 2858 unmodified lines
2859	u32 btf_id;	2859	u32 btf_id;
2860	int ret;	2860	int ret;
2861		2861
2862	if (atype != BPF_READ) {
2863	verbose(env, "only read is supported\n");
2864	return -EACCES;
2865	}
2866
2867	if (off < 0) {	2862	if (off < 0) {
2868	verbose(env,	2863	verbose(env,
2869	"R%d is ptr_%s invalid negative access: off=%d\n",	2864	"R%d is ptr_%s invalid negative access: off=%d\n",
@@ -2880,17 +2875,32 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 10 unmodified lines
2880	return -EACCES;	2875	return -EACCES;
2881	}	2876	}
2882		2877
2883	ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id);	2878	if (env->ops->btf_struct_access) {
		2879	ret = env->ops->btf_struct_access(&env->log, t, off, size,
		2880	atype, &btf_id);
		2881	} else {
		2882	if (atype != BPF_READ) {
		2883	verbose(env, "only read is supported\n");
		2884	return -EACCES;
		2885	}
		2886
		2887	ret = btf_struct_access(&env->log, t, off, size, atype,
		2888	&btf_id);
		2889	}
		2890
2884	if (ret < 0)	2891	if (ret < 0)
2885	return ret;	2892	return ret;
2886		2893
2887	if (ret == SCALAR_VALUE) {	2894	if (atype == BPF_READ) {
2888	mark_reg_unknown(env, regs, value_regno);	2895	if (ret == SCALAR_VALUE) {
2889	return 0;	2896	mark_reg_unknown(env, regs, value_regno);
		2897	return 0;
		2898	}
		2899	mark_reg_known_zero(env, regs, value_regno);
		2900	regs[value_regno].type = PTR_TO_BTF_ID;
		2901	regs[value_regno].btf_id = btf_id;
2890	}	2902	}
2891	mark_reg_known_zero(env, regs, value_regno);	2903
2892	regs[value_regno].type = PTR_TO_BTF_ID;
2893	regs[value_regno].btf_id = btf_id;
2894	return 0;	2904	return 0;
2895	}	2905	}
2896		2906
@@ -6349,8 +6359,30 @@ static int check_ld_abs(struct bpf_verifier_env env, struct bpf_insn insn) · 3452 unmodified lines
6349	static int check_return_code(struct bpf_verifier_env *env)	6359	static int check_return_code(struct bpf_verifier_env *env)
6350	{	6360	{
6351	struct tnum enforce_attach_type_range = tnum_unknown;	6361	struct tnum enforce_attach_type_range = tnum_unknown;
		6362	const struct bpf_prog *prog = env->prog;
6352	struct bpf_reg_state *reg;	6363	struct bpf_reg_state *reg;
6353	struct tnum range = tnum_range(0, 1);	6364	struct tnum range = tnum_range(0, 1);
		6365	int err;
		6366
		6367	/* The struct_ops func-ptr's return type could be "void" */
		6368	if (env->prog->type == BPF_PROG_TYPE_STRUCT_OPS &&
		6369	!prog->aux->attach_func_proto->type)
		6370	return 0;
		6371
		6372	/* eBPF calling convetion is such that R0 is used
		6373	* to return the value from eBPF program.
		6374	* Make sure that it's readable at this time
		6375	* of bpf_exit, which means that program wrote
		6376	* something into it earlier
		6377	*/
		6378	err = check_reg_arg(env, BPF_REG_0, SRC_OP);
		6379	if (err)
		6380	return err;
		6381
		6382	if (is_pointer_value(env, BPF_REG_0)) {
		6383	verbose(env, "R0 leaks addr as return value\n");
		6384	return -EACCES;
		6385	}
6354		6386
6355	switch (env->prog->type) {	6387	switch (env->prog->type) {
6356	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:	6388	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
@@ -8016,21 +8048,6 @@ static int do_check(struct bpf_verifier_env *env) · 1659 unmodified lines
8016	if (err)	8048	if (err)
8017	return err;	8049	return err;
8018		8050
8019	/* eBPF calling convetion is such that R0 is used
8020	* to return the value from eBPF program.
8021	* Make sure that it's readable at this time
8022	* of bpf_exit, which means that program wrote
8023	* something into it earlier
8024	*/
8025	err = check_reg_arg(env, BPF_REG_0, SRC_OP);
8026	if (err)
8027	return err;
8028
8029	if (is_pointer_value(env, BPF_REG_0)) {
8030	verbose(env, "R0 leaks addr as return value\n");
8031	return -EACCES;
8032	}
8033
8034	err = check_return_code(env);	8051	err = check_return_code(env);
8035	if (err)	8052	if (err)
8036	return err;	8053	return err;
@@ -8138,6 +8155,11 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 101 unmodified lines
8138	return -EINVAL;	8155	return -EINVAL;
8139	}	8156	}
8140		8157
		8158	if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
		8159	verbose(env, "bpf_struct_ops map cannot be used in prog\n");
		8160	return -EINVAL;
		8161	}
		8162
8141	return 0;	8163	return 0;
8142	}	8164	}
8143		8165
@@ -8829,12 +8851,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 685 unmodified lines
8829	convert_ctx_access = bpf_xdp_sock_convert_ctx_access;	8851	convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
8830	break;	8852	break;
8831	case PTR_TO_BTF_ID:	8853	case PTR_TO_BTF_ID:
8832	if (type == BPF_WRITE) {	8854	if (type == BPF_READ) {
		8855	insn->code = BPF_LDX \| BPF_PROBE_MEM \|
		8856	BPF_SIZE((insn)->code);
		8857	env->prog->aux->num_exentries++;
		8858	} else if (env->prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
8833	verbose(env, "Writes through BTF pointers are not allowed\n");	8859	verbose(env, "Writes through BTF pointers are not allowed\n");
8834	return -EINVAL;	8860	return -EINVAL;
8835	}	8861	}
8836	insn->code = BPF_LDX \| BPF_PROBE_MEM \| BPF_SIZE((insn)->code);
8837	env->prog->aux->num_exentries++;
8838	continue;	8862	continue;
8839	default:	8863	default:
8840	continue;	8864	continue;
@@ -9502,6 +9526,58 @@ static void print_verification_stats(struct bpf_verifier_env *env) · 661 unmodified lines
9502	env->peak_states, env->longest_mark_read_walk);	9526	env->peak_states, env->longest_mark_read_walk);
9503	}	9527	}
9504		9528
		9529	static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
		9530	{
		9531	const struct btf_type t, func_proto;
		9532	const struct bpf_struct_ops *st_ops;
		9533	const struct btf_member *member;
		9534	struct bpf_prog *prog = env->prog;
		9535	u32 btf_id, member_idx;
		9536	const char *mname;
		9537
		9538	btf_id = prog->aux->attach_btf_id;
		9539	st_ops = bpf_struct_ops_find(btf_id);
		9540	if (!st_ops) {
		9541	verbose(env, "attach_btf_id %u is not a supported struct\n",
		9542	btf_id);
		9543	return -ENOTSUPP;
		9544	}
		9545
		9546	t = st_ops->type;
		9547	member_idx = prog->expected_attach_type;
		9548	if (member_idx >= btf_type_vlen(t)) {
		9549	verbose(env, "attach to invalid member idx %u of struct %s\n",
		9550	member_idx, st_ops->name);
		9551	return -EINVAL;
		9552	}
		9553
		9554	member = &btf_type_member(t)[member_idx];
		9555	mname = btf_name_by_offset(btf_vmlinux, member->name_off);
		9556	func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type,
		9557	NULL);
		9558	if (!func_proto) {
		9559	verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
		9560	mname, member_idx, st_ops->name);
		9561	return -EINVAL;
		9562	}
		9563
		9564	if (st_ops->check_member) {
		9565	int err = st_ops->check_member(t, member);
		9566
		9567	if (err) {
		9568	verbose(env, "attach to unsupported member %s of struct %s\n",
		9569	mname, st_ops->name);
		9570	return err;
		9571	}
		9572	}
		9573
		9574	prog->aux->attach_func_proto = func_proto;
		9575	prog->aux->attach_func_name = mname;
		9576	env->ops = st_ops->verifier_ops;
		9577
		9578	return 0;
		9579	}
		9580
9505	static int check_attach_btf_id(struct bpf_verifier_env *env)	9581	static int check_attach_btf_id(struct bpf_verifier_env *env)
9506	{	9582	{
9507	struct bpf_prog *prog = env->prog;	9583	struct bpf_prog *prog = env->prog;
@@ -9517,6 +9593,9 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 9 unmodified lines
9517	long addr;	9593	long addr;
9518	u64 key;	9594	u64 key;
9519		9595
		9596	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
		9597	return check_struct_ops_btf_id(env);
		9598
9520	if (prog->type != BPF_PROG_TYPE_TRACING)	9599	if (prog->type != BPF_PROG_TYPE_TRACING)
9521	return 0;	9600	return 0;
9522		9601

net/core/filter.c+1−1

@@ -5935,7 +5935,7 @@ bool bpf_helper_changes_pkt_data(void *func) · 5934 unmodified lines
5935	return false;	5935	return false;
5936	}	5936	}
5937		5937
5938	static const struct bpf_func_proto *	5938	const struct bpf_func_proto *
5939	bpf_base_func_proto(enum bpf_func_id func_id)	5939	bpf_base_func_proto(enum bpf_func_id func_id)
5940	{	5940	{
5941	switch (func_id) {	5941	switch (func_id) {

net/ipv4/Makefile+4−0

@@ -65,3 +65,7 @@ obj-$(CONFIG_NETLABEL) += cipso_ipv4.o · 64 unmodified lines
65		65
66	obj-$(CONFIG_XFRM) += xfrm4_policy.o xfrm4_state.o xfrm4_input.o \	66	obj-$(CONFIG_XFRM) += xfrm4_policy.o xfrm4_state.o xfrm4_input.o \
67	xfrm4_output.o xfrm4_protocol.o	67	xfrm4_output.o xfrm4_protocol.o
		68
		69	ifeq ($(CONFIG_BPF_JIT),y)
		70	obj-$(CONFIG_BPF_SYSCALL) += bpf_tcp_ca.o
		71	endif

net/ipv4/bpf_tcp_ca.c+252−0

@@ -0,0 +1,252 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2019 Facebook */
	3
	4	#include <linux/types.h>
	5	#include <linux/bpf_verifier.h>
	6	#include <linux/bpf.h>
	7	#include <linux/btf.h>
	8	#include <linux/filter.h>
	9	#include <net/tcp.h>
	10
	11	static u32 optional_ops[] = {
	12	offsetof(struct tcp_congestion_ops, init),
	13	offsetof(struct tcp_congestion_ops, release),
	14	offsetof(struct tcp_congestion_ops, set_state),
	15	offsetof(struct tcp_congestion_ops, cwnd_event),
	16	offsetof(struct tcp_congestion_ops, in_ack_event),
	17	offsetof(struct tcp_congestion_ops, pkts_acked),
	18	offsetof(struct tcp_congestion_ops, min_tso_segs),
	19	offsetof(struct tcp_congestion_ops, sndbuf_expand),
	20	offsetof(struct tcp_congestion_ops, cong_control),
	21	};
	22
	23	static u32 unsupported_ops[] = {
	24	offsetof(struct tcp_congestion_ops, get_info),
	25	};
	26
	27	static const struct btf_type *tcp_sock_type;
	28	static u32 tcp_sock_id, sock_id;
	29
	30	static int bpf_tcp_ca_init(struct btf *btf)
	31	{
	32	s32 type_id;
	33
	34	type_id = btf_find_by_name_kind(btf, "sock", BTF_KIND_STRUCT);
	35	if (type_id < 0)
	36	return -EINVAL;
	37	sock_id = type_id;
	38
	39	type_id = btf_find_by_name_kind(btf, "tcp_sock", BTF_KIND_STRUCT);
	40	if (type_id < 0)
	41	return -EINVAL;
	42	tcp_sock_id = type_id;
	43	tcp_sock_type = btf_type_by_id(btf, tcp_sock_id);
	44
	45	return 0;
	46	}
	47
	48	static bool is_optional(u32 member_offset)
	49	{
	50	unsigned int i;
	51
	52	for (i = 0; i < ARRAY_SIZE(optional_ops); i++) {
	53	if (member_offset == optional_ops[i])
	54	return true;
	55	}
	56
	57	return false;
	58	}
	59
	60	static bool is_unsupported(u32 member_offset)
	61	{
	62	unsigned int i;
	63
	64	for (i = 0; i < ARRAY_SIZE(unsupported_ops); i++) {
	65	if (member_offset == unsupported_ops[i])
	66	return true;
	67	}
	68
	69	return false;
	70	}
	71
	72	extern struct btf *btf_vmlinux;
	73
	74	static bool bpf_tcp_ca_is_valid_access(int off, int size,
	75	enum bpf_access_type type,
	76	const struct bpf_prog *prog,
	77	struct bpf_insn_access_aux *info)
	78	{
	79	if (off < 0 \|\| off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
	80	return false;
	81	if (type != BPF_READ)
	82	return false;
	83	if (off % size != 0)
	84	return false;
	85
	86	if (!btf_ctx_access(off, size, type, prog, info))
	87	return false;
	88
	89	if (info->reg_type == PTR_TO_BTF_ID && info->btf_id == sock_id)
	90	/* promote it to tcp_sock */
	91	info->btf_id = tcp_sock_id;
	92
	93	return true;
	94	}
	95
	96	static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log,
	97	const struct btf_type *t, int off,
	98	int size, enum bpf_access_type atype,
	99	u32 *next_btf_id)
	100	{
	101	size_t end;
	102
	103	if (atype == BPF_READ)
	104	return btf_struct_access(log, t, off, size, atype, next_btf_id);
	105
	106	if (t != tcp_sock_type) {
	107	bpf_log(log, "only read is supported\n");
	108	return -EACCES;
	109	}
	110
	111	switch (off) {
	112	case bpf_ctx_range(struct inet_connection_sock, icsk_ca_priv):
	113	end = offsetofend(struct inet_connection_sock, icsk_ca_priv);
	114	break;
	115	case offsetof(struct inet_connection_sock, icsk_ack.pending):
	116	end = offsetofend(struct inet_connection_sock,
	117	icsk_ack.pending);
	118	break;
	119	case offsetof(struct tcp_sock, snd_cwnd):
	120	end = offsetofend(struct tcp_sock, snd_cwnd);
	121	break;
	122	case offsetof(struct tcp_sock, snd_cwnd_cnt):
	123	end = offsetofend(struct tcp_sock, snd_cwnd_cnt);
	124	break;
	125	case offsetof(struct tcp_sock, snd_ssthresh):
	126	end = offsetofend(struct tcp_sock, snd_ssthresh);
	127	break;
	128	case offsetof(struct tcp_sock, ecn_flags):
	129	end = offsetofend(struct tcp_sock, ecn_flags);
	130	break;
	131	default:
	132	bpf_log(log, "no write support to tcp_sock at off %d\n", off);
	133	return -EACCES;
	134	}
	135
	136	if (off + size > end) {
	137	bpf_log(log,
	138	"write access at off %d with size %d beyond the member of tcp_sock ended at %zu\n",
	139	off, size, end);
	140	return -EACCES;
	141	}
	142
	143	return NOT_INIT;
	144	}
	145
	146	BPF_CALL_2(bpf_tcp_send_ack, struct tcp_sock *, tp, u32, rcv_nxt)
	147	{
	148	/* bpf_tcp_ca prog cannot have NULL tp */
	149	__tcp_send_ack((struct sock *)tp, rcv_nxt);
	150	return 0;
	151	}
	152
	153	static const struct bpf_func_proto bpf_tcp_send_ack_proto = {
	154	.func = bpf_tcp_send_ack,
	155	.gpl_only = false,
	156	/* In case we want to report error later */
	157	.ret_type = RET_INTEGER,
	158	.arg1_type = ARG_PTR_TO_BTF_ID,
	159	.arg2_type = ARG_ANYTHING,
	160	.btf_id = &tcp_sock_id,
	161	};
	162
	163	static const struct bpf_func_proto *
	164	bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id,
	165	const struct bpf_prog *prog)
	166	{
	167	switch (func_id) {
	168	case BPF_FUNC_tcp_send_ack:
	169	return &bpf_tcp_send_ack_proto;
	170	default:
	171	return bpf_base_func_proto(func_id);
	172	}
	173	}
	174
	175	static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {
	176	.get_func_proto = bpf_tcp_ca_get_func_proto,
	177	.is_valid_access = bpf_tcp_ca_is_valid_access,
	178	.btf_struct_access = bpf_tcp_ca_btf_struct_access,
	179	};
	180
	181	static int bpf_tcp_ca_init_member(const struct btf_type *t,
	182	const struct btf_member *member,
	183	void kdata, const void udata)
	184	{
	185	const struct tcp_congestion_ops *utcp_ca;
	186	struct tcp_congestion_ops *tcp_ca;
	187	size_t tcp_ca_name_len;
	188	int prog_fd;
	189	u32 moff;
	190
	191	utcp_ca = (const struct tcp_congestion_ops *)udata;
	192	tcp_ca = (struct tcp_congestion_ops *)kdata;
	193
	194	moff = btf_member_bit_offset(t, member) / 8;
	195	switch (moff) {
	196	case offsetof(struct tcp_congestion_ops, flags):
	197	if (utcp_ca->flags & ~TCP_CONG_MASK)
	198	return -EINVAL;
	199	tcp_ca->flags = utcp_ca->flags;
	200	return 1;
	201	case offsetof(struct tcp_congestion_ops, name):
	202	tcp_ca_name_len = strnlen(utcp_ca->name, sizeof(utcp_ca->name));
	203	if (!tcp_ca_name_len \|\|
	204	tcp_ca_name_len == sizeof(utcp_ca->name))
	205	return -EINVAL;
	206	if (tcp_ca_find(utcp_ca->name))
	207	return -EEXIST;
	208	memcpy(tcp_ca->name, utcp_ca->name, sizeof(tcp_ca->name));
	209	return 1;
	210	}
	211
	212	if (!btf_type_resolve_func_ptr(btf_vmlinux, member->type, NULL))
	213	return 0;
	214
	215	/* Ensure bpf_prog is provided for compulsory func ptr */
	216	prog_fd = (int)((unsigned long )(udata + moff));
	217	if (!prog_fd && !is_optional(moff) && !is_unsupported(moff))
	218	return -EINVAL;
	219
	220	return 0;
	221	}
	222
	223	static int bpf_tcp_ca_check_member(const struct btf_type *t,
	224	const struct btf_member *member)
	225	{
	226	if (is_unsupported(btf_member_bit_offset(t, member) / 8))
	227	return -ENOTSUPP;
	228	return 0;
	229	}
	230
	231	static int bpf_tcp_ca_reg(void *kdata)
	232	{
	233	return tcp_register_congestion_control(kdata);
	234	}
	235
	236	static void bpf_tcp_ca_unreg(void *kdata)
	237	{
	238	tcp_unregister_congestion_control(kdata);
	239	}
	240
	241	/* Avoid sparse warning. It is only used in bpf_struct_ops.c. */
	242	extern struct bpf_struct_ops bpf_tcp_congestion_ops;
	243
	244	struct bpf_struct_ops bpf_tcp_congestion_ops = {
	245	.verifier_ops = &bpf_tcp_ca_verifier_ops,
	246	.reg = bpf_tcp_ca_reg,
	247	.unreg = bpf_tcp_ca_unreg,
	248	.check_member = bpf_tcp_ca_check_member,
	249	.init_member = bpf_tcp_ca_init_member,
	250	.init = bpf_tcp_ca_init,
	251	.name = "tcp_congestion_ops",
	252	};

net/ipv4/tcp_cong.c+8−8

@@ -21,7 +21,7 @@ static DEFINE_SPINLOCK(tcp_cong_list_lock); · 20 unmodified lines
21	static LIST_HEAD(tcp_cong_list);	21	static LIST_HEAD(tcp_cong_list);
22		22
23	/* Simple linear search, don't expect many entries! */	23	/* Simple linear search, don't expect many entries! */
24	static struct tcp_congestion_ops tcp_ca_find(const char name)	24	struct tcp_congestion_ops tcp_ca_find(const char name)
25	{	25	{
26	struct tcp_congestion_ops *e;	26	struct tcp_congestion_ops *e;
27		27
@@ -162,7 +162,7 @@ void tcp_assign_congestion_control(struct sock *sk) · 134 unmodified lines
162		162
163	rcu_read_lock();	163	rcu_read_lock();
164	ca = rcu_dereference(net->ipv4.tcp_congestion_control);	164	ca = rcu_dereference(net->ipv4.tcp_congestion_control);
165	if (unlikely(!try_module_get(ca->owner)))	165	if (unlikely(!bpf_try_module_get(ca, ca->owner)))
166	ca = &tcp_reno;	166	ca = &tcp_reno;
167	icsk->icsk_ca_ops = ca;	167	icsk->icsk_ca_ops = ca;
168	rcu_read_unlock();	168	rcu_read_unlock();
@@ -208,7 +208,7 @@ void tcp_cleanup_congestion_control(struct sock *sk) · 39 unmodified lines
208		208
209	if (icsk->icsk_ca_ops->release)	209	if (icsk->icsk_ca_ops->release)
210	icsk->icsk_ca_ops->release(sk);	210	icsk->icsk_ca_ops->release(sk);
211	module_put(icsk->icsk_ca_ops->owner);	211	bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
212	}	212	}
213		213
214	/* Used by sysctl to change default congestion control */	214	/* Used by sysctl to change default congestion control */
@@ -222,12 +222,12 @@ int tcp_set_default_congestion_control(struct net net, const char name) · 7 unmodified lines
222	ca = tcp_ca_find_autoload(net, name);	222	ca = tcp_ca_find_autoload(net, name);
223	if (!ca) {	223	if (!ca) {
224	ret = -ENOENT;	224	ret = -ENOENT;
225	} else if (!try_module_get(ca->owner)) {	225	} else if (!bpf_try_module_get(ca, ca->owner)) {
226	ret = -EBUSY;	226	ret = -EBUSY;
227	} else {	227	} else {
228	prev = xchg(&net->ipv4.tcp_congestion_control, ca);	228	prev = xchg(&net->ipv4.tcp_congestion_control, ca);
229	if (prev)	229	if (prev)
230	module_put(prev->owner);	230	bpf_module_put(prev, prev->owner);
231		231
232	ca->flags \|= TCP_CONG_NON_RESTRICTED;	232	ca->flags \|= TCP_CONG_NON_RESTRICTED;
233	ret = 0;	233	ret = 0;
@@ -366,19 +366,19 @@ int tcp_set_congestion_control(struct sock sk, const char name, bool load, · 132 unmodified lines
366	} else if (!load) {	366	} else if (!load) {
367	const struct tcp_congestion_ops *old_ca = icsk->icsk_ca_ops;	367	const struct tcp_congestion_ops *old_ca = icsk->icsk_ca_ops;
368		368
369	if (try_module_get(ca->owner)) {	369	if (bpf_try_module_get(ca, ca->owner)) {
370	if (reinit) {	370	if (reinit) {
371	tcp_reinit_congestion_control(sk, ca);	371	tcp_reinit_congestion_control(sk, ca);
372	} else {	372	} else {
373	icsk->icsk_ca_ops = ca;	373	icsk->icsk_ca_ops = ca;
374	module_put(old_ca->owner);	374	bpf_module_put(old_ca, old_ca->owner);
375	}	375	}
376	} else {	376	} else {
377	err = -EBUSY;	377	err = -EBUSY;
378	}	378	}
379	} else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) \|\| cap_net_admin)) {	379	} else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) \|\| cap_net_admin)) {
380	err = -EPERM;	380	err = -EPERM;
381	} else if (!try_module_get(ca->owner)) {	381	} else if (!bpf_try_module_get(ca, ca->owner)) {
382	err = -EBUSY;	382	err = -EBUSY;
383	} else {	383	} else {
384	tcp_reinit_congestion_control(sk, ca);	384	tcp_reinit_congestion_control(sk, ca);

net/ipv4/tcp_ipv4.c+4−2

 	int cpu;
 	if (net->ipv4.tcp_congestion_control)
-		module_put(net->ipv4.tcp_congestion_control->owner);
+		bpf_module_put(net->ipv4.tcp_congestion_control,
+			       net->ipv4.tcp_congestion_control->owner);
 	for_each_possible_cpu(cpu)
 		inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
 	/* Reno is always built in */
 	if (!net_eq(net, &init_net) &&
-	    try_module_get(init_net.ipv4.tcp_congestion_control->owner))
+	    bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
+			       init_net.ipv4.tcp_congestion_control->owner))
 		net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
 	else
 		net->ipv4.tcp_congestion_control = &tcp_reno;

net/ipv4/tcp_minisocks.c+2−2

@@ -414,7 +414,7 @@ void tcp_ca_openreq_child(struct sock sk, const struct dst_entry dst) · 413 unmodified lines
414		414
415	rcu_read_lock();	415	rcu_read_lock();
416	ca = tcp_ca_find_key(ca_key);	416	ca = tcp_ca_find_key(ca_key);
417	if (likely(ca && try_module_get(ca->owner))) {	417	if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
418	icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);	418	icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
419	icsk->icsk_ca_ops = ca;	419	icsk->icsk_ca_ops = ca;
420	ca_got_dst = true;	420	ca_got_dst = true;
@@ -425,7 +425,7 @@ void tcp_ca_openreq_child(struct sock sk, const struct dst_entry dst) · 4 unmodified lines
425	/* If no valid choice made yet, assign current system default ca. */	425	/* If no valid choice made yet, assign current system default ca. */
426	if (!ca_got_dst &&	426	if (!ca_got_dst &&
427	(!icsk->icsk_ca_setsockopt \|\|	427	(!icsk->icsk_ca_setsockopt \|\|
428	!try_module_get(icsk->icsk_ca_ops->owner)))	428	!bpf_try_module_get(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner)))
429	tcp_assign_congestion_control(sk);	429	tcp_assign_congestion_control(sk);
430		430
431	tcp_set_ca_state(sk, TCP_CA_Open);	431	tcp_set_ca_state(sk, TCP_CA_Open);

net/ipv4/tcp_output.c+2−2

@@ -3368,8 +3368,8 @@ static void tcp_ca_dst_init(struct sock sk, const struct dst_entry dst) · 3367 unmodified lines
3368		3368
3369	rcu_read_lock();	3369	rcu_read_lock();
3370	ca = tcp_ca_find_key(ca_key);	3370	ca = tcp_ca_find_key(ca_key);
3371	if (likely(ca && try_module_get(ca->owner))) {	3371	if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
3372	module_put(icsk->icsk_ca_ops->owner);	3372	bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
3373	icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);	3373	icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
3374	icsk->icsk_ca_ops = ca;	3374	icsk->icsk_ca_ops = ca;
3375	}	3375	}

tools/include/uapi/linux/bpf.h+17−2

 	BPF_MAP_TYPE_STACK,
 	BPF_MAP_TYPE_SK_STORAGE,
 	BPF_MAP_TYPE_DEVMAP_HASH,
+	BPF_MAP_TYPE_STRUCT_OPS,
 };
 /* Note that tracing related programs such as
 	BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE,
 	BPF_PROG_TYPE_CGROUP_SOCKOPT,
 	BPF_PROG_TYPE_TRACING,
+	BPF_PROG_TYPE_STRUCT_OPS,
 };
 enum bpf_attach_type {
 		__u32	btf_fd;		/* fd pointing to a BTF type data */
 		__u32	btf_key_type_id;	/* BTF type_id of the key */
 		__u32	btf_value_type_id;	/* BTF type_id of the value */
+		__u32	btf_vmlinux_value_type_id;/* BTF type_id of a kernel-
+						   * struct stored as the
+						   * map value
+						   */
 	};
 	struct { /* anonymous struct used by BPF_MAP_*_ELEM commands */
  * 	Return
  * 		On success, the strictly positive length of the string,	including
  * 		the trailing NUL character. On error, a negative value.
+ *
+ * int bpf_tcp_send_ack(void *tp, u32 rcv_nxt)
+ *	Description
+ *		Send out a tcp-ack. *tp* is the in-kernel struct tcp_sock.
+ *		*rcv_nxt* is the ack_seq to be sent out.
+ *	Return
+ *		0 on success, or a negative error in case of failure.
+ *
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
 	FN(probe_read_user),		\
 	FN(probe_read_kernel),		\
 	FN(probe_read_user_str),	\
-	FN(probe_read_kernel_str),
+	FN(probe_read_kernel_str),	\
+	FN(tcp_send_ack),
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
  * function eBPF program intends to call
 	__u32 map_flags;
 	char  name[BPF_OBJ_NAME_LEN];
 	__u32 ifindex;
-	__u32 :32;
+	__u32 btf_vmlinux_value_type_id;
 	__u64 netns_dev;
 	__u64 netns_ino;
 	__u32 btf_id;

tools/lib/bpf/bpf.c+8−2

 	attr.btf_key_type_id = create_attr->btf_key_type_id;
 	attr.btf_value_type_id = create_attr->btf_value_type_id;
 	attr.map_ifindex = create_attr->map_ifindex;
-	attr.inner_map_fd = create_attr->inner_map_fd;
+	if (attr.map_type == BPF_MAP_TYPE_STRUCT_OPS)
+		attr.btf_vmlinux_value_type_id =
+			create_attr->btf_vmlinux_value_type_id;
+	else
+		attr.inner_map_fd = create_attr->inner_map_fd;
 	return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
 }
 	memset(&attr, 0, sizeof(attr));
 	attr.prog_type = load_attr->prog_type;
 	attr.expected_attach_type = load_attr->expected_attach_type;
-	if (attr.prog_type == BPF_PROG_TYPE_TRACING) {
+	if (attr.prog_type == BPF_PROG_TYPE_STRUCT_OPS) {
+		attr.attach_btf_id = load_attr->attach_btf_id;
+	} else if (attr.prog_type == BPF_PROG_TYPE_TRACING) {
 		attr.attach_btf_id = load_attr->attach_btf_id;
 		attr.attach_prog_fd = load_attr->attach_prog_fd;
 	} else {

tools/lib/bpf/bpf.h+4−1

 	__u32 btf_key_type_id;
 	__u32 btf_value_type_id;
 	__u32 map_ifindex;
-	__u32 inner_map_fd;
+	union {
+		__u32 inner_map_fd;
+		__u32 btf_vmlinux_value_type_id;
+	};
 };
 LIBBPF_API int

tools/lib/bpf/libbpf.c+640−9

@@ -70,6 +70,13 @@ · 69 unmodified lines
70		70
71	#define __printf(a, b) __attribute__((format(printf, a, b)))	71	#define __printf(a, b) __attribute__((format(printf, a, b)))
72		72
		73	static struct btf *bpf_find_kernel_btf(void);
		74	static struct bpf_map bpf_object__add_map(struct bpf_object obj);
		75	static struct bpf_program bpf_object__find_prog_by_idx(struct bpf_object obj,
		76	int idx);
		77	static const struct btf_type *
		78	skip_mods_and_typedefs(const struct btf btf, __u32 id, __u32 res_id);
		79
73	static int __base_pr(enum libbpf_print_level level, const char *format,	80	static int __base_pr(enum libbpf_print_level level, const char *format,
74	va_list args)	81	va_list args)
75	{	82	{
@@ -229,10 +236,32 @@ struct bpf_program { · 153 unmodified lines
229	__u32 prog_flags;	236	__u32 prog_flags;
230	};	237	};
231		238
		239	struct bpf_struct_ops {
		240	const char *tname;
		241	const struct btf_type *type;
		242	struct bpf_program **progs;
		243	__u32 *kern_func_off;
		244	/* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
		245	void *data;
		246	/* e.g. struct bpf_struct_ops_tcp_congestion_ops in
		247	* btf_vmlinux's format.
		248	* struct bpf_struct_ops_tcp_congestion_ops {
		249	* [... some other kernel fields ...]
		250	* struct tcp_congestion_ops data;
		251	* }
		252	* kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
		253	* bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
		254	* from "data".
		255	*/
		256	void *kern_vdata;
		257	__u32 type_id;
		258	};
		259
232	#define DATA_SEC ".data"	260	#define DATA_SEC ".data"
233	#define BSS_SEC ".bss"	261	#define BSS_SEC ".bss"
234	#define RODATA_SEC ".rodata"	262	#define RODATA_SEC ".rodata"
235	#define KCONFIG_SEC ".kconfig"	263	#define KCONFIG_SEC ".kconfig"
		264	#define STRUCT_OPS_SEC ".struct_ops"
236		265
237	enum libbpf_map_type {	266	enum libbpf_map_type {
238	LIBBPF_MAP_UNSPEC,	267	LIBBPF_MAP_UNSPEC,
@@ -259,10 +288,12 @@ struct bpf_map { · 20 unmodified lines
259	struct bpf_map_def def;	288	struct bpf_map_def def;
260	__u32 btf_key_type_id;	289	__u32 btf_key_type_id;
261	__u32 btf_value_type_id;	290	__u32 btf_value_type_id;
		291	__u32 btf_vmlinux_value_type_id;
262	void *priv;	292	void *priv;
263	bpf_map_clear_priv_t clear_priv;	293	bpf_map_clear_priv_t clear_priv;
264	enum libbpf_map_type libbpf_type;	294	enum libbpf_map_type libbpf_type;
265	void *mmaped;	295	void *mmaped;
		296	struct bpf_struct_ops *st_ops;
266	char *pin_path;	297	char *pin_path;
267	bool pinned;	298	bool pinned;
268	bool reused;	299	bool reused;
@@ -326,6 +357,7 @@ struct bpf_object { · 57 unmodified lines
326	Elf_Data *data;	357	Elf_Data *data;
327	Elf_Data *rodata;	358	Elf_Data *rodata;
328	Elf_Data *bss;	359	Elf_Data *bss;
		360	Elf_Data *st_ops_data;
329	size_t strtabidx;	361	size_t strtabidx;
330	struct {	362	struct {
331	GElf_Shdr shdr;	363	GElf_Shdr shdr;
@@ -339,6 +371,7 @@ struct bpf_object { · 7 unmodified lines
339	int data_shndx;	371	int data_shndx;
340	int rodata_shndx;	372	int rodata_shndx;
341	int bss_shndx;	373	int bss_shndx;
		374	int st_ops_shndx;
342	} efile;	375	} efile;
343	/*	376	/*
344	* All loaded bpf_object is linked in a list, which is	377	* All loaded bpf_object is linked in a list, which is
@@ -566,6 +599,359 @@ static __u32 get_kernel_version(void) · 221 unmodified lines
566	return KERNEL_VERSION(major, minor, patch);	599	return KERNEL_VERSION(major, minor, patch);
567	}	600	}
568		601
		602	static const struct btf_member *
		603	find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
		604	{
		605	struct btf_member *m;
		606	int i;
		607
		608	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
		609	if (btf_member_bit_offset(t, i) == bit_offset)
		610	return m;
		611	}
		612
		613	return NULL;
		614	}
		615
		616	static const struct btf_member *
		617	find_member_by_name(const struct btf btf, const struct btf_type t,
		618	const char *name)
		619	{
		620	struct btf_member *m;
		621	int i;
		622
		623	for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
		624	if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
		625	return m;
		626	}
		627
		628	return NULL;
		629	}
		630
		631	#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
		632	#define STRUCT_OPS_VALUE_PREFIX_LEN (sizeof(STRUCT_OPS_VALUE_PREFIX) - 1)
		633
		634	static int
		635	find_struct_ops_kern_types(const struct btf btf, const char tname,
		636	const struct btf_type *type, __u32 type_id,
		637	const struct btf_type *vtype, __u32 vtype_id,
		638	const struct btf_member **data_member)
		639	{
		640	const struct btf_type kern_type, kern_vtype;
		641	const struct btf_member *kern_data_member;
		642	__s32 kern_vtype_id, kern_type_id;
		643	char vtname[128] = STRUCT_OPS_VALUE_PREFIX;
		644	__u32 i;
		645
		646	kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
		647	if (kern_type_id < 0) {
		648	pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
		649	tname);
		650	return kern_type_id;
		651	}
		652	kern_type = btf__type_by_id(btf, kern_type_id);
		653
		654	/* Find the corresponding "map_value" type that will be used
		655	* in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example,
		656	* find "struct bpf_struct_ops_tcp_congestion_ops" from the
		657	* btf_vmlinux.
		658	*/
		659	strncat(vtname + STRUCT_OPS_VALUE_PREFIX_LEN, tname,
		660	sizeof(vtname) - STRUCT_OPS_VALUE_PREFIX_LEN - 1);
		661	kern_vtype_id = btf__find_by_name_kind(btf, vtname,
		662	BTF_KIND_STRUCT);
		663	if (kern_vtype_id < 0) {
		664	pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
		665	vtname);
		666	return kern_vtype_id;
		667	}
		668	kern_vtype = btf__type_by_id(btf, kern_vtype_id);
		669
		670	/* Find "struct tcp_congestion_ops" from
		671	* struct bpf_struct_ops_tcp_congestion_ops {
		672	* [ ... ]
		673	* struct tcp_congestion_ops data;
		674	* }
		675	*/
		676	kern_data_member = btf_members(kern_vtype);
		677	for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
		678	if (kern_data_member->type == kern_type_id)
		679	break;
		680	}
		681	if (i == btf_vlen(kern_vtype)) {
		682	pr_warn("struct_ops init_kern: struct %s data is not found in struct %s\n",
		683	tname, vtname);
		684	return -EINVAL;
		685	}
		686
		687	*type = kern_type;
		688	*type_id = kern_type_id;
		689	*vtype = kern_vtype;
		690	*vtype_id = kern_vtype_id;
		691	*data_member = kern_data_member;
		692
		693	return 0;
		694	}
		695
		696	static bool bpf_map__is_struct_ops(const struct bpf_map *map)
		697	{
		698	return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
		699	}
		700
		701	/* Init the map's fields that depend on kern_btf */
		702	static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
		703	const struct btf *btf,
		704	const struct btf *kern_btf)
		705	{
		706	const struct btf_member member, kern_member, *kern_data_member;
		707	const struct btf_type type, kern_type, *kern_vtype;
		708	__u32 i, kern_type_id, kern_vtype_id, kern_data_off;
		709	struct bpf_struct_ops *st_ops;
		710	void data, kern_data;
		711	const char *tname;
		712	int err;
		713
		714	st_ops = map->st_ops;
		715	type = st_ops->type;
		716	tname = st_ops->tname;
		717	err = find_struct_ops_kern_types(kern_btf, tname,
		718	&kern_type, &kern_type_id,
		719	&kern_vtype, &kern_vtype_id,
		720	&kern_data_member);
		721	if (err)
		722	return err;
		723
		724	pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
		725	map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
		726
		727	map->def.value_size = kern_vtype->size;
		728	map->btf_vmlinux_value_type_id = kern_vtype_id;
		729
		730	st_ops->kern_vdata = calloc(1, kern_vtype->size);
		731	if (!st_ops->kern_vdata)
		732	return -ENOMEM;
		733
		734	data = st_ops->data;
		735	kern_data_off = kern_data_member->offset / 8;
		736	kern_data = st_ops->kern_vdata + kern_data_off;
		737
		738	member = btf_members(type);
		739	for (i = 0; i < btf_vlen(type); i++, member++) {
		740	const struct btf_type mtype, kern_mtype;
		741	__u32 mtype_id, kern_mtype_id;
		742	void mdata, kern_mdata;
		743	__s64 msize, kern_msize;
		744	__u32 moff, kern_moff;
		745	__u32 kern_member_idx;
		746	const char *mname;
		747
		748	mname = btf__name_by_offset(btf, member->name_off);
		749	kern_member = find_member_by_name(kern_btf, kern_type, mname);
		750	if (!kern_member) {
		751	pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
		752	map->name, mname);
		753	return -ENOTSUP;
		754	}
		755
		756	kern_member_idx = kern_member - btf_members(kern_type);
		757	if (btf_member_bitfield_size(type, i) \|\|
		758	btf_member_bitfield_size(kern_type, kern_member_idx)) {
		759	pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
		760	map->name, mname);
		761	return -ENOTSUP;
		762	}
		763
		764	moff = member->offset / 8;
		765	kern_moff = kern_member->offset / 8;
		766
		767	mdata = data + moff;
		768	kern_mdata = kern_data + kern_moff;
		769
		770	mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
		771	kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
		772	&kern_mtype_id);
		773	if (BTF_INFO_KIND(mtype->info) !=
		774	BTF_INFO_KIND(kern_mtype->info)) {
		775	pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
		776	map->name, mname, BTF_INFO_KIND(mtype->info),
		777	BTF_INFO_KIND(kern_mtype->info));
		778	return -ENOTSUP;
		779	}
		780
		781	if (btf_is_ptr(mtype)) {
		782	struct bpf_program *prog;
		783
		784	mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
		785	kern_mtype = skip_mods_and_typedefs(kern_btf,
		786	kern_mtype->type,
		787	&kern_mtype_id);
		788	if (!btf_is_func_proto(mtype) \|\|
		789	!btf_is_func_proto(kern_mtype)) {
		790	pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
		791	map->name, mname);
		792	return -ENOTSUP;
		793	}
		794
		795	prog = st_ops->progs[i];
		796	if (!prog) {
		797	pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
		798	map->name, mname);
		799	continue;
		800	}
		801
		802	prog->attach_btf_id = kern_type_id;
		803	prog->expected_attach_type = kern_member_idx;
		804
		805	st_ops->kern_func_off[i] = kern_data_off + kern_moff;
		806
		807	pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
		808	map->name, mname, prog->name, moff,
		809	kern_moff);
		810
		811	continue;
		812	}
		813
		814	msize = btf__resolve_size(btf, mtype_id);
		815	kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
		816	if (msize < 0 \|\| kern_msize < 0 \|\| msize != kern_msize) {
		817	pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
		818	map->name, mname, (ssize_t)msize,
		819	(ssize_t)kern_msize);
		820	return -ENOTSUP;
		821	}
		822
		823	pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
		824	map->name, mname, (unsigned int)msize,
		825	moff, kern_moff);
		826	memcpy(kern_mdata, mdata, msize);
		827	}
		828
		829	return 0;
		830	}
		831
		832	static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
		833	{
		834	struct btf *kern_btf = NULL;
		835	struct bpf_map *map;
		836	size_t i;
		837	int err;
		838
		839	for (i = 0; i < obj->nr_maps; i++) {
		840	map = &obj->maps[i];
		841
		842	if (!bpf_map__is_struct_ops(map))
		843	continue;
		844
		845	if (!kern_btf) {
		846	kern_btf = bpf_find_kernel_btf();
		847	if (IS_ERR(kern_btf))
		848	return PTR_ERR(kern_btf);
		849	}
		850
		851	err = bpf_map__init_kern_struct_ops(map, obj->btf, kern_btf);
		852	if (err) {
		853	btf__free(kern_btf);
		854	return err;
		855	}
		856	}
		857
		858	btf__free(kern_btf);
		859	return 0;
		860	}
		861
		862	static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
		863	{
		864	const struct btf_type type, datasec;
		865	const struct btf_var_secinfo *vsi;
		866	struct bpf_struct_ops *st_ops;
		867	const char tname, var_name;
		868	__s32 type_id, datasec_id;
		869	const struct btf *btf;
		870	struct bpf_map *map;
		871	__u32 i;
		872
		873	if (obj->efile.st_ops_shndx == -1)
		874	return 0;
		875
		876	btf = obj->btf;
		877	datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
		878	BTF_KIND_DATASEC);
		879	if (datasec_id < 0) {
		880	pr_warn("struct_ops init: DATASEC %s not found\n",
		881	STRUCT_OPS_SEC);
		882	return -EINVAL;
		883	}
		884
		885	datasec = btf__type_by_id(btf, datasec_id);
		886	vsi = btf_var_secinfos(datasec);
		887	for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
		888	type = btf__type_by_id(obj->btf, vsi->type);
		889	var_name = btf__name_by_offset(obj->btf, type->name_off);
		890
		891	type_id = btf__resolve_type(obj->btf, vsi->type);
		892	if (type_id < 0) {
		893	pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
		894	vsi->type, STRUCT_OPS_SEC);
		895	return -EINVAL;
		896	}
		897
		898	type = btf__type_by_id(obj->btf, type_id);
		899	tname = btf__name_by_offset(obj->btf, type->name_off);
		900	if (!tname[0]) {
		901	pr_warn("struct_ops init: anonymous type is not supported\n");
		902	return -ENOTSUP;
		903	}
		904	if (!btf_is_struct(type)) {
		905	pr_warn("struct_ops init: %s is not a struct\n", tname);
		906	return -EINVAL;
		907	}
		908
		909	map = bpf_object__add_map(obj);
		910	if (IS_ERR(map))
		911	return PTR_ERR(map);
		912
		913	map->sec_idx = obj->efile.st_ops_shndx;
		914	map->sec_offset = vsi->offset;
		915	map->name = strdup(var_name);
		916	if (!map->name)
		917	return -ENOMEM;
		918
		919	map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
		920	map->def.key_size = sizeof(int);
		921	map->def.value_size = type->size;
		922	map->def.max_entries = 1;
		923
		924	map->st_ops = calloc(1, sizeof(*map->st_ops));
		925	if (!map->st_ops)
		926	return -ENOMEM;
		927	st_ops = map->st_ops;
		928	st_ops->data = malloc(type->size);
		929	st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
		930	st_ops->kern_func_off = malloc(btf_vlen(type) *
		931	sizeof(*st_ops->kern_func_off));
		932	if (!st_ops->data \|\| !st_ops->progs \|\| !st_ops->kern_func_off)
		933	return -ENOMEM;
		934
		935	if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
		936	pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
		937	var_name, STRUCT_OPS_SEC);
		938	return -EINVAL;
		939	}
		940
		941	memcpy(st_ops->data,
		942	obj->efile.st_ops_data->d_buf + vsi->offset,
		943	type->size);
		944	st_ops->tname = tname;
		945	st_ops->type = type;
		946	st_ops->type_id = type_id;
		947
		948	pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
		949	tname, type_id, var_name, vsi->offset);
		950	}
		951
		952	return 0;
		953	}
		954
569	static struct bpf_object bpf_object__new(const char path,	955	static struct bpf_object bpf_object__new(const char path,
570	const void *obj_buf,	956	const void *obj_buf,
571	size_t obj_buf_sz,	957	size_t obj_buf_sz,
@@ -607,6 +993,7 @@ static struct bpf_object bpf_object__new(const char path, · 35 unmodified lines
607	obj->efile.data_shndx = -1;	993	obj->efile.data_shndx = -1;
608	obj->efile.rodata_shndx = -1;	994	obj->efile.rodata_shndx = -1;
609	obj->efile.bss_shndx = -1;	995	obj->efile.bss_shndx = -1;
		996	obj->efile.st_ops_shndx = -1;
610	obj->kconfig_map_idx = -1;	997	obj->kconfig_map_idx = -1;
611		998
612	obj->kern_version = get_kernel_version();	999	obj->kern_version = get_kernel_version();
@@ -630,6 +1017,7 @@ static void bpf_object__elf_finish(struct bpf_object *obj) · 17 unmodified lines
630	obj->efile.data = NULL;	1017	obj->efile.data = NULL;
631	obj->efile.rodata = NULL;	1018	obj->efile.rodata = NULL;
632	obj->efile.bss = NULL;	1019	obj->efile.bss = NULL;
		1020	obj->efile.st_ops_data = NULL;
633		1021
634	zfree(&obj->efile.reloc_sects);	1022	zfree(&obj->efile.reloc_sects);
635	obj->efile.nr_reloc_sects = 0;	1023	obj->efile.nr_reloc_sects = 0;
@@ -815,6 +1203,9 @@ int bpf_object__section_size(const struct bpf_object obj, const char name, · 179 unmodified lines
815	} else if (!strcmp(name, RODATA_SEC)) {	1203	} else if (!strcmp(name, RODATA_SEC)) {
816	if (obj->efile.rodata)	1204	if (obj->efile.rodata)
817	*size = obj->efile.rodata->d_size;	1205	*size = obj->efile.rodata->d_size;
		1206	} else if (!strcmp(name, STRUCT_OPS_SEC)) {
		1207	if (obj->efile.st_ops_data)
		1208	*size = obj->efile.st_ops_data->d_size;
818	} else {	1209	} else {
819	ret = bpf_object_search_section_size(obj, name, &d_size);	1210	ret = bpf_object_search_section_size(obj, name, &d_size);
820	if (!ret)	1211	if (!ret)
@@ -1440,6 +1831,20 @@ skip_mods_and_typedefs(const struct btf btf, __u32 id, __u32 res_id) · 619 unmodified lines
1440	return t;	1831	return t;
1441	}	1832	}
1442		1833
		1834	static const struct btf_type *
		1835	resolve_func_ptr(const struct btf btf, __u32 id, __u32 res_id)
		1836	{
		1837	const struct btf_type *t;
		1838
		1839	t = skip_mods_and_typedefs(btf, id, NULL);
		1840	if (!btf_is_ptr(t))
		1841	return NULL;
		1842
		1843	t = skip_mods_and_typedefs(btf, t->type, res_id);
		1844
		1845	return btf_is_func_proto(t) ? t : NULL;
		1846	}
		1847
1443	/*	1848	/*
1444	* Fetch integer attribute of BTF map definition. Such attributes are	1849	* Fetch integer attribute of BTF map definition. Such attributes are
1445	* represented using a pointer to an array, in which dimensionality of array	1850	* represented using a pointer to an array, in which dimensionality of array
@@ -1787,6 +2192,7 @@ static int bpf_object__init_maps(struct bpf_object *obj, · 341 unmodified lines
1787	err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);	2192	err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
1788	err = err ?: bpf_object__init_global_data_maps(obj);	2193	err = err ?: bpf_object__init_global_data_maps(obj);
1789	err = err ?: bpf_object__init_kconfig_map(obj);	2194	err = err ?: bpf_object__init_kconfig_map(obj);
		2195	err = err ?: bpf_object__init_struct_ops_maps(obj);
1790	if (err)	2196	if (err)
1791	return err;	2197	return err;
1792		2198
@@ -1889,7 +2295,8 @@ static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) · 96 unmodified lines
1889	static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj)	2295	static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj)
1890	{	2296	{
1891	return obj->efile.btf_maps_shndx >= 0 \|\|	2297	return obj->efile.btf_maps_shndx >= 0 \|\|
1892	obj->nr_extern > 0;	2298	obj->efile.st_ops_shndx >= 0 \|\|
		2299	obj->nr_extern > 0;
1893	}	2300	}
1894		2301
1895	static int bpf_object__init_btf(struct bpf_object *obj,	2302	static int bpf_object__init_btf(struct bpf_object *obj,
@@ -2088,6 +2495,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj) · 192 unmodified lines
2088	} else if (strcmp(name, RODATA_SEC) == 0) {	2495	} else if (strcmp(name, RODATA_SEC) == 0) {
2089	obj->efile.rodata = data;	2496	obj->efile.rodata = data;
2090	obj->efile.rodata_shndx = idx;	2497	obj->efile.rodata_shndx = idx;
		2498	} else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
		2499	obj->efile.st_ops_data = data;
		2500	obj->efile.st_ops_shndx = idx;
2091	} else {	2501	} else {
2092	pr_debug("skip section(%d) %s\n", idx, name);	2502	pr_debug("skip section(%d) %s\n", idx, name);
2093	}	2503	}
@@ -2097,7 +2507,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj) · 3 unmodified lines
2097	int sec = sh.sh_info; /* points to other section */	2507	int sec = sh.sh_info; /* points to other section */
2098		2508
2099	/* Only do relo for section with exec instructions */	2509	/* Only do relo for section with exec instructions */
2100	if (!section_have_execinstr(obj, sec)) {	2510	if (!section_have_execinstr(obj, sec) &&
		2511	strcmp(name, ".rel" STRUCT_OPS_SEC)) {
2101	pr_debug("skip relo %s(%d) for section(%d)\n",	2512	pr_debug("skip relo %s(%d) for section(%d)\n",
2102	name, idx, sec);	2513	name, idx, sec);
2103	continue;	2514	continue;
@@ -2599,8 +3010,12 @@ static int bpf_map_find_btf_info(struct bpf_object obj, struct bpf_map map) · 495 unmodified lines
2599	__u32 key_type_id = 0, value_type_id = 0;	3010	__u32 key_type_id = 0, value_type_id = 0;
2600	int ret;	3011	int ret;
2601		3012
2602	/* if it's BTF-defined map, we don't need to search for type IDs */	3013	/* if it's BTF-defined map, we don't need to search for type IDs.
2603	if (map->sec_idx == obj->efile.btf_maps_shndx)	3014	* For struct_ops map, it does not need btf_key_type_id and
		3015	* btf_value_type_id.
		3016	*/
		3017	if (map->sec_idx == obj->efile.btf_maps_shndx \|\|
		3018	bpf_map__is_struct_ops(map))
2604	return 0;	3019	return 0;
2605		3020
2606	if (!bpf_map__is_internal(map)) {	3021	if (!bpf_map__is_internal(map)) {
@@ -3025,6 +3440,9 @@ bpf_object__create_maps(struct bpf_object *obj) · 418 unmodified lines
3025	if (bpf_map_type__is_map_in_map(def->type) &&	3440	if (bpf_map_type__is_map_in_map(def->type) &&
3026	map->inner_map_fd >= 0)	3441	map->inner_map_fd >= 0)
3027	create_attr.inner_map_fd = map->inner_map_fd;	3442	create_attr.inner_map_fd = map->inner_map_fd;
		3443	if (bpf_map__is_struct_ops(map))
		3444	create_attr.btf_vmlinux_value_type_id =
		3445	map->btf_vmlinux_value_type_id;
3028		3446
3029	if (obj->btf && !bpf_map_find_btf_info(obj, map)) {	3447	if (obj->btf && !bpf_map_find_btf_info(obj, map)) {
3030	create_attr.btf_fd = btf__fd(obj->btf);	3448	create_attr.btf_fd = btf__fd(obj->btf);
@@ -3899,7 +4317,7 @@ cleanup: · 868 unmodified lines
3899	* Probe few well-known locations for vmlinux kernel image and try to load BTF	4317	* Probe few well-known locations for vmlinux kernel image and try to load BTF
3900	* data out of it to use for target BTF.	4318	* data out of it to use for target BTF.
3901	*/	4319	*/
3902	static struct btf *bpf_core_find_kernel_btf(void)	4320	static struct btf *bpf_find_kernel_btf(void)
3903	{	4321	{
3904	struct {	4322	struct {
3905	const char *path_fmt;	4323	const char *path_fmt;
@@ -4180,7 +4598,7 @@ bpf_core_reloc_fields(struct bpf_object obj, const char targ_btf_path) · 274 unmodified lines
4180	if (targ_btf_path)	4598	if (targ_btf_path)
4181	targ_btf = btf__parse_elf(targ_btf_path, NULL);	4599	targ_btf = btf__parse_elf(targ_btf_path, NULL);
4182	else	4600	else
4183	targ_btf = bpf_core_find_kernel_btf();	4601	targ_btf = bpf_find_kernel_btf();
4184	if (IS_ERR(targ_btf)) {	4602	if (IS_ERR(targ_btf)) {
4185	pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));	4603	pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
4186	return PTR_ERR(targ_btf);	4604	return PTR_ERR(targ_btf);
@@ -4379,6 +4797,10 @@ bpf_object__relocate(struct bpf_object obj, const char targ_btf_path) · 192 unmodified lines
4379	return 0;	4797	return 0;
4380	}	4798	}
4381		4799
		4800	static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
		4801	GElf_Shdr *shdr,
		4802	Elf_Data *data);
		4803
4382	static int bpf_object__collect_reloc(struct bpf_object *obj)	4804	static int bpf_object__collect_reloc(struct bpf_object *obj)
4383	{	4805	{
4384	int i, err;	4806	int i, err;
@@ -4399,6 +4821,15 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) · 14 unmodified lines
4399	return -LIBBPF_ERRNO__INTERNAL;	4821	return -LIBBPF_ERRNO__INTERNAL;
4400	}	4822	}
4401		4823
		4824	if (idx == obj->efile.st_ops_shndx) {
		4825	err = bpf_object__collect_struct_ops_map_reloc(obj,
		4826	shdr,
		4827	data);
		4828	if (err)
		4829	return err;
		4830	continue;
		4831	}
		4832
4402	prog = bpf_object__find_prog_by_idx(obj, idx);	4833	prog = bpf_object__find_prog_by_idx(obj, idx);
4403	if (!prog) {	4834	if (!prog) {
4404	pr_warn("relocation failed: no section(%d)\n", idx);	4835	pr_warn("relocation failed: no section(%d)\n", idx);
@@ -4433,7 +4864,9 @@ load_program(struct bpf_program prog, struct bpf_insn insns, int insns_cnt, · 28 unmodified lines
4433	load_attr.insns = insns;	4864	load_attr.insns = insns;
4434	load_attr.insns_cnt = insns_cnt;	4865	load_attr.insns_cnt = insns_cnt;
4435	load_attr.license = license;	4866	load_attr.license = license;
4436	if (prog->type == BPF_PROG_TYPE_TRACING) {	4867	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
		4868	load_attr.attach_btf_id = prog->attach_btf_id;
		4869	} else if (prog->type == BPF_PROG_TYPE_TRACING) {
4437	load_attr.attach_prog_fd = prog->attach_prog_fd;	4870	load_attr.attach_prog_fd = prog->attach_prog_fd;
4438	load_attr.attach_btf_id = prog->attach_btf_id;	4871	load_attr.attach_btf_id = prog->attach_btf_id;
4439	} else {	4872	} else {
@@ -4679,6 +5112,9 @@ __bpf_object__open(const char path, const void obj_buf, size_t obj_buf_sz, · 239 unmodified lines
4679	enum bpf_prog_type prog_type;	5112	enum bpf_prog_type prog_type;
4680	enum bpf_attach_type attach_type;	5113	enum bpf_attach_type attach_type;
4681		5114
		5115	if (prog->type != BPF_PROG_TYPE_UNSPEC)
		5116	continue;
		5117
4682	err = libbpf_prog_type_by_name(prog->section_name, &prog_type,	5118	err = libbpf_prog_type_by_name(prog->section_name, &prog_type,
4683	&attach_type);	5119	&attach_type);
4684	if (err == -ESRCH)	5120	if (err == -ESRCH)
@@ -4774,8 +5210,11 @@ int bpf_object__unload(struct bpf_object *obj) · 89 unmodified lines
4774	if (!obj)	5210	if (!obj)
4775	return -EINVAL;	5211	return -EINVAL;
4776		5212
4777	for (i = 0; i < obj->nr_maps; i++)	5213	for (i = 0; i < obj->nr_maps; i++) {
4778	zclose(obj->maps[i].fd);	5214	zclose(obj->maps[i].fd);
		5215	if (obj->maps[i].st_ops)
		5216	zfree(&obj->maps[i].st_ops->kern_vdata);
		5217	}
4779		5218
4780	for (i = 0; i < obj->nr_programs; i++)	5219	for (i = 0; i < obj->nr_programs; i++)
4781	bpf_program__unload(&obj->programs[i]);	5220	bpf_program__unload(&obj->programs[i]);
@@ -4891,6 +5330,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) · 109 unmodified lines
4891	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);	5330	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
4892	err = err ? : bpf_object__sanitize_and_load_btf(obj);	5331	err = err ? : bpf_object__sanitize_and_load_btf(obj);
4893	err = err ? : bpf_object__sanitize_maps(obj);	5332	err = err ? : bpf_object__sanitize_maps(obj);
		5333	err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
4894	err = err ? : bpf_object__create_maps(obj);	5334	err = err ? : bpf_object__create_maps(obj);
4895	err = err ? : bpf_object__relocate(obj, attr->target_btf_path);	5335	err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
4896	err = err ? : bpf_object__load_progs(obj, attr->log_level);	5336	err = err ? : bpf_object__load_progs(obj, attr->log_level);
@@ -5478,6 +5918,13 @@ void bpf_object__close(struct bpf_object *obj) · 581 unmodified lines
5478	map->mmaped = NULL;	5918	map->mmaped = NULL;
5479	}	5919	}
5480		5920
		5921	if (map->st_ops) {
		5922	zfree(&map->st_ops->data);
		5923	zfree(&map->st_ops->progs);
		5924	zfree(&map->st_ops->kern_func_off);
		5925	zfree(&map->st_ops);
		5926	}
		5927
5481	zfree(&map->name);	5928	zfree(&map->name);
5482	zfree(&map->pin_path);	5929	zfree(&map->pin_path);
5483	}	5930	}
@@ -5746,6 +6193,7 @@ BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); · 262 unmodified lines
5746	BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);	6193	BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
5747	BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);	6194	BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
5748	BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);	6195	BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
		6196	BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
5749		6197
5750	enum bpf_attach_type	6198	enum bpf_attach_type
5751	bpf_program__get_expected_attach_type(struct bpf_program *prog)	6199	bpf_program__get_expected_attach_type(struct bpf_program *prog)
@@ -5899,6 +6347,7 @@ static const struct bpf_sec_def section_defs[] = { · 147 unmodified lines
5899	BPF_CGROUP_GETSOCKOPT),	6347	BPF_CGROUP_GETSOCKOPT),
5900	BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,	6348	BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
5901	BPF_CGROUP_SETSOCKOPT),	6349	BPF_CGROUP_SETSOCKOPT),
		6350	BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
5902	};	6351	};
5903		6352
5904	#undef BPF_PROG_SEC_IMPL	6353	#undef BPF_PROG_SEC_IMPL
@@ -5975,11 +6424,141 @@ int libbpf_prog_type_by_name(const char name, enum bpf_prog_type prog_type, · 70 unmodified lines
5975	return -ESRCH;	6424	return -ESRCH;
5976	}	6425	}
5977		6426
		6427	static struct bpf_map find_struct_ops_map_by_offset(struct bpf_object obj,
		6428	size_t offset)
		6429	{
		6430	struct bpf_map *map;
		6431	size_t i;
		6432
		6433	for (i = 0; i < obj->nr_maps; i++) {
		6434	map = &obj->maps[i];
		6435	if (!bpf_map__is_struct_ops(map))
		6436	continue;
		6437	if (map->sec_offset <= offset &&
		6438	offset - map->sec_offset < map->def.value_size)
		6439	return map;
		6440	}
		6441
		6442	return NULL;
		6443	}
		6444
		6445	/* Collect the reloc from ELF and populate the st_ops->progs[] */
		6446	static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
		6447	GElf_Shdr *shdr,
		6448	Elf_Data *data)
		6449	{
		6450	const struct btf_member *member;
		6451	struct bpf_struct_ops *st_ops;
		6452	struct bpf_program *prog;
		6453	unsigned int shdr_idx;
		6454	const struct btf *btf;
		6455	struct bpf_map *map;
		6456	Elf_Data *symbols;
		6457	unsigned int moff;
		6458	const char *name;
		6459	u32 member_idx;
		6460	GElf_Sym sym;
		6461	GElf_Rel rel;
		6462	int i, nrels;
		6463
		6464	symbols = obj->efile.symbols;
		6465	btf = obj->btf;
		6466	nrels = shdr->sh_size / shdr->sh_entsize;
		6467	for (i = 0; i < nrels; i++) {
		6468	if (!gelf_getrel(data, i, &rel)) {
		6469	pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
		6470	return -LIBBPF_ERRNO__FORMAT;
		6471	}
		6472
		6473	if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
		6474	pr_warn("struct_ops reloc: symbol %zx not found\n",
		6475	(size_t)GELF_R_SYM(rel.r_info));
		6476	return -LIBBPF_ERRNO__FORMAT;
		6477	}
		6478
		6479	name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
		6480	sym.st_name) ? : "<?>";
		6481	map = find_struct_ops_map_by_offset(obj, rel.r_offset);
		6482	if (!map) {
		6483	pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
		6484	(size_t)rel.r_offset);
		6485	return -EINVAL;
		6486	}
		6487
		6488	moff = rel.r_offset - map->sec_offset;
		6489	shdr_idx = sym.st_shndx;
		6490	st_ops = map->st_ops;
		6491	pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
		6492	map->name,
		6493	(long long)(rel.r_info >> 32),
		6494	(long long)sym.st_value,
		6495	shdr_idx, (size_t)rel.r_offset,
		6496	map->sec_offset, sym.st_name, name);
		6497
		6498	if (shdr_idx >= SHN_LORESERVE) {
		6499	pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
		6500	map->name, (size_t)rel.r_offset, shdr_idx);
		6501	return -LIBBPF_ERRNO__RELOC;
		6502	}
		6503
		6504	member = find_member_by_offset(st_ops->type, moff * 8);
		6505	if (!member) {
		6506	pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
		6507	map->name, moff);
		6508	return -EINVAL;
		6509	}
		6510	member_idx = member - btf_members(st_ops->type);
		6511	name = btf__name_by_offset(btf, member->name_off);
		6512
		6513	if (!resolve_func_ptr(btf, member->type, NULL)) {
		6514	pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
		6515	map->name, name);
		6516	return -EINVAL;
		6517	}
		6518
		6519	prog = bpf_object__find_prog_by_idx(obj, shdr_idx);
		6520	if (!prog) {
		6521	pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
		6522	map->name, shdr_idx, name);
		6523	return -EINVAL;
		6524	}
		6525
		6526	if (prog->type == BPF_PROG_TYPE_UNSPEC) {
		6527	const struct bpf_sec_def *sec_def;
		6528
		6529	sec_def = find_sec_def(prog->section_name);
		6530	if (sec_def &&
		6531	sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
		6532	/* for pr_warn */
		6533	prog->type = sec_def->prog_type;
		6534	goto invalid_prog;
		6535	}
		6536
		6537	prog->type = BPF_PROG_TYPE_STRUCT_OPS;
		6538	prog->attach_btf_id = st_ops->type_id;
		6539	prog->expected_attach_type = member_idx;
		6540	} else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS \|\|
		6541	prog->attach_btf_id != st_ops->type_id \|\|
		6542	prog->expected_attach_type != member_idx) {
		6543	goto invalid_prog;
		6544	}
		6545	st_ops->progs[member_idx] = prog;
		6546	}
		6547
		6548	return 0;
		6549
		6550	invalid_prog:
		6551	pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
		6552	map->name, prog->name, prog->section_name, prog->type,
		6553	prog->attach_btf_id, prog->expected_attach_type, name);
		6554	return -EINVAL;
		6555	}
		6556
5978	#define BTF_PREFIX "btf_trace_"	6557	#define BTF_PREFIX "btf_trace_"
5979	int libbpf_find_vmlinux_btf_id(const char *name,	6558	int libbpf_find_vmlinux_btf_id(const char *name,
5980	enum bpf_attach_type attach_type)	6559	enum bpf_attach_type attach_type)
5981	{	6560	{
5982	struct btf *btf = bpf_core_find_kernel_btf();	6561	struct btf *btf = bpf_find_kernel_btf();
5983	char raw_tp_btf[128] = BTF_PREFIX;	6562	char raw_tp_btf[128] = BTF_PREFIX;
5984	char *dst = raw_tp_btf + sizeof(BTF_PREFIX) - 1;	6563	char *dst = raw_tp_btf + sizeof(BTF_PREFIX) - 1;
5985	const char *btf_name;	6564	const char *btf_name;
@@ -6805,6 +7384,58 @@ struct bpf_link bpf_program__attach(struct bpf_program prog) · 819 unmodified lines
6805	return sec_def->attach_fn(sec_def, prog);	7384	return sec_def->attach_fn(sec_def, prog);
6806	}	7385	}
6807		7386
		7387	static int bpf_link__detach_struct_ops(struct bpf_link *link)
		7388	{
		7389	struct bpf_link_fd l = (void )link;
		7390	__u32 zero = 0;
		7391
		7392	if (bpf_map_delete_elem(l->fd, &zero))
		7393	return -errno;
		7394
		7395	return 0;
		7396	}
		7397
		7398	struct bpf_link bpf_map__attach_struct_ops(struct bpf_map map)
		7399	{
		7400	struct bpf_struct_ops *st_ops;
		7401	struct bpf_link_fd *link;
		7402	__u32 i, zero = 0;
		7403	int err;
		7404
		7405	if (!bpf_map__is_struct_ops(map) \|\| map->fd == -1)
		7406	return ERR_PTR(-EINVAL);
		7407
		7408	link = calloc(1, sizeof(*link));
		7409	if (!link)
		7410	return ERR_PTR(-EINVAL);
		7411
		7412	st_ops = map->st_ops;
		7413	for (i = 0; i < btf_vlen(st_ops->type); i++) {
		7414	struct bpf_program *prog = st_ops->progs[i];
		7415	void *kern_data;
		7416	int prog_fd;
		7417
		7418	if (!prog)
		7419	continue;
		7420
		7421	prog_fd = bpf_program__fd(prog);
		7422	kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
		7423	(unsigned long )kern_data = prog_fd;
		7424	}
		7425
		7426	err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
		7427	if (err) {
		7428	err = -errno;
		7429	free(link);
		7430	return ERR_PTR(err);
		7431	}
		7432
		7433	link->link.detach = bpf_link__detach_struct_ops;
		7434	link->fd = map->fd;
		7435
		7436	return (struct bpf_link *)link;
		7437	}
		7438
6808	enum bpf_perf_event_ret	7439	enum bpf_perf_event_ret
6809	bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,	7440	bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
6810	void *copy_mem, size_t copy_size,	7441	void *copy_mem, size_t copy_size,

tools/lib/bpf/libbpf.h+4−1

 LIBBPF_API struct bpf_link *
 bpf_program__attach_trace(struct bpf_program *prog);
+struct bpf_map;
+LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map);
 struct bpf_insn;
 /*
 LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog);
 LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog);
 LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog);
+LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog);
 LIBBPF_API enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog);
 LIBBPF_API void bpf_program__set_type(struct bpf_program *prog,
 LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog);
 LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog);
 LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog);
+LIBBPF_API bool bpf_program__is_struct_ops(const struct bpf_program *prog);
 /*
  * No need for __attribute__((packed)), all members of 'bpf_map_def'
  * The 'struct bpf_map' in include/linux/bpf.h is internal to the kernel,
  * so no need to worry about a name clash.
  */
-struct bpf_map;
 LIBBPF_API struct bpf_map *
 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name);

tools/lib/bpf/libbpf.map+3−0

 	global:
 		btf_dump__emit_type_decl;
 		bpf_link__disconnect;
+		bpf_map__attach_struct_ops;
 		bpf_object__find_program_by_name;
 		bpf_object__attach_skeleton;
 		bpf_object__destroy_skeleton;
 		bpf_prog_attach_xattr;
 		bpf_program__attach;
 		bpf_program__name;
+		bpf_program__is_struct_ops;
+		bpf_program__set_struct_ops;
 		btf__align_of;
 } LIBBPF_0.0.6;

tools/lib/bpf/libbpf_probes.c+2−0

 	case BPF_PROG_TYPE_CGROUP_SYSCTL:
 	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
 	case BPF_PROG_TYPE_TRACING:
+	case BPF_PROG_TYPE_STRUCT_OPS:
 	default:
 		break;
 	}
 	case BPF_MAP_TYPE_XSKMAP:
 	case BPF_MAP_TYPE_SOCKHASH:
 	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
+	case BPF_MAP_TYPE_STRUCT_OPS:
 	default:
 		break;
 	}

tools/testing/selftests/bpf/bpf_tcp_helpers.h+228−0

@@ -0,0 +1,228 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2	#ifndef __BPF_TCP_HELPERS_H
	3	#define __BPF_TCP_HELPERS_H
	4
	5	#include <stdbool.h>
	6	#include <linux/types.h>
	7	#include <bpf_helpers.h>
	8	#include <bpf_core_read.h>
	9	#include "bpf_trace_helpers.h"
	10
	11	/* "struct_ops/" is only a convention. not a requirement. */
	12	#define BPF_TCP_OPS_0(fname, ret_type, ...) BPF_TRACE_x(0, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	13	#define BPF_TCP_OPS_1(fname, ret_type, ...) BPF_TRACE_x(1, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	14	#define BPF_TCP_OPS_2(fname, ret_type, ...) BPF_TRACE_x(2, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	15	#define BPF_TCP_OPS_3(fname, ret_type, ...) BPF_TRACE_x(3, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	16	#define BPF_TCP_OPS_4(fname, ret_type, ...) BPF_TRACE_x(4, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	17	#define BPF_TCP_OPS_5(fname, ret_type, ...) BPF_TRACE_x(5, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	18
	19	struct sock_common {
	20	unsigned char skc_state;
	21	} __attribute__((preserve_access_index));
	22
	23	struct sock {
	24	struct sock_common __sk_common;
	25	} __attribute__((preserve_access_index));
	26
	27	struct inet_sock {
	28	struct sock sk;
	29	} __attribute__((preserve_access_index));
	30
	31	struct inet_connection_sock {
	32	struct inet_sock icsk_inet;
	33	__u8 icsk_ca_state:6,
	34	icsk_ca_setsockopt:1,
	35	icsk_ca_dst_locked:1;
	36	struct {
	37	__u8 pending;
	38	} icsk_ack;
	39	__u64 icsk_ca_priv[104 / sizeof(__u64)];
	40	} __attribute__((preserve_access_index));
	41
	42	struct tcp_sock {
	43	struct inet_connection_sock inet_conn;
	44
	45	__u32 rcv_nxt;
	46	__u32 snd_nxt;
	47	__u32 snd_una;
	48	__u8 ecn_flags;
	49	__u32 delivered;
	50	__u32 delivered_ce;
	51	__u32 snd_cwnd;
	52	__u32 snd_cwnd_cnt;
	53	__u32 snd_cwnd_clamp;
	54	__u32 snd_ssthresh;
	55	__u8 syn_data:1, /* SYN includes data */
	56	syn_fastopen:1, /* SYN includes Fast Open option */
	57	syn_fastopen_exp:1,/* SYN includes Fast Open exp. option */
	58	syn_fastopen_ch:1, /* Active TFO re-enabling probe */
	59	syn_data_acked:1,/* data in SYN is acked by SYN-ACK */
	60	save_syn:1, /* Save headers of SYN packet */
	61	is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */
	62	syn_smc:1; /* SYN includes SMC */
	63	__u32 max_packets_out;
	64	__u32 lsndtime;
	65	__u32 prior_cwnd;
	66	} __attribute__((preserve_access_index));
	67
	68	static __always_inline struct inet_connection_sock inet_csk(const struct sock sk)
	69	{
	70	return (struct inet_connection_sock *)sk;
	71	}
	72
	73	static __always_inline void inet_csk_ca(const struct sock sk)
	74	{
	75	return (void *)inet_csk(sk)->icsk_ca_priv;
	76	}
	77
	78	static __always_inline struct tcp_sock tcp_sk(const struct sock sk)
	79	{
	80	return (struct tcp_sock *)sk;
	81	}
	82
	83	static __always_inline bool before(__u32 seq1, __u32 seq2)
	84	{
	85	return (__s32)(seq1-seq2) < 0;
	86	}
	87	#define after(seq2, seq1) before(seq1, seq2)
	88
	89	#define TCP_ECN_OK 1
	90	#define TCP_ECN_QUEUE_CWR 2
	91	#define TCP_ECN_DEMAND_CWR 4
	92	#define TCP_ECN_SEEN 8
	93
	94	enum inet_csk_ack_state_t {
	95	ICSK_ACK_SCHED = 1,
	96	ICSK_ACK_TIMER = 2,
	97	ICSK_ACK_PUSHED = 4,
	98	ICSK_ACK_PUSHED2 = 8,
	99	ICSK_ACK_NOW = 16 /* Send the next ACK immediately (once) */
	100	};
	101
	102	enum tcp_ca_event {
	103	CA_EVENT_TX_START = 0,
	104	CA_EVENT_CWND_RESTART = 1,
	105	CA_EVENT_COMPLETE_CWR = 2,
	106	CA_EVENT_LOSS = 3,
	107	CA_EVENT_ECN_NO_CE = 4,
	108	CA_EVENT_ECN_IS_CE = 5,
	109	};
	110
	111	enum tcp_ca_state {
	112	TCP_CA_Open = 0,
	113	TCP_CA_Disorder = 1,
	114	TCP_CA_CWR = 2,
	115	TCP_CA_Recovery = 3,
	116	TCP_CA_Loss = 4
	117	};
	118
	119	struct ack_sample {
	120	__u32 pkts_acked;
	121	__s32 rtt_us;
	122	__u32 in_flight;
	123	} __attribute__((preserve_access_index));
	124
	125	struct rate_sample {
	126	__u64 prior_mstamp; /* starting timestamp for interval */
	127	__u32 prior_delivered; /* tp->delivered at "prior_mstamp" */
	128	__s32 delivered; /* number of packets delivered over interval */
	129	long interval_us; /* time for tp->delivered to incr "delivered" */
	130	__u32 snd_interval_us; /* snd interval for delivered packets */
	131	__u32 rcv_interval_us; /* rcv interval for delivered packets */
	132	long rtt_us; /* RTT of last (S)ACKed packet (or -1) */
	133	int losses; /* number of packets marked lost upon ACK */
	134	__u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */
	135	__u32 prior_in_flight; /* in flight before this ACK */
	136	bool is_app_limited; /* is sample from packet with bubble in pipe? */
	137	bool is_retrans; /* is sample from retransmission? */
	138	bool is_ack_delayed; /* is this (likely) a delayed ACK? */
	139	} __attribute__((preserve_access_index));
	140
	141	#define TCP_CA_NAME_MAX 16
	142	#define TCP_CONG_NEEDS_ECN 0x2
	143
	144	struct tcp_congestion_ops {
	145	char name[TCP_CA_NAME_MAX];
	146	__u32 flags;
	147
	148	/* initialize private data (optional) */
	149	void (init)(struct sock sk);
	150	/* cleanup private data (optional) */
	151	void (release)(struct sock sk);
	152
	153	/* return slow start threshold (required) */
	154	__u32 (ssthresh)(struct sock sk);
	155	/* do new cwnd calculation (required) */
	156	void (cong_avoid)(struct sock sk, __u32 ack, __u32 acked);
	157	/* call before changing ca_state (optional) */
	158	void (set_state)(struct sock sk, __u8 new_state);
	159	/* call when cwnd event occurs (optional) */
	160	void (cwnd_event)(struct sock sk, enum tcp_ca_event ev);
	161	/* call when ack arrives (optional) */
	162	void (in_ack_event)(struct sock sk, __u32 flags);
	163	/* new value of cwnd after loss (required) */
	164	__u32 (undo_cwnd)(struct sock sk);
	165	/* hook for packet ack accounting (optional) */
	166	void (pkts_acked)(struct sock sk, const struct ack_sample *sample);
	167	/* override sysctl_tcp_min_tso_segs */
	168	__u32 (min_tso_segs)(struct sock sk);
	169	/* returns the multiplier used in tcp_sndbuf_expand (optional) */
	170	__u32 (sndbuf_expand)(struct sock sk);
	171	/* call when packets are delivered to update cwnd and pacing rate,
	172	* after all the ca_state processing. (optional)
	173	*/
	174	void (cong_control)(struct sock sk, const struct rate_sample *rs);
	175	};
	176
	177	#define min(a, b) ((a) < (b) ? (a) : (b))
	178	#define max(a, b) ((a) > (b) ? (a) : (b))
	179	#define min_not_zero(x, y) ({ \
	180	typeof(x) __x = (x); \
	181	typeof(y) __y = (y); \
	182	__x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
	183
	184	static __always_inline __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked)
	185	{
	186	__u32 cwnd = min(tp->snd_cwnd + acked, tp->snd_ssthresh);
	187
	188	acked -= cwnd - tp->snd_cwnd;
	189	tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp);
	190
	191	return acked;
	192	}
	193
	194	static __always_inline bool tcp_in_slow_start(const struct tcp_sock *tp)
	195	{
	196	return tp->snd_cwnd < tp->snd_ssthresh;
	197	}
	198
	199	static __always_inline bool tcp_is_cwnd_limited(const struct sock *sk)
	200	{
	201	const struct tcp_sock *tp = tcp_sk(sk);
	202
	203	/* If in slow start, ensure cwnd grows to twice what was ACKed. */
	204	if (tcp_in_slow_start(tp))
	205	return tp->snd_cwnd < 2 * tp->max_packets_out;
	206
	207	return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);
	208	}
	209
	210	static __always_inline void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked)
	211	{
	212	/* If credits accumulated at a higher w, apply them gently now. */
	213	if (tp->snd_cwnd_cnt >= w) {
	214	tp->snd_cwnd_cnt = 0;
	215	tp->snd_cwnd++;
	216	}
	217
	218	tp->snd_cwnd_cnt += acked;
	219	if (tp->snd_cwnd_cnt >= w) {
	220	__u32 delta = tp->snd_cwnd_cnt / w;
	221
	222	tp->snd_cwnd_cnt -= delta * w;
	223	tp->snd_cwnd += delta;
	224	}
	225	tp->snd_cwnd = min(tp->snd_cwnd, tp->snd_cwnd_clamp);
	226	}
	227
	228	#endif

tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+187−0

@@ -0,0 +1,187 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2019 Facebook */
	3
	4	#include <linux/err.h>
	5	#include <test_progs.h>
	6	#include "bpf_dctcp.skel.h"
	7
	8	#define min(a, b) ((a) < (b) ? (a) : (b))
	9
	10	static const unsigned int total_bytes = 10 * 1024 * 1024;
	11	static const struct timeval timeo_sec = { .tv_sec = 10 };
	12	static const size_t timeo_optlen = sizeof(timeo_sec);
	13	static int stop, duration;
	14
	15	static int settimeo(int fd)
	16	{
	17	int err;
	18
	19	err = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeo_sec,
	20	timeo_optlen);
	21	if (CHECK(err == -1, "setsockopt(fd, SO_RCVTIMEO)", "errno:%d\n",
	22	errno))
	23	return -1;
	24
	25	err = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeo_sec,
	26	timeo_optlen);
	27	if (CHECK(err == -1, "setsockopt(fd, SO_SNDTIMEO)", "errno:%d\n",
	28	errno))
	29	return -1;
	30
	31	return 0;
	32	}
	33
	34	static int settcpca(int fd, const char *tcp_ca)
	35	{
	36	int err;
	37
	38	err = setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, tcp_ca, strlen(tcp_ca));
	39	if (CHECK(err == -1, "setsockopt(fd, TCP_CONGESTION)", "errno:%d\n",
	40	errno))
	41	return -1;
	42
	43	return 0;
	44	}
	45
	46	static void server(void arg)
	47	{
	48	int lfd = (int)(long)arg, err = 0, fd;
	49	ssize_t nr_sent = 0, bytes = 0;
	50	char batch[1500];
	51
	52	fd = accept(lfd, NULL, NULL);
	53	while (fd == -1) {
	54	if (errno == EINTR)
	55	continue;
	56	err = -errno;
	57	goto done;
	58	}
	59
	60	if (settimeo(fd)) {
	61	err = -errno;
	62	goto done;
	63	}
	64
	65	while (bytes < total_bytes && !READ_ONCE(stop)) {
	66	nr_sent = send(fd, &batch,
	67	min(total_bytes - bytes, sizeof(batch)), 0);
	68	if (nr_sent == -1 && errno == EINTR)
	69	continue;
	70	if (nr_sent == -1) {
	71	err = -errno;
	72	break;
	73	}
	74	bytes += nr_sent;
	75	}
	76
	77	CHECK(bytes != total_bytes, "send", "%zd != %u nr_sent:%zd errno:%d\n",
	78	bytes, total_bytes, nr_sent, errno);
	79
	80	done:
	81	if (fd != -1)
	82	close(fd);
	83	if (err) {
	84	WRITE_ONCE(stop, 1);
	85	return ERR_PTR(err);
	86	}
	87	return NULL;
	88	}
	89
	90	static void do_test(const char *tcp_ca)
	91	{
	92	struct sockaddr_in6 sa6 = {};
	93	ssize_t nr_recv = 0, bytes = 0;
	94	int lfd = -1, fd = -1;
	95	pthread_t srv_thread;
	96	socklen_t addrlen = sizeof(sa6);
	97	void *thread_ret;
	98	char batch[1500];
	99	int err;
	100
	101	WRITE_ONCE(stop, 0);
	102
	103	lfd = socket(AF_INET6, SOCK_STREAM, 0);
	104	if (CHECK(lfd == -1, "socket", "errno:%d\n", errno))
	105	return;
	106	fd = socket(AF_INET6, SOCK_STREAM, 0);
	107	if (CHECK(fd == -1, "socket", "errno:%d\n", errno)) {
	108	close(lfd);
	109	return;
	110	}
	111
	112	if (settcpca(lfd, tcp_ca) \|\| settcpca(fd, tcp_ca) \|\|
	113	settimeo(lfd) \|\| settimeo(fd))
	114	goto done;
	115
	116	/* bind, listen and start server thread to accept */
	117	sa6.sin6_family = AF_INET6;
	118	sa6.sin6_addr = in6addr_loopback;
	119	err = bind(lfd, (struct sockaddr *)&sa6, addrlen);
	120	if (CHECK(err == -1, "bind", "errno:%d\n", errno))
	121	goto done;
	122	err = getsockname(lfd, (struct sockaddr *)&sa6, &addrlen);
	123	if (CHECK(err == -1, "getsockname", "errno:%d\n", errno))
	124	goto done;
	125	err = listen(lfd, 1);
	126	if (CHECK(err == -1, "listen", "errno:%d\n", errno))
	127	goto done;
	128	err = pthread_create(&srv_thread, NULL, server, (void *)(long)lfd);
	129	if (CHECK(err != 0, "pthread_create", "err:%d\n", err))
	130	goto done;
	131
	132	/* connect to server */
	133	err = connect(fd, (struct sockaddr *)&sa6, addrlen);
	134	if (CHECK(err == -1, "connect", "errno:%d\n", errno))
	135	goto wait_thread;
	136
	137	/* recv total_bytes */
	138	while (bytes < total_bytes && !READ_ONCE(stop)) {
	139	nr_recv = recv(fd, &batch,
	140	min(total_bytes - bytes, sizeof(batch)), 0);
	141	if (nr_recv == -1 && errno == EINTR)
	142	continue;
	143	if (nr_recv == -1)
	144	break;
	145	bytes += nr_recv;
	146	}
	147
	148	CHECK(bytes != total_bytes, "recv", "%zd != %u nr_recv:%zd errno:%d\n",
	149	bytes, total_bytes, nr_recv, errno);
	150
	151	wait_thread:
	152	WRITE_ONCE(stop, 1);
	153	pthread_join(srv_thread, &thread_ret);
	154	CHECK(IS_ERR(thread_ret), "pthread_join", "thread_ret:%ld",
	155	PTR_ERR(thread_ret));
	156	done:
	157	close(lfd);
	158	close(fd);
	159	}
	160
	161	static void test_dctcp(void)
	162	{
	163	struct bpf_dctcp *dctcp_skel;
	164	struct bpf_link *link;
	165
	166	dctcp_skel = bpf_dctcp__open_and_load();
	167	if (CHECK(!dctcp_skel, "bpf_dctcp__open_and_load", "failed\n"))
	168	return;
	169
	170	link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp);
	171	if (CHECK(IS_ERR(link), "bpf_map__attach_struct_ops", "err:%ld\n",
	172	PTR_ERR(link))) {
	173	bpf_dctcp__destroy(dctcp_skel);
	174	return;
	175	}
	176
	177	do_test("bpf_dctcp");
	178
	179	bpf_link__destroy(link);
	180	bpf_dctcp__destroy(dctcp_skel);
	181	}
	182
	183	void test_bpf_tcp_ca(void)
	184	{
	185	if (test__start_subtest("dctcp"))
	186	test_dctcp();
	187	}

tools/testing/selftests/bpf/progs/bpf_dctcp.c+210−0

@@ -0,0 +1,210 @@
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Copyright (c) 2019 Facebook */
	3
	4	/* WARNING: This implemenation is not necessarily the same
	5	* as the tcp_dctcp.c. The purpose is mainly for testing
	6	* the kernel BPF logic.
	7	*/
	8
	9	#include <linux/bpf.h>
	10	#include <linux/types.h>
	11	#include "bpf_tcp_helpers.h"
	12
	13	char _license[] SEC("license") = "GPL";
	14
	15	#define DCTCP_MAX_ALPHA 1024U
	16
	17	struct dctcp {
	18	__u32 old_delivered;
	19	__u32 old_delivered_ce;
	20	__u32 prior_rcv_nxt;
	21	__u32 dctcp_alpha;
	22	__u32 next_seq;
	23	__u32 ce_state;
	24	__u32 loss_cwnd;
	25	};
	26
	27	static unsigned int dctcp_shift_g = 4; /* g = 1/2^4 */
	28	static unsigned int dctcp_alpha_on_init = DCTCP_MAX_ALPHA;
	29
	30	static __always_inline void dctcp_reset(const struct tcp_sock *tp,
	31	struct dctcp *ca)
	32	{
	33	ca->next_seq = tp->snd_nxt;
	34
	35	ca->old_delivered = tp->delivered;
	36	ca->old_delivered_ce = tp->delivered_ce;
	37	}
	38
	39	BPF_TCP_OPS_1(dctcp_init, void, struct sock *, sk)
	40	{
	41	const struct tcp_sock *tp = tcp_sk(sk);
	42	struct dctcp *ca = inet_csk_ca(sk);
	43
	44	ca->prior_rcv_nxt = tp->rcv_nxt;
	45	ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
	46	ca->loss_cwnd = 0;
	47	ca->ce_state = 0;
	48
	49	dctcp_reset(tp, ca);
	50	}
	51
	52	BPF_TCP_OPS_1(dctcp_ssthresh, __u32, struct sock *, sk)
	53	{
	54	struct dctcp *ca = inet_csk_ca(sk);
	55	struct tcp_sock *tp = tcp_sk(sk);
	56
	57	ca->loss_cwnd = tp->snd_cwnd;
	58	return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
	59	}
	60
	61	BPF_TCP_OPS_2(dctcp_update_alpha, void,
	62	struct sock *, sk, __u32, flags)
	63	{
	64	const struct tcp_sock *tp = tcp_sk(sk);
	65	struct dctcp *ca = inet_csk_ca(sk);
	66
	67	/* Expired RTT */
	68	if (!before(tp->snd_una, ca->next_seq)) {
	69	__u32 delivered_ce = tp->delivered_ce - ca->old_delivered_ce;
	70	__u32 alpha = ca->dctcp_alpha;
	71
	72	/* alpha = (1 - g) * alpha + g * F */
	73
	74	alpha -= min_not_zero(alpha, alpha >> dctcp_shift_g);
	75	if (delivered_ce) {
	76	__u32 delivered = tp->delivered - ca->old_delivered;
	77
	78	/* If dctcp_shift_g == 1, a 32bit value would overflow
	79	* after 8 M packets.
	80	*/
	81	delivered_ce <<= (10 - dctcp_shift_g);
	82	delivered_ce /= max(1U, delivered);
	83
	84	alpha = min(alpha + delivered_ce, DCTCP_MAX_ALPHA);
	85	}
	86	ca->dctcp_alpha = alpha;
	87	dctcp_reset(tp, ca);
	88	}
	89	}
	90
	91	static __always_inline void dctcp_react_to_loss(struct sock *sk)
	92	{
	93	struct dctcp *ca = inet_csk_ca(sk);
	94	struct tcp_sock *tp = tcp_sk(sk);
	95
	96	ca->loss_cwnd = tp->snd_cwnd;
	97	tp->snd_ssthresh = max(tp->snd_cwnd >> 1U, 2U);
	98	}
	99
	100	BPF_TCP_OPS_2(dctcp_state, void, struct sock *, sk, __u8, new_state)
	101	{
	102	if (new_state == TCP_CA_Recovery &&
	103	new_state != BPF_CORE_READ_BITFIELD(inet_csk(sk), icsk_ca_state))
	104	dctcp_react_to_loss(sk);
	105	/* We handle RTO in dctcp_cwnd_event to ensure that we perform only
	106	* one loss-adjustment per RTT.
	107	*/
	108	}
	109
	110	static __always_inline void dctcp_ece_ack_cwr(struct sock *sk, __u32 ce_state)
	111	{
	112	struct tcp_sock *tp = tcp_sk(sk);
	113
	114	if (ce_state == 1)
	115	tp->ecn_flags \|= TCP_ECN_DEMAND_CWR;
	116	else
	117	tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
	118	}
	119
	120	/* Minimal DCTP CE state machine:
	121	*
	122	* S: 0 <- last pkt was non-CE
	123	* 1 <- last pkt was CE
	124	*/
	125	static __always_inline
	126	void dctcp_ece_ack_update(struct sock *sk, enum tcp_ca_event evt,
	127	__u32 prior_rcv_nxt, __u32 ce_state)
	128	{
	129	__u32 new_ce_state = (evt == CA_EVENT_ECN_IS_CE) ? 1 : 0;
	130
	131	if (*ce_state != new_ce_state) {
	132	/* CE state has changed, force an immediate ACK to
	133	* reflect the new CE state. If an ACK was delayed,
	134	* send that first to reflect the prior CE state.
	135	*/
	136	if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) {
	137	dctcp_ece_ack_cwr(sk, *ce_state);
	138	bpf_tcp_send_ack(sk, *prior_rcv_nxt);
	139	}
	140	inet_csk(sk)->icsk_ack.pending \|= ICSK_ACK_NOW;
	141	}
	142	*prior_rcv_nxt = tcp_sk(sk)->rcv_nxt;
	143	*ce_state = new_ce_state;
	144	dctcp_ece_ack_cwr(sk, new_ce_state);
	145	}
	146
	147	BPF_TCP_OPS_2(dctcp_cwnd_event, void,
	148	struct sock *, sk, enum tcp_ca_event, ev)
	149	{
	150	struct dctcp *ca = inet_csk_ca(sk);
	151
	152	switch (ev) {
	153	case CA_EVENT_ECN_IS_CE:
	154	case CA_EVENT_ECN_NO_CE:
	155	dctcp_ece_ack_update(sk, ev, &ca->prior_rcv_nxt, &ca->ce_state);
	156	break;
	157	case CA_EVENT_LOSS:
	158	dctcp_react_to_loss(sk);
	159	break;
	160	default:
	161	/* Don't care for the rest. */
	162	break;
	163	}
	164	}
	165
	166	BPF_TCP_OPS_1(dctcp_cwnd_undo, __u32, struct sock *, sk)
	167	{
	168	const struct dctcp *ca = inet_csk_ca(sk);
	169
	170	return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
	171	}
	172
	173	BPF_TCP_OPS_3(tcp_reno_cong_avoid, void,
	174	struct sock *, sk, __u32, ack, __u32, acked)
	175	{
	176	struct tcp_sock *tp = tcp_sk(sk);
	177
	178	if (!tcp_is_cwnd_limited(sk))
	179	return;
	180
	181	/* In "safe" area, increase. */
	182	if (tcp_in_slow_start(tp)) {
	183	acked = tcp_slow_start(tp, acked);
	184	if (!acked)
	185	return;
	186	}
	187	/* In dangerous area, increase slowly. */
	188	tcp_cong_avoid_ai(tp, tp->snd_cwnd, acked);
	189	}
	190
	191	SEC(".struct_ops")
	192	struct tcp_congestion_ops dctcp_nouse = {
	193	.init = (void *)dctcp_init,
	194	.set_state = (void *)dctcp_state,
	195	.flags = TCP_CONG_NEEDS_ECN,
	196	.name = "bpf_dctcp_nouse",
	197	};
	198
	199	SEC(".struct_ops")
	200	struct tcp_congestion_ops dctcp = {
	201	.init = (void *)dctcp_init,
	202	.in_ack_event = (void *)dctcp_update_alpha,
	203	.cwnd_event = (void *)dctcp_cwnd_event,
	204	.ssthresh = (void *)dctcp_ssthresh,
	205	.cong_avoid = (void *)tcp_reno_cong_avoid,
	206	.undo_cwnd = (void *)dctcp_cwnd_undo,
	207	.set_state = (void *)dctcp_state,
	208	.flags = TCP_CONG_NEEDS_ECN,
	209	.name = "bpf_dctcp",
	210	};