Merge branch 'tcp-bpf-cc'

Martin Lau says: ==================== This series introduces BPF STRUCT_OPS. It is an infra to allow implementing some specific kernel's function pointers in BPF. The first use case included in this series is to implement TCP congestion control algorithm in BPF (i.e. implement struct tcp_congestion_ops in BPF). There has been attempt to move the TCP CC to the user space (e.g. CCP in TCP). The common arguments are faster turn around, get away from long-tail kernel versions in production...etc, which are legit points. BPF has been the continuous effort to join both kernel and userspace upsides together (e.g. XDP to gain the performance advantage without bypassing the kernel). The recent BPF advancements (in particular BTF-aware verifier, BPF trampoline, BPF CO-RE...) made implementing kernel struct ops (e.g. tcp cc) possible in BPF. The idea is to allow implementing tcp_congestion_ops in bpf. It allows a faster turnaround for testing algorithm in the production while leveraging the existing (and continue growing) BPF feature/framework instead of building one specifically for userspace TCP CC. Please see individual patch for details. The bpftool support will be posted in follow-up patches. v4: - Expose tcp_ca_find() to tcp.h in patch 7. It is used to check the same bpf-tcp-cc does not exist to guarantee the register() will succeed. - set_memory_ro() and then set_memory_x() only after all trampolines are written to the image in patch 6. (Daniel) spinlock is replaced by mutex because set_memory_* requires sleepable context. v3: - Fix kbuild error by considering CONFIG_BPF_SYSCALL (kbuild) - Support anonymous bitfield in patch 4 (Andrii, Yonghong) - Push boundary safety check to a specific arch's trampoline function (in patch 6) (Yonghong). Reuse the WANR_ON_ONCE check in arch_prepare_bpf_trampoline() in x86. - Check module field is 0 in udata in patch 6 (Yonghong) - Check zero holes in patch 6 (Andrii) - s/_btf_vmlinux/btf/ in patch 5 and 7 (Andrii) - s/check_xxx/is_xxx/ in patch 7 (Andrii) - Use "struct_ops/" convention in patch 11 (Andrii) - Use the skel instead of bpf_object in patch 11 (Andrii) - libbpf: Decide BPF_PROG_TYPE_STRUCT_OPS at open phase by using find_sec_def() - libbpf: Avoid a debug message at open phase (Andrii) - libbpf: Add bpf_program__(is|set)_struct_ops() for consistency (Andrii) - libbpf: Add "struct_ops" to section_defs (Andrii) - libbpf: Some code shuffling in init_kern_struct_ops() (Andrii) - libbpf: A few safety checks (Andrii) v2: - Dropped cubic for now. They will be reposted once there are more clarity in "jiffies" on both bpf side (about the helper) and tcp_cubic side (some of jiffies usages are being replaced by tp->tcp_mstamp) - Remove unnecssary check on bitfield support from btf_struct_access() (Yonghong) - BTF_TYPE_EMIT macro (Yonghong, Andrii) - value_name's length check to avoid an unlikely type match during truncation case (Yonghong) - BUILD_BUG_ON to ensure no trampoline-image overrun in the future (Yonghong) - Simplify get_next_key() (Yonghong) - Added comment to explain how to check mandatory func ptr in net/ipv4/bpf_tcp_ca.c (Yonghong) - Rename "__bpf_" to "bpf_struct_ops_" for value prefix (Andrii) - Add comment to highlight the bpf_dctcp.c is not necessarily the same as tcp_dctcp.c. (Alexei, Eric) - libbpf: Renmae "struct_ops" to ".struct_ops" for elf sec (Andrii) - libbpf: Expose struct_ops as a bpf_map (Andrii) - libbpf: Support multiple struct_ops in SEC(".struct_ops") (Andrii) - libbpf: Add bpf_map__attach_struct_ops() (Andrii) ====================

Alexei Starovoitov committed 6 years ago

Commit 417759f2 parents 65726b5 099038632 files · +2654 −148Browse files

Unified · Split

32 files

arch/x86/net/bpf_jit_comp.c +8 −10
include/linux/bpf.h +77 −2
include/linux/bpf_types.h +7 −0
include/linux/btf.h +47 −0
include/linux/filter.h +2 −0
include/net/tcp.h +2 −0
include/uapi/linux/bpf.h +17 −2
kernel/bpf/Makefile +3 −0
kernel/bpf/bpf_struct_ops.c +628 −0
kernel/bpf/bpf_struct_ops_types.h +9 −0
kernel/bpf/btf.c +116 −47
kernel/bpf/map_in_map.c +2 −1
kernel/bpf/syscall.c +46 −23
kernel/bpf/trampoline.c +5 −3
kernel/bpf/verifier.c +109 −30
net/core/filter.c +1 −1
net/ipv4/Makefile +4 −0
net/ipv4/bpf_tcp_ca.c +252 −0
net/ipv4/tcp_cong.c +8 −8
net/ipv4/tcp_ipv4.c +4 −2
net/ipv4/tcp_minisocks.c +2 −2
net/ipv4/tcp_output.c +2 −2
tools/include/uapi/linux/bpf.h +17 −2
tools/lib/bpf/bpf.c +8 −2
tools/lib/bpf/bpf.h +4 −1
tools/lib/bpf/libbpf.c +640 −9
tools/lib/bpf/libbpf.h +4 −1
tools/lib/bpf/libbpf.map +3 −0
tools/lib/bpf/libbpf_probes.c +2 −0
tools/testing/selftests/bpf/bpf_tcp_helpers.h +228 −0
tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c +187 −0
tools/testing/selftests/bpf/progs/bpf_dctcp.c +210 −0

arch/x86/net/bpf_jit_comp.c+8−10

@@ -1328,7 +1328,7 @@ emit_jmp: · 1327 unmodified lines
1328	1328	return proglen;
1329	1329	}
1330	1330
1331		−static void save_regs(struct btf_func_model m, u8 *prog, int nr_args,
	1331	+static void save_regs(const struct btf_func_model m, u8 *prog, int nr_args,
1332	1332	int stack_size)
1333	1333	{
1334	1334	int i;
@@ -1344,7 +1344,7 @@ static void save_regs(struct btf_func_model m, u8 *prog, int nr_args, · 9 unmodified lines
1344	1344	-(stack_size - i * 8));
1345	1345	}
1346	1346
1347		−static void restore_regs(struct btf_func_model m, u8 *prog, int nr_args,
	1347	+static void restore_regs(const struct btf_func_model m, u8 *prog, int nr_args,
1348	1348	int stack_size)
1349	1349	{
1350	1350	int i;
@@ -1361,7 +1361,7 @@ static void restore_regs(struct btf_func_model m, u8 *prog, int nr_args, · 10 unmodified lines
1361	1361	-(stack_size - i * 8));
1362	1362	}
1363	1363
1364		−static int invoke_bpf(struct btf_func_model m, u8 *pprog,
	1364	+static int invoke_bpf(const struct btf_func_model m, u8 *pprog,
1365	1365	struct bpf_prog **progs, int prog_cnt, int stack_size)
1366	1366	{
1367	1367	u8 prog = pprog;
@@ -1456,7 +1456,8 @@ static int invoke_bpf(struct btf_func_model m, u8 *pprog, · 88 unmodified lines
1456	1456	* add rsp, 8 // skip eth_type_trans's frame
1457	1457	* ret // return to its caller
1458	1458	*/
1459		−int arch_prepare_bpf_trampoline(void image, struct btf_func_model m, u32 flags,
	1459	+int arch_prepare_bpf_trampoline(void image, void image_end,
	1460	+ const struct btf_func_model *m, u32 flags,
1460	1461	struct bpf_prog **fentry_progs, int fentry_cnt,
1461	1462	struct bpf_prog **fexit_progs, int fexit_cnt,
1462	1463	void *orig_call)
@@ -1523,13 +1524,10 @@ int arch_prepare_bpf_trampoline(void image, struct btf_func_model m, u32 flags · 60 unmodified lines
1523	1524	/* skip our return address and return to parent */
1524	1525	EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */
1525	1526	EMIT1(0xC3); /* ret */
1526		− /* One half of the page has active running trampoline.
1527		− * Another half is an area for next trampoline.
1528		− * Make sure the trampoline generation logic doesn't overflow.
1529		− */
1530		− if (WARN_ON_ONCE(prog - (u8 *)image > PAGE_SIZE / 2 - BPF_INSN_SAFETY))
	1527	+ /* Make sure the trampoline generation logic doesn't overflow */
	1528	+ if (WARN_ON_ONCE(prog > (u8 *)image_end - BPF_INSN_SAFETY))
1531	1529	return -EFAULT;
1532		− return 0;
	1530	+ return prog - (u8 *)image;
1533	1531	}
1534	1532
1535	1533	static int emit_cond_near_jump(u8 *pprog, void func, void *ip, u8 jmp_cond)

include/linux/bpf.h+77−2

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <linux/u64_stats_sync.h>
18	18	#include <linux/refcount.h>
19	19	#include <linux/mutex.h>
	20	+#include <linux/module.h>
20	21
21	22	struct bpf_verifier_env;
22	23	struct bpf_verifier_log;
@@ -106,6 +107,7 @@ struct bpf_map { · 83 unmodified lines
106	107	struct btf *btf;
107	108	struct bpf_map_memory memory;
108	109	char name[BPF_OBJ_NAME_LEN];
	110	+ u32 btf_vmlinux_value_type_id;
109	111	bool unpriv_array;
110	112	bool frozen; /* write-once; write-protected by freeze_mutex */
111	113	/* 22 bytes hole */
@@ -183,7 +185,8 @@ static inline bool bpf_map_offload_neutral(const struct bpf_map *map) · 71 unmodified lines
183	185
184	186	static inline bool bpf_map_support_seq_show(const struct bpf_map *map)
185	187	{
186		− return map->btf && map->ops->map_seq_show_elem;
	188	+ return (map->btf_value_type_id \|\| map->btf_vmlinux_value_type_id) &&
	189	+ map->ops->map_seq_show_elem;
187	190	}
188	191
189	192	int map_check_no_btf(const struct bpf_map *map,
@@ -349,6 +352,10 @@ struct bpf_verifier_ops { · 159 unmodified lines
349	352	const struct bpf_insn *src,
350	353	struct bpf_insn *dst,
351	354	struct bpf_prog prog, u32 target_size);
	355	+ int (btf_struct_access)(struct bpf_verifier_log log,
	356	+ const struct btf_type *t, int off, int size,
	357	+ enum bpf_access_type atype,
	358	+ u32 *next_btf_id);
352	359	};
353	360
354	361	struct bpf_prog_offload_ops {
@@ -437,7 +444,8 @@ struct btf_func_model { · 82 unmodified lines
437	444	* fentry = a set of program to run before calling original function
438	445	* fexit = a set of program to run after original function
439	446	*/
440		−int arch_prepare_bpf_trampoline(void image, struct btf_func_model m, u32 flags,
	447	+int arch_prepare_bpf_trampoline(void image, void image_end,
	448	+ const struct btf_func_model *m, u32 flags,
441	449	struct bpf_prog **fentry_progs, int fentry_cnt,
442	450	struct bpf_prog **fexit_progs, int fexit_cnt,
443	451	void *orig_call);
@@ -668,6 +676,73 @@ struct bpf_array_aux { · 224 unmodified lines
668	676	struct work_struct work;
669	677	};
670	678
	679	+struct bpf_struct_ops_value;
	680	+struct btf_type;
	681	+struct btf_member;
	682	+
	683	+#define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64
	684	+struct bpf_struct_ops {
	685	+ const struct bpf_verifier_ops *verifier_ops;
	686	+ int (init)(struct btf btf);
	687	+ int (check_member)(const struct btf_type t,
	688	+ const struct btf_member *member);
	689	+ int (init_member)(const struct btf_type t,
	690	+ const struct btf_member *member,
	691	+ void kdata, const void udata);
	692	+ int (reg)(void kdata);
	693	+ void (unreg)(void kdata);
	694	+ const struct btf_type *type;
	695	+ const struct btf_type *value_type;
	696	+ const char *name;
	697	+ struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS];
	698	+ u32 type_id;
	699	+ u32 value_id;
	700	+};
	701	+
	702	+#if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL)
	703	+#define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
	704	+const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id);
	705	+void bpf_struct_ops_init(struct btf *btf);
	706	+bool bpf_struct_ops_get(const void *kdata);
	707	+void bpf_struct_ops_put(const void *kdata);
	708	+int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map map, void key,
	709	+ void *value);
	710	+static inline bool bpf_try_module_get(const void data, struct module owner)
	711	+{
	712	+ if (owner == BPF_MODULE_OWNER)
	713	+ return bpf_struct_ops_get(data);
	714	+ else
	715	+ return try_module_get(owner);
	716	+}
	717	+static inline void bpf_module_put(const void data, struct module owner)
	718	+{
	719	+ if (owner == BPF_MODULE_OWNER)
	720	+ bpf_struct_ops_put(data);
	721	+ else
	722	+ module_put(owner);
	723	+}
	724	+#else
	725	+static inline const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
	726	+{
	727	+ return NULL;
	728	+}
	729	+static inline void bpf_struct_ops_init(struct btf *btf) { }
	730	+static inline bool bpf_try_module_get(const void data, struct module owner)
	731	+{
	732	+ return try_module_get(owner);
	733	+}
	734	+static inline void bpf_module_put(const void data, struct module owner)
	735	+{
	736	+ module_put(owner);
	737	+}
	738	+static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map,
	739	+ void *key,
	740	+ void *value)
	741	+{
	742	+ return -EINVAL;
	743	+}
	744	+#endif
	745	+
671	746	struct bpf_array {
672	747	struct bpf_map map;
673	748	u32 elem_size;

include/linux/bpf_types.h+7−0

@@ -65,6 +65,10 @@ BPF_PROG_TYPE(BPF_PROG_TYPE_LIRC_MODE2, lirc_mode2, · 64 unmodified lines
65	65	BPF_PROG_TYPE(BPF_PROG_TYPE_SK_REUSEPORT, sk_reuseport,
66	66	struct sk_reuseport_md, struct sk_reuseport_kern)
67	67	#endif
	68	+#if defined(CONFIG_BPF_JIT)
	69	+BPF_PROG_TYPE(BPF_PROG_TYPE_STRUCT_OPS, bpf_struct_ops,
	70	+ void , void )
	71	+#endif
68	72
69	73	BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY, array_map_ops)
70	74	BPF_MAP_TYPE(BPF_MAP_TYPE_PERCPU_ARRAY, percpu_array_map_ops)
@@ -105,3 +109,6 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, reuseport_array_ops) · 34 unmodified lines
105	109	#endif
106	110	BPF_MAP_TYPE(BPF_MAP_TYPE_QUEUE, queue_map_ops)
107	111	BPF_MAP_TYPE(BPF_MAP_TYPE_STACK, stack_map_ops)
	112	+#if defined(CONFIG_BPF_JIT)
	113	+BPF_MAP_TYPE(BPF_MAP_TYPE_STRUCT_OPS, bpf_struct_ops_map_ops)
	114	+#endif

include/linux/btf.h+47−0

@@ -7,6 +7,8 @@ · 6 unmodified lines
7	7	#include <linux/types.h>
8	8	#include <uapi/linux/btf.h>
9	9
	10	+#define BTF_TYPE_EMIT(type) ((void)(type *)0)
	11	+
10	12	struct btf;
11	13	struct btf_member;
12	14	struct btf_type;
@@ -53,6 +55,22 @@ bool btf_member_is_reg_int(const struct btf btf, const struct btf_type s, · 40 unmodified lines
53	55	u32 expected_offset, u32 expected_size);
54	56	int btf_find_spin_lock(const struct btf btf, const struct btf_type t);
55	57	bool btf_type_is_void(const struct btf_type *t);
	58	+s32 btf_find_by_name_kind(const struct btf btf, const char name, u8 kind);
	59	+const struct btf_type btf_type_skip_modifiers(const struct btf btf,
	60	+ u32 id, u32 *res_id);
	61	+const struct btf_type btf_type_resolve_ptr(const struct btf btf,
	62	+ u32 id, u32 *res_id);
	63	+const struct btf_type btf_type_resolve_func_ptr(const struct btf btf,
	64	+ u32 id, u32 *res_id);
	65	+const struct btf_type *
	66	+btf_resolve_size(const struct btf btf, const struct btf_type type,
	67	+ u32 type_size, const struct btf_type *elem_type,
	68	+ u32 *total_nelems);
	69	+
	70	+#define for_each_member(i, struct_type, member) \
	71	+ for (i = 0, member = btf_type_member(struct_type); \
	72	+ i < btf_type_vlen(struct_type); \
	73	+ i++, member++)
56	74
57	75	static inline bool btf_type_is_ptr(const struct btf_type *t)
58	76	{
@@ -84,6 +102,35 @@ static inline bool btf_type_is_func_proto(const struct btf_type *t) · 25 unmodified lines
84	102	return BTF_INFO_KIND(t->info) == BTF_KIND_FUNC_PROTO;
85	103	}
86	104
	105	+static inline u16 btf_type_vlen(const struct btf_type *t)
	106	+{
	107	+ return BTF_INFO_VLEN(t->info);
	108	+}
	109	+
	110	+static inline bool btf_type_kflag(const struct btf_type *t)
	111	+{
	112	+ return BTF_INFO_KFLAG(t->info);
	113	+}
	114	+
	115	+static inline u32 btf_member_bit_offset(const struct btf_type *struct_type,
	116	+ const struct btf_member *member)
	117	+{
	118	+ return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
	119	+ : member->offset;
	120	+}
	121	+
	122	+static inline u32 btf_member_bitfield_size(const struct btf_type *struct_type,
	123	+ const struct btf_member *member)
	124	+{
	125	+ return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
	126	+ : 0;
	127	+}
	128	+
	129	+static inline const struct btf_member btf_type_member(const struct btf_type t)
	130	+{
	131	+ return (const struct btf_member *)(t + 1);
	132	+}
	133	+
87	134	#ifdef CONFIG_BPF_SYSCALL
88	135	const struct btf_type btf_type_by_id(const struct btf btf, u32 type_id);
89	136	const char btf_name_by_offset(const struct btf btf, u32 offset);

include/linux/filter.h+2−0

@@ -843,6 +843,8 @@ int bpf_prog_create(struct bpf_prog *pfp, struct sock_fprog_kern fprog); · 842 unmodified lines
843	843	int bpf_prog_create_from_user(struct bpf_prog *pfp, struct sock_fprog fprog,
844	844	bpf_aux_classic_check_t trans, bool save_orig);
845	845	void bpf_prog_destroy(struct bpf_prog *fp);
	846	+const struct bpf_func_proto *
	847	+bpf_base_func_proto(enum bpf_func_id func_id);
846	848
847	849	int sk_attach_filter(struct sock_fprog fprog, struct sock sk);
848	850	int sk_attach_bpf(u32 ufd, struct sock *sk);

include/net/tcp.h+2−0

@@ -1007,6 +1007,7 @@ enum tcp_ca_ack_event_flags { · 1006 unmodified lines
1007	1007	#define TCP_CONG_NON_RESTRICTED 0x1
1008	1008	/* Requires ECN/ECT set on all packets */
1009	1009	#define TCP_CONG_NEEDS_ECN 0x2
	1010	+#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED \| TCP_CONG_NEEDS_ECN)
1010	1011
1011	1012	union tcp_cc_info;
1012	1013
@@ -1101,6 +1102,7 @@ u32 tcp_reno_undo_cwnd(struct sock *sk); · 88 unmodified lines
1101	1102	void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1102	1103	extern struct tcp_congestion_ops tcp_reno;
1103	1104
	1105	+struct tcp_congestion_ops tcp_ca_find(const char name);
1104	1106	struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
1105	1107	u32 tcp_ca_get_key_by_name(struct net net, const char name, bool *ecn_ca);
1106	1108	#ifdef CONFIG_INET

include/uapi/linux/bpf.h+17−2

@@ -136,6 +136,7 @@ enum bpf_map_type { · 135 unmodified lines
136	136	BPF_MAP_TYPE_STACK,
137	137	BPF_MAP_TYPE_SK_STORAGE,
138	138	BPF_MAP_TYPE_DEVMAP_HASH,
	139	+ BPF_MAP_TYPE_STRUCT_OPS,
139	140	};
140	141
141	142	/* Note that tracing related programs such as
@@ -174,6 +175,7 @@ enum bpf_prog_type { · 32 unmodified lines
174	175	BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE,
175	176	BPF_PROG_TYPE_CGROUP_SOCKOPT,
176	177	BPF_PROG_TYPE_TRACING,
	178	+ BPF_PROG_TYPE_STRUCT_OPS,
177	179	};
178	180
179	181	enum bpf_attach_type {
@@ -397,6 +399,10 @@ union bpf_attr { · 217 unmodified lines
397	399	__u32 btf_fd; /* fd pointing to a BTF type data */
398	400	__u32 btf_key_type_id; /* BTF type_id of the key */
399	401	__u32 btf_value_type_id; /* BTF type_id of the value */
	402	+ __u32 btf_vmlinux_value_type_id;/* BTF type_id of a kernel-
	403	+ * struct stored as the
	404	+ * map value
	405	+ */
400	406	};
401	407
402	408	struct { /* anonymous struct used by BPF_MAP__ELEM commands /
@@ -2831,6 +2837,14 @@ union bpf_attr { · 2428 unmodified lines
2831	2837	* Return
2832	2838	* On success, the strictly positive length of the string, including
2833	2839	* the trailing NUL character. On error, a negative value.
	2840	+ *
	2841	+ * int bpf_tcp_send_ack(void *tp, u32 rcv_nxt)
	2842	+ * Description
	2843	+ * Send out a tcp-ack. tp is the in-kernel struct tcp_sock.
	2844	+ * rcv_nxt is the ack_seq to be sent out.
	2845	+ * Return
	2846	+ * 0 on success, or a negative error in case of failure.
	2847	+ *
2834	2848	*/
2835	2849	#define __BPF_FUNC_MAPPER(FN) \
2836	2850	FN(unspec), \
@@ -2948,7 +2962,8 @@ union bpf_attr { · 111 unmodified lines
2948	2962	FN(probe_read_user), \
2949	2963	FN(probe_read_kernel), \
2950	2964	FN(probe_read_user_str), \
2951		− FN(probe_read_kernel_str),
	2965	+ FN(probe_read_kernel_str), \
	2966	+ FN(tcp_send_ack),
2952	2967
2953	2968	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2954	2969	* function eBPF program intends to call
@@ -3349,7 +3364,7 @@ struct bpf_map_info { · 394 unmodified lines
3349	3364	__u32 map_flags;
3350	3365	char name[BPF_OBJ_NAME_LEN];
3351	3366	__u32 ifindex;
3352		− __u32 :32;
	3367	+ __u32 btf_vmlinux_value_type_id;
3353	3368	__u64 netns_dev;
3354	3369	__u64 netns_ino;
3355	3370	__u32 btf_id;

kernel/bpf/Makefile+3−0

@@ -27,3 +27,6 @@ endif · 26 unmodified lines
27	27	ifeq ($(CONFIG_SYSFS),y)
28	28	obj-$(CONFIG_DEBUG_INFO_BTF) += sysfs_btf.o
29	29	endif
	30	+ifeq ($(CONFIG_BPF_JIT),y)
	31	+obj-$(CONFIG_BPF_SYSCALL) += bpf_struct_ops.o
	32	+endif

kernel/bpf/bpf_struct_ops.c+628−0

@@ -0,0 +1,628 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/* Copyright (c) 2019 Facebook */
	3	+
	4	+#include <linux/bpf.h>
	5	+#include <linux/bpf_verifier.h>
	6	+#include <linux/btf.h>
	7	+#include <linux/filter.h>
	8	+#include <linux/slab.h>
	9	+#include <linux/numa.h>
	10	+#include <linux/seq_file.h>
	11	+#include <linux/refcount.h>
	12	+#include <linux/mutex.h>
	13	+
	14	+enum bpf_struct_ops_state {
	15	+ BPF_STRUCT_OPS_STATE_INIT,
	16	+ BPF_STRUCT_OPS_STATE_INUSE,
	17	+ BPF_STRUCT_OPS_STATE_TOBEFREE,
	18	+};
	19	+
	20	+#define BPF_STRUCT_OPS_COMMON_VALUE \
	21	+ refcount_t refcnt; \
	22	+ enum bpf_struct_ops_state state
	23	+
	24	+struct bpf_struct_ops_value {
	25	+ BPF_STRUCT_OPS_COMMON_VALUE;
	26	+ char data[0] ____cacheline_aligned_in_smp;
	27	+};
	28	+
	29	+struct bpf_struct_ops_map {
	30	+ struct bpf_map map;
	31	+ const struct bpf_struct_ops *st_ops;
	32	+ /* protect map_update */
	33	+ struct mutex lock;
	34	+ /* progs has all the bpf_prog that is populated
	35	+ * to the func ptr of the kernel's struct
	36	+ * (in kvalue.data).
	37	+ */
	38	+ struct bpf_prog **progs;
	39	+ /* image is a page that has all the trampolines
	40	+ * that stores the func args before calling the bpf_prog.
	41	+ * A PAGE_SIZE "image" is enough to store all trampoline for
	42	+ * "progs[]".
	43	+ */
	44	+ void *image;
	45	+ /* uvalue->data stores the kernel struct
	46	+ * (e.g. tcp_congestion_ops) that is more useful
	47	+ * to userspace than the kvalue. For example,
	48	+ * the bpf_prog's id is stored instead of the kernel
	49	+ * address of a func ptr.
	50	+ */
	51	+ struct bpf_struct_ops_value *uvalue;
	52	+ /* kvalue.data stores the actual kernel's struct
	53	+ * (e.g. tcp_congestion_ops) that will be
	54	+ * registered to the kernel subsystem.
	55	+ */
	56	+ struct bpf_struct_ops_value kvalue;
	57	+};
	58	+
	59	+#define VALUE_PREFIX "bpf_struct_ops_"
	60	+#define VALUE_PREFIX_LEN (sizeof(VALUE_PREFIX) - 1)
	61	+
	62	+/* bpf_struct_ops_##_name (e.g. bpf_struct_ops_tcp_congestion_ops) is
	63	+ * the map's value exposed to the userspace and its btf-type-id is
	64	+ * stored at the map->btf_vmlinux_value_type_id.
	65	+ *
	66	+ */
	67	+#define BPF_STRUCT_OPS_TYPE(_name) \
	68	+extern struct bpf_struct_ops bpf_##_name; \
	69	+ \
	70	+struct bpf_struct_ops_##_name { \
	71	+ BPF_STRUCT_OPS_COMMON_VALUE; \
	72	+ struct _name data ____cacheline_aligned_in_smp; \
	73	+};
	74	+#include "bpf_struct_ops_types.h"
	75	+#undef BPF_STRUCT_OPS_TYPE
	76	+
	77	+enum {
	78	+#define BPF_STRUCT_OPS_TYPE(_name) BPF_STRUCT_OPS_TYPE_##_name,
	79	+#include "bpf_struct_ops_types.h"
	80	+#undef BPF_STRUCT_OPS_TYPE
	81	+ __NR_BPF_STRUCT_OPS_TYPE,
	82	+};
	83	+
	84	+static struct bpf_struct_ops * const bpf_struct_ops[] = {
	85	+#define BPF_STRUCT_OPS_TYPE(_name) \
	86	+ [BPF_STRUCT_OPS_TYPE_##_name] = &bpf_##_name,
	87	+#include "bpf_struct_ops_types.h"
	88	+#undef BPF_STRUCT_OPS_TYPE
	89	+};
	90	+
	91	+const struct bpf_verifier_ops bpf_struct_ops_verifier_ops = {
	92	+};
	93	+
	94	+const struct bpf_prog_ops bpf_struct_ops_prog_ops = {
	95	+};
	96	+
	97	+static const struct btf_type *module_type;
	98	+
	99	+void bpf_struct_ops_init(struct btf *btf)
	100	+{
	101	+ s32 type_id, value_id, module_id;
	102	+ const struct btf_member *member;
	103	+ struct bpf_struct_ops *st_ops;
	104	+ struct bpf_verifier_log log = {};
	105	+ const struct btf_type *t;
	106	+ char value_name[128];
	107	+ const char *mname;
	108	+ u32 i, j;
	109	+
	110	+ /* Ensure BTF type is emitted for "struct bpf_struct_ops_##_name" */
	111	+#define BPF_STRUCT_OPS_TYPE(_name) BTF_TYPE_EMIT(struct bpf_struct_ops_##_name);
	112	+#include "bpf_struct_ops_types.h"
	113	+#undef BPF_STRUCT_OPS_TYPE
	114	+
	115	+ module_id = btf_find_by_name_kind(btf, "module", BTF_KIND_STRUCT);
	116	+ if (module_id < 0) {
	117	+ pr_warn("Cannot find struct module in btf_vmlinux\n");
	118	+ return;
	119	+ }
	120	+ module_type = btf_type_by_id(btf, module_id);
	121	+
	122	+ for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
	123	+ st_ops = bpf_struct_ops[i];
	124	+
	125	+ if (strlen(st_ops->name) + VALUE_PREFIX_LEN >=
	126	+ sizeof(value_name)) {
	127	+ pr_warn("struct_ops name %s is too long\n",
	128	+ st_ops->name);
	129	+ continue;
	130	+ }
	131	+ sprintf(value_name, "%s%s", VALUE_PREFIX, st_ops->name);
	132	+
	133	+ value_id = btf_find_by_name_kind(btf, value_name,
	134	+ BTF_KIND_STRUCT);
	135	+ if (value_id < 0) {
	136	+ pr_warn("Cannot find struct %s in btf_vmlinux\n",
	137	+ value_name);
	138	+ continue;
	139	+ }
	140	+
	141	+ type_id = btf_find_by_name_kind(btf, st_ops->name,
	142	+ BTF_KIND_STRUCT);
	143	+ if (type_id < 0) {
	144	+ pr_warn("Cannot find struct %s in btf_vmlinux\n",
	145	+ st_ops->name);
	146	+ continue;
	147	+ }
	148	+ t = btf_type_by_id(btf, type_id);
	149	+ if (btf_type_vlen(t) > BPF_STRUCT_OPS_MAX_NR_MEMBERS) {
	150	+ pr_warn("Cannot support #%u members in struct %s\n",
	151	+ btf_type_vlen(t), st_ops->name);
	152	+ continue;
	153	+ }
	154	+
	155	+ for_each_member(j, t, member) {
	156	+ const struct btf_type *func_proto;
	157	+
	158	+ mname = btf_name_by_offset(btf, member->name_off);
	159	+ if (!*mname) {
	160	+ pr_warn("anon member in struct %s is not supported\n",
	161	+ st_ops->name);
	162	+ break;
	163	+ }
	164	+
	165	+ if (btf_member_bitfield_size(t, member)) {
	166	+ pr_warn("bit field member %s in struct %s is not supported\n",
	167	+ mname, st_ops->name);
	168	+ break;
	169	+ }
	170	+
	171	+ func_proto = btf_type_resolve_func_ptr(btf,
	172	+ member->type,
	173	+ NULL);
	174	+ if (func_proto &&
	175	+ btf_distill_func_proto(&log, btf,
	176	+ func_proto, mname,
	177	+ &st_ops->func_models[j])) {
	178	+ pr_warn("Error in parsing func ptr %s in struct %s\n",
	179	+ mname, st_ops->name);
	180	+ break;
	181	+ }
	182	+ }
	183	+
	184	+ if (j == btf_type_vlen(t)) {
	185	+ if (st_ops->init(btf)) {
	186	+ pr_warn("Error in init bpf_struct_ops %s\n",
	187	+ st_ops->name);
	188	+ } else {
	189	+ st_ops->type_id = type_id;
	190	+ st_ops->type = t;
	191	+ st_ops->value_id = value_id;
	192	+ st_ops->value_type = btf_type_by_id(btf,
	193	+ value_id);
	194	+ }
	195	+ }
	196	+ }
	197	+}
	198	+
	199	+extern struct btf *btf_vmlinux;
	200	+
	201	+static const struct bpf_struct_ops *
	202	+bpf_struct_ops_find_value(u32 value_id)
	203	+{
	204	+ unsigned int i;
	205	+
	206	+ if (!value_id \|\| !btf_vmlinux)
	207	+ return NULL;
	208	+
	209	+ for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
	210	+ if (bpf_struct_ops[i]->value_id == value_id)
	211	+ return bpf_struct_ops[i];
	212	+ }
	213	+
	214	+ return NULL;
	215	+}
	216	+
	217	+const struct bpf_struct_ops *bpf_struct_ops_find(u32 type_id)
	218	+{
	219	+ unsigned int i;
	220	+
	221	+ if (!type_id \|\| !btf_vmlinux)
	222	+ return NULL;
	223	+
	224	+ for (i = 0; i < ARRAY_SIZE(bpf_struct_ops); i++) {
	225	+ if (bpf_struct_ops[i]->type_id == type_id)
	226	+ return bpf_struct_ops[i];
	227	+ }
	228	+
	229	+ return NULL;
	230	+}
	231	+
	232	+static int bpf_struct_ops_map_get_next_key(struct bpf_map map, void key,
	233	+ void *next_key)
	234	+{
	235	+ if (key && (u32 )key == 0)
	236	+ return -ENOENT;
	237	+
	238	+ (u32 )next_key = 0;
	239	+ return 0;
	240	+}
	241	+
	242	+int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map map, void key,
	243	+ void *value)
	244	+{
	245	+ struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;
	246	+ struct bpf_struct_ops_value uvalue, kvalue;
	247	+ enum bpf_struct_ops_state state;
	248	+
	249	+ if (unlikely((u32 )key != 0))
	250	+ return -ENOENT;
	251	+
	252	+ kvalue = &st_map->kvalue;
	253	+ /* Pair with smp_store_release() during map_update */
	254	+ state = smp_load_acquire(&kvalue->state);
	255	+ if (state == BPF_STRUCT_OPS_STATE_INIT) {
	256	+ memset(value, 0, map->value_size);
	257	+ return 0;
	258	+ }
	259	+
	260	+ /* No lock is needed. state and refcnt do not need
	261	+ * to be updated together under atomic context.
	262	+ */
	263	+ uvalue = (struct bpf_struct_ops_value *)value;
	264	+ memcpy(uvalue, st_map->uvalue, map->value_size);
	265	+ uvalue->state = state;
	266	+ refcount_set(&uvalue->refcnt, refcount_read(&kvalue->refcnt));
	267	+
	268	+ return 0;
	269	+}
	270	+
	271	+static void bpf_struct_ops_map_lookup_elem(struct bpf_map map, void *key)
	272	+{
	273	+ return ERR_PTR(-EINVAL);
	274	+}
	275	+
	276	+static void bpf_struct_ops_map_put_progs(struct bpf_struct_ops_map *st_map)
	277	+{
	278	+ const struct btf_type *t = st_map->st_ops->type;
	279	+ u32 i;
	280	+
	281	+ for (i = 0; i < btf_type_vlen(t); i++) {
	282	+ if (st_map->progs[i]) {
	283	+ bpf_prog_put(st_map->progs[i]);
	284	+ st_map->progs[i] = NULL;
	285	+ }
	286	+ }
	287	+}
	288	+
	289	+static int check_zero_holes(const struct btf_type t, void data)
	290	+{
	291	+ const struct btf_member *member;
	292	+ u32 i, moff, msize, prev_mend = 0;
	293	+ const struct btf_type *mtype;
	294	+
	295	+ for_each_member(i, t, member) {
	296	+ moff = btf_member_bit_offset(t, member) / 8;
	297	+ if (moff > prev_mend &&
	298	+ memchr_inv(data + prev_mend, 0, moff - prev_mend))
	299	+ return -EINVAL;
	300	+
	301	+ mtype = btf_type_by_id(btf_vmlinux, member->type);
	302	+ mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
	303	+ NULL, NULL);
	304	+ if (IS_ERR(mtype))
	305	+ return PTR_ERR(mtype);
	306	+ prev_mend = moff + msize;
	307	+ }
	308	+
	309	+ if (t->size > prev_mend &&
	310	+ memchr_inv(data + prev_mend, 0, t->size - prev_mend))
	311	+ return -EINVAL;
	312	+
	313	+ return 0;
	314	+}
	315	+
	316	+static int bpf_struct_ops_map_update_elem(struct bpf_map map, void key,
	317	+ void *value, u64 flags)
	318	+{
	319	+ struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;
	320	+ const struct bpf_struct_ops *st_ops = st_map->st_ops;
	321	+ struct bpf_struct_ops_value uvalue, kvalue;
	322	+ const struct btf_member *member;
	323	+ const struct btf_type *t = st_ops->type;
	324	+ void udata, kdata;
	325	+ int prog_fd, err = 0;
	326	+ void *image;
	327	+ u32 i;
	328	+
	329	+ if (flags)
	330	+ return -EINVAL;
	331	+
	332	+ if ((u32 )key != 0)
	333	+ return -E2BIG;
	334	+
	335	+ err = check_zero_holes(st_ops->value_type, value);
	336	+ if (err)
	337	+ return err;
	338	+
	339	+ uvalue = (struct bpf_struct_ops_value *)value;
	340	+ err = check_zero_holes(t, uvalue->data);
	341	+ if (err)
	342	+ return err;
	343	+
	344	+ if (uvalue->state \|\| refcount_read(&uvalue->refcnt))
	345	+ return -EINVAL;
	346	+
	347	+ uvalue = (struct bpf_struct_ops_value *)st_map->uvalue;
	348	+ kvalue = (struct bpf_struct_ops_value *)&st_map->kvalue;
	349	+
	350	+ mutex_lock(&st_map->lock);
	351	+
	352	+ if (kvalue->state != BPF_STRUCT_OPS_STATE_INIT) {
	353	+ err = -EBUSY;
	354	+ goto unlock;
	355	+ }
	356	+
	357	+ memcpy(uvalue, value, map->value_size);
	358	+
	359	+ udata = &uvalue->data;
	360	+ kdata = &kvalue->data;
	361	+ image = st_map->image;
	362	+
	363	+ for_each_member(i, t, member) {
	364	+ const struct btf_type mtype, ptype;
	365	+ struct bpf_prog *prog;
	366	+ u32 moff;
	367	+
	368	+ moff = btf_member_bit_offset(t, member) / 8;
	369	+ ptype = btf_type_resolve_ptr(btf_vmlinux, member->type, NULL);
	370	+ if (ptype == module_type) {
	371	+ if ((void *)(udata + moff))
	372	+ goto reset_unlock;
	373	+ (void *)(kdata + moff) = BPF_MODULE_OWNER;
	374	+ continue;
	375	+ }
	376	+
	377	+ err = st_ops->init_member(t, member, kdata, udata);
	378	+ if (err < 0)
	379	+ goto reset_unlock;
	380	+
	381	+ /* The ->init_member() has handled this member */
	382	+ if (err > 0)
	383	+ continue;
	384	+
	385	+ /* If st_ops->init_member does not handle it,
	386	+ * we will only handle func ptrs and zero-ed members
	387	+ * here. Reject everything else.
	388	+ */
	389	+
	390	+ /* All non func ptr member must be 0 */
	391	+ if (!ptype \|\| !btf_type_is_func_proto(ptype)) {
	392	+ u32 msize;
	393	+
	394	+ mtype = btf_type_by_id(btf_vmlinux, member->type);
	395	+ mtype = btf_resolve_size(btf_vmlinux, mtype, &msize,
	396	+ NULL, NULL);
	397	+ if (IS_ERR(mtype)) {
	398	+ err = PTR_ERR(mtype);
	399	+ goto reset_unlock;
	400	+ }
	401	+
	402	+ if (memchr_inv(udata + moff, 0, msize)) {
	403	+ err = -EINVAL;
	404	+ goto reset_unlock;
	405	+ }
	406	+
	407	+ continue;
	408	+ }
	409	+
	410	+ prog_fd = (int)((unsigned long )(udata + moff));
	411	+ /* Similar check as the attr->attach_prog_fd */
	412	+ if (!prog_fd)
	413	+ continue;
	414	+
	415	+ prog = bpf_prog_get(prog_fd);
	416	+ if (IS_ERR(prog)) {
	417	+ err = PTR_ERR(prog);
	418	+ goto reset_unlock;
	419	+ }
	420	+ st_map->progs[i] = prog;
	421	+
	422	+ if (prog->type != BPF_PROG_TYPE_STRUCT_OPS \|\|
	423	+ prog->aux->attach_btf_id != st_ops->type_id \|\|
	424	+ prog->expected_attach_type != i) {
	425	+ err = -EINVAL;
	426	+ goto reset_unlock;
	427	+ }
	428	+
	429	+ err = arch_prepare_bpf_trampoline(image,
	430	+ st_map->image + PAGE_SIZE,
	431	+ &st_ops->func_models[i], 0,
	432	+ &prog, 1, NULL, 0, NULL);
	433	+ if (err < 0)
	434	+ goto reset_unlock;
	435	+
	436	+ (void *)(kdata + moff) = image;
	437	+ image += err;
	438	+
	439	+ /* put prog_id to udata */
	440	+ (unsigned long )(udata + moff) = prog->aux->id;
	441	+ }
	442	+
	443	+ refcount_set(&kvalue->refcnt, 1);
	444	+ bpf_map_inc(map);
	445	+
	446	+ set_memory_ro((long)st_map->image, 1);
	447	+ set_memory_x((long)st_map->image, 1);
	448	+ err = st_ops->reg(kdata);
	449	+ if (likely(!err)) {
	450	+ /* Pair with smp_load_acquire() during lookup_elem().
	451	+ * It ensures the above udata updates (e.g. prog->aux->id)
	452	+ * can be seen once BPF_STRUCT_OPS_STATE_INUSE is set.
	453	+ */
	454	+ smp_store_release(&kvalue->state, BPF_STRUCT_OPS_STATE_INUSE);
	455	+ goto unlock;
	456	+ }
	457	+
	458	+ /* Error during st_ops->reg(). It is very unlikely since
	459	+ * the above init_member() should have caught it earlier
	460	+ * before reg(). The only possibility is if there was a race
	461	+ * in registering the struct_ops (under the same name) to
	462	+ * a sub-system through different struct_ops's maps.
	463	+ */
	464	+ set_memory_nx((long)st_map->image, 1);
	465	+ set_memory_rw((long)st_map->image, 1);
	466	+ bpf_map_put(map);
	467	+
	468	+reset_unlock:
	469	+ bpf_struct_ops_map_put_progs(st_map);
	470	+ memset(uvalue, 0, map->value_size);
	471	+ memset(kvalue, 0, map->value_size);
	472	+unlock:
	473	+ mutex_unlock(&st_map->lock);
	474	+ return err;
	475	+}
	476	+
	477	+static int bpf_struct_ops_map_delete_elem(struct bpf_map map, void key)
	478	+{
	479	+ enum bpf_struct_ops_state prev_state;
	480	+ struct bpf_struct_ops_map *st_map;
	481	+
	482	+ st_map = (struct bpf_struct_ops_map *)map;
	483	+ prev_state = cmpxchg(&st_map->kvalue.state,
	484	+ BPF_STRUCT_OPS_STATE_INUSE,
	485	+ BPF_STRUCT_OPS_STATE_TOBEFREE);
	486	+ if (prev_state == BPF_STRUCT_OPS_STATE_INUSE) {
	487	+ st_map->st_ops->unreg(&st_map->kvalue.data);
	488	+ if (refcount_dec_and_test(&st_map->kvalue.refcnt))
	489	+ bpf_map_put(map);
	490	+ }
	491	+
	492	+ return 0;
	493	+}
	494	+
	495	+static void bpf_struct_ops_map_seq_show_elem(struct bpf_map map, void key,
	496	+ struct seq_file *m)
	497	+{
	498	+ void *value;
	499	+
	500	+ value = bpf_struct_ops_map_lookup_elem(map, key);
	501	+ if (!value)
	502	+ return;
	503	+
	504	+ btf_type_seq_show(btf_vmlinux, map->btf_vmlinux_value_type_id,
	505	+ value, m);
	506	+ seq_puts(m, "\n");
	507	+}
	508	+
	509	+static void bpf_struct_ops_map_free(struct bpf_map *map)
	510	+{
	511	+ struct bpf_struct_ops_map st_map = (struct bpf_struct_ops_map )map;
	512	+
	513	+ if (st_map->progs)
	514	+ bpf_struct_ops_map_put_progs(st_map);
	515	+ bpf_map_area_free(st_map->progs);
	516	+ bpf_jit_free_exec(st_map->image);
	517	+ bpf_map_area_free(st_map->uvalue);
	518	+ bpf_map_area_free(st_map);
	519	+}
	520	+
	521	+static int bpf_struct_ops_map_alloc_check(union bpf_attr *attr)
	522	+{
	523	+ if (attr->key_size != sizeof(unsigned int) \|\| attr->max_entries != 1 \|\|
	524	+ attr->map_flags \|\| !attr->btf_vmlinux_value_type_id)
	525	+ return -EINVAL;
	526	+ return 0;
	527	+}
	528	+
	529	+static struct bpf_map bpf_struct_ops_map_alloc(union bpf_attr attr)
	530	+{
	531	+ const struct bpf_struct_ops *st_ops;
	532	+ size_t map_total_size, st_map_size;
	533	+ struct bpf_struct_ops_map *st_map;
	534	+ const struct btf_type t, vt;
	535	+ struct bpf_map_memory mem;
	536	+ struct bpf_map *map;
	537	+ int err;
	538	+
	539	+ if (!capable(CAP_SYS_ADMIN))
	540	+ return ERR_PTR(-EPERM);
	541	+
	542	+ st_ops = bpf_struct_ops_find_value(attr->btf_vmlinux_value_type_id);
	543	+ if (!st_ops)
	544	+ return ERR_PTR(-ENOTSUPP);
	545	+
	546	+ vt = st_ops->value_type;
	547	+ if (attr->value_size != vt->size)
	548	+ return ERR_PTR(-EINVAL);
	549	+
	550	+ t = st_ops->type;
	551	+
	552	+ st_map_size = sizeof(*st_map) +
	553	+ /* kvalue stores the
	554	+ * struct bpf_struct_ops_tcp_congestions_ops
	555	+ */
	556	+ (vt->size - sizeof(struct bpf_struct_ops_value));
	557	+ map_total_size = st_map_size +
	558	+ /* uvalue */
	559	+ sizeof(vt->size) +
	560	+ /* struct bpf_progs *progs /
	561	+ btf_type_vlen(t) * sizeof(struct bpf_prog *);
	562	+ err = bpf_map_charge_init(&mem, map_total_size);
	563	+ if (err < 0)
	564	+ return ERR_PTR(err);
	565	+
	566	+ st_map = bpf_map_area_alloc(st_map_size, NUMA_NO_NODE);
	567	+ if (!st_map) {
	568	+ bpf_map_charge_finish(&mem);
	569	+ return ERR_PTR(-ENOMEM);
	570	+ }
	571	+ st_map->st_ops = st_ops;
	572	+ map = &st_map->map;
	573	+
	574	+ st_map->uvalue = bpf_map_area_alloc(vt->size, NUMA_NO_NODE);
	575	+ st_map->progs =
	576	+ bpf_map_area_alloc(btf_type_vlen(t) * sizeof(struct bpf_prog *),
	577	+ NUMA_NO_NODE);
	578	+ st_map->image = bpf_jit_alloc_exec(PAGE_SIZE);
	579	+ if (!st_map->uvalue \|\| !st_map->progs \|\| !st_map->image) {
	580	+ bpf_struct_ops_map_free(map);
	581	+ bpf_map_charge_finish(&mem);
	582	+ return ERR_PTR(-ENOMEM);
	583	+ }
	584	+
	585	+ mutex_init(&st_map->lock);
	586	+ set_vm_flush_reset_perms(st_map->image);
	587	+ bpf_map_init_from_attr(map, attr);
	588	+ bpf_map_charge_move(&map->memory, &mem);
	589	+
	590	+ return map;
	591	+}
	592	+
	593	+const struct bpf_map_ops bpf_struct_ops_map_ops = {
	594	+ .map_alloc_check = bpf_struct_ops_map_alloc_check,
	595	+ .map_alloc = bpf_struct_ops_map_alloc,
	596	+ .map_free = bpf_struct_ops_map_free,
	597	+ .map_get_next_key = bpf_struct_ops_map_get_next_key,
	598	+ .map_lookup_elem = bpf_struct_ops_map_lookup_elem,
	599	+ .map_delete_elem = bpf_struct_ops_map_delete_elem,
	600	+ .map_update_elem = bpf_struct_ops_map_update_elem,
	601	+ .map_seq_show_elem = bpf_struct_ops_map_seq_show_elem,
	602	+};
	603	+
	604	+/* "const void *" because some subsystem is
	605	+ * passing a const (e.g. const struct tcp_congestion_ops *)
	606	+ */
	607	+bool bpf_struct_ops_get(const void *kdata)
	608	+{
	609	+ struct bpf_struct_ops_value *kvalue;
	610	+
	611	+ kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
	612	+
	613	+ return refcount_inc_not_zero(&kvalue->refcnt);
	614	+}
	615	+
	616	+void bpf_struct_ops_put(const void *kdata)
	617	+{
	618	+ struct bpf_struct_ops_value *kvalue;
	619	+
	620	+ kvalue = container_of(kdata, struct bpf_struct_ops_value, data);
	621	+ if (refcount_dec_and_test(&kvalue->refcnt)) {
	622	+ struct bpf_struct_ops_map *st_map;
	623	+
	624	+ st_map = container_of(kvalue, struct bpf_struct_ops_map,
	625	+ kvalue);
	626	+ bpf_map_put(&st_map->map);
	627	+ }
	628	+}

kernel/bpf/bpf_struct_ops_types.h+9−0

@@ -0,0 +1,9 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/* internal file - do not include directly */
	3	+
	4	+#ifdef CONFIG_BPF_JIT
	5	+#ifdef CONFIG_INET
	6	+#include <net/tcp.h>
	7	+BPF_STRUCT_OPS_TYPE(tcp_congestion_ops)
	8	+#endif
	9	+#endif

kernel/bpf/btf.c+116−47

@@ -180,11 +180,6 @@ · 179 unmodified lines
180	180	*/
181	181	#define BTF_MAX_SIZE (16 * 1024 * 1024)
182	182
183		−#define for_each_member(i, struct_type, member) \
184		− for (i = 0, member = btf_type_member(struct_type); \
185		− i < btf_type_vlen(struct_type); \
186		− i++, member++)
187		−
188	183	#define for_each_member_from(i, from, struct_type, member) \
189	184	for (i = from, member = btf_type_member(struct_type) + from; \
190	185	i < btf_type_vlen(struct_type); \
@@ -382,6 +377,65 @@ static bool btf_type_is_datasec(const struct btf_type *t) · 191 unmodified lines
382	377	return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
383	378	}
384	379
	380	+s32 btf_find_by_name_kind(const struct btf btf, const char name, u8 kind)
	381	+{
	382	+ const struct btf_type *t;
	383	+ const char *tname;
	384	+ u32 i;
	385	+
	386	+ for (i = 1; i <= btf->nr_types; i++) {
	387	+ t = btf->types[i];
	388	+ if (BTF_INFO_KIND(t->info) != kind)
	389	+ continue;
	390	+
	391	+ tname = btf_name_by_offset(btf, t->name_off);
	392	+ if (!strcmp(tname, name))
	393	+ return i;
	394	+ }
	395	+
	396	+ return -ENOENT;
	397	+}
	398	+
	399	+const struct btf_type btf_type_skip_modifiers(const struct btf btf,
	400	+ u32 id, u32 *res_id)
	401	+{
	402	+ const struct btf_type *t = btf_type_by_id(btf, id);
	403	+
	404	+ while (btf_type_is_modifier(t)) {
	405	+ id = t->type;
	406	+ t = btf_type_by_id(btf, t->type);
	407	+ }
	408	+
	409	+ if (res_id)
	410	+ *res_id = id;
	411	+
	412	+ return t;
	413	+}
	414	+
	415	+const struct btf_type btf_type_resolve_ptr(const struct btf btf,
	416	+ u32 id, u32 *res_id)
	417	+{
	418	+ const struct btf_type *t;
	419	+
	420	+ t = btf_type_skip_modifiers(btf, id, NULL);
	421	+ if (!btf_type_is_ptr(t))
	422	+ return NULL;
	423	+
	424	+ return btf_type_skip_modifiers(btf, t->type, res_id);
	425	+}
	426	+
	427	+const struct btf_type btf_type_resolve_func_ptr(const struct btf btf,
	428	+ u32 id, u32 *res_id)
	429	+{
	430	+ const struct btf_type *ptype;
	431	+
	432	+ ptype = btf_type_resolve_ptr(btf, id, res_id);
	433	+ if (ptype && btf_type_is_func_proto(ptype))
	434	+ return ptype;
	435	+
	436	+ return NULL;
	437	+}
	438	+
385	439	/* Types that act only as a source, not sink or intermediate
386	440	* type when resolving.
387	441	*/
@@ -446,30 +500,6 @@ static const char *btf_int_encoding_str(u8 encoding) · 58 unmodified lines
446	500	return "UNKN";
447	501	}
448	502
449		−static u16 btf_type_vlen(const struct btf_type *t)
450		−{
451		− return BTF_INFO_VLEN(t->info);
452		−}
453		−
454		−static bool btf_type_kflag(const struct btf_type *t)
455		−{
456		− return BTF_INFO_KFLAG(t->info);
457		−}
458		−
459		−static u32 btf_member_bit_offset(const struct btf_type *struct_type,
460		− const struct btf_member *member)
461		−{
462		− return btf_type_kflag(struct_type) ? BTF_MEMBER_BIT_OFFSET(member->offset)
463		− : member->offset;
464		−}
465		−
466		−static u32 btf_member_bitfield_size(const struct btf_type *struct_type,
467		− const struct btf_member *member)
468		−{
469		− return btf_type_kflag(struct_type) ? BTF_MEMBER_BITFIELD_SIZE(member->offset)
470		− : 0;
471		−}
472		−
473	503	static u32 btf_type_int(const struct btf_type *t)
474	504	{
475	505	return (u32 )(t + 1);
@@ -480,11 +510,6 @@ static const struct btf_array btf_type_array(const struct btf_type t) · 4 unmodified lines
480	510	return (const struct btf_array *)(t + 1);
481	511	}
482	512
483		−static const struct btf_member btf_type_member(const struct btf_type t)
484		−{
485		− return (const struct btf_member *)(t + 1);
486		−}
487		−
488	513	static const struct btf_enum btf_type_enum(const struct btf_type t)
489	514	{
490	515	return (const struct btf_enum *)(t + 1);
@@ -1057,7 +1082,7 @@ static const struct resolve_vertex env_stack_peak(struct btf_verifier_env env) · 566 unmodified lines
1057	1082	* *elem_type: same as return type ("struct X")
1058	1083	* *total_nelems: 1
1059	1084	*/
1060		−static const struct btf_type *
	1085	+const struct btf_type *
1061	1086	btf_resolve_size(const struct btf btf, const struct btf_type type,
1062	1087	u32 type_size, const struct btf_type *elem_type,
1063	1088	u32 *total_nelems)
@@ -1111,8 +1136,10 @@ resolved: · 47 unmodified lines
1111	1136	return ERR_PTR(-EINVAL);
1112	1137
1113	1138	type_size = nelems size;
1114		− *total_nelems = nelems;
1115		− *elem_type = type;
	1139	+ if (total_nelems)
	1140	+ *total_nelems = nelems;
	1141	+ if (elem_type)
	1142	+ *elem_type = type;
1116	1143
1117	1144	return array_type ? : type;
1118	1145	}
@@ -1826,7 +1853,10 @@ static void btf_modifier_seq_show(const struct btf *btf, · 707 unmodified lines
1826	1853	u32 type_id, void *data,
1827	1854	u8 bits_offset, struct seq_file *m)
1828	1855	{
1829		− t = btf_type_id_resolve(btf, &type_id);
	1856	+ if (btf->resolved_ids)
	1857	+ t = btf_type_id_resolve(btf, &type_id);
	1858	+ else
	1859	+ t = btf_type_skip_modifiers(btf, type_id, NULL);
1830	1860
1831	1861	btf_type_ops(t)->seq_show(btf, t, type_id, data, bits_offset, m);
1832	1862	}
@@ -3605,6 +3635,8 @@ struct btf *btf_parse_vmlinux(void) · 1772 unmodified lines
3605	3635	goto errout;
3606	3636	}
3607	3637
	3638	+ bpf_struct_ops_init(btf);
	3639	+
3608	3640	btf_verifier_env_free(env);
3609	3641	refcount_set(&btf->refcnt, 1);
3610	3642	return btf;
@@ -3677,7 +3709,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 66 unmodified lines
3677	3709	/* skip modifiers */
3678	3710	while (btf_type_is_modifier(t))
3679	3711	t = btf_type_by_id(btf, t->type);
3680		− if (btf_type_is_int(t))
	3712	+ if (btf_type_is_int(t) \|\| btf_type_is_enum(t))
3681	3713	/* accessing a scalar */
3682	3714	return true;
3683	3715	if (!btf_type_is_ptr(t)) {
@@ -3697,7 +3729,6 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 13 unmodified lines
3697	3729
3698	3730	/* this is a pointer to another type */
3699	3731	info->reg_type = PTR_TO_BTF_ID;
3700		− info->btf_id = t->type;
3701	3732
3702	3733	if (tgt_prog) {
3703	3734	ret = btf_translate_to_vmlinux(log, btf, t, tgt_prog->type);
@@ -3708,10 +3739,14 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type, · 4 unmodified lines
3708	3739	return false;
3709	3740	}
3710	3741	}
	3742	+
	3743	+ info->btf_id = t->type;
3711	3744	t = btf_type_by_id(btf, t->type);
3712	3745	/* skip modifiers */
3713		− while (btf_type_is_modifier(t))
	3746	+ while (btf_type_is_modifier(t)) {
	3747	+ info->btf_id = t->type;
3714	3748	t = btf_type_by_id(btf, t->type);
	3749	+ }
3715	3750	if (!btf_type_is_struct(t)) {
3716	3751	bpf_log(log,
3717	3752	"func '%s' arg%d type %s is not a struct\n",
@@ -3737,23 +3772,57 @@ int btf_struct_access(struct bpf_verifier_log *log, · 19 unmodified lines
3737	3772	again:
3738	3773	tname = __btf_name_by_offset(btf_vmlinux, t->name_off);
3739	3774	if (!btf_type_is_struct(t)) {
3740		− bpf_log(log, "Type '%s' is not a struct", tname);
	3775	+ bpf_log(log, "Type '%s' is not a struct\n", tname);
3741	3776	return -EINVAL;
3742	3777	}
3743	3778
3744		− for_each_member(i, t, member) {
3745		− if (btf_member_bitfield_size(t, member))
3746		− /* bitfields are not supported yet */
3747		− continue;
	3779	+ if (off + size > t->size) {
	3780	+ bpf_log(log, "access beyond struct %s at off %u size %u\n",
	3781	+ tname, off, size);
	3782	+ return -EACCES;
	3783	+ }
3748	3784
	3785	+ for_each_member(i, t, member) {
3749	3786	/* offset of the field in bytes */
3750	3787	moff = btf_member_bit_offset(t, member) / 8;
3751	3788	if (off + size <= moff)
3752	3789	/* won't find anything, field is already too far */
3753	3790	break;
	3791	+
	3792	+ if (btf_member_bitfield_size(t, member)) {
	3793	+ u32 end_bit = btf_member_bit_offset(t, member) +
	3794	+ btf_member_bitfield_size(t, member);
	3795	+
	3796	+ /* off <= moff instead of off == moff because clang
	3797	+ * does not generate a BTF member for anonymous
	3798	+ * bitfield like the ":16" here:
	3799	+ * struct {
	3800	+ * int :16;
	3801	+ * int x:8;
	3802	+ * };
	3803	+ */
	3804	+ if (off <= moff &&
	3805	+ BITS_ROUNDUP_BYTES(end_bit) <= off + size)
	3806	+ return SCALAR_VALUE;
	3807	+
	3808	+ /* off may be accessing a following member
	3809	+ *
	3810	+ * or
	3811	+ *
	3812	+ * Doing partial access at either end of this
	3813	+ * bitfield. Continue on this case also to
	3814	+ * treat it as not accessing this bitfield
	3815	+ * and eventually error out as field not
	3816	+ * found to keep it simple.
	3817	+ * It could be relaxed if there was a legit
	3818	+ * partial access case later.
	3819	+ */
	3820	+ continue;
	3821	+ }
	3822	+
3754	3823	/* In case of "off" is pointing to holes of a struct */
3755	3824	if (off < moff)
3756		− continue;
	3825	+ break;
3757	3826
3758	3827	/* type of the field */
3759	3828	mtype = btf_type_by_id(btf_vmlinux, member->type);

kernel/bpf/map_in_map.c+2−1

@@ -22,7 +22,8 @@ struct bpf_map *bpf_map_meta_alloc(int inner_map_ufd) · 21 unmodified lines
22	22	*/
23	23	if (inner_map->map_type == BPF_MAP_TYPE_PROG_ARRAY \|\|
24	24	inner_map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE \|\|
25		− inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) {
	25	+ inner_map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE \|\|
	26	+ inner_map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
26	27	fdput(f);
27	28	return ERR_PTR(-ENOTSUPP);
28	29	}

kernel/bpf/syscall.c+46−23

@@ -628,7 +628,7 @@ static int map_check_btf(struct bpf_map map, const struct btf btf, · 627 unmodified lines
628	628	return ret;
629	629	}
630	630
631		−#define BPF_MAP_CREATE_LAST_FIELD btf_value_type_id
	631	+#define BPF_MAP_CREATE_LAST_FIELD btf_vmlinux_value_type_id
632	632	/* called via syscall */
633	633	static int map_create(union bpf_attr *attr)
634	634	{
@@ -642,6 +642,14 @@ static int map_create(union bpf_attr *attr) · 7 unmodified lines
642	642	if (err)
643	643	return -EINVAL;
644	644
	645	+ if (attr->btf_vmlinux_value_type_id) {
	646	+ if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS \|\|
	647	+ attr->btf_key_type_id \|\| attr->btf_value_type_id)
	648	+ return -EINVAL;
	649	+ } else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
	650	+ return -EINVAL;
	651	+ }
	652	+
645	653	f_flags = bpf_get_file_flag(attr->map_flags);
646	654	if (f_flags < 0)
647	655	return f_flags;
@@ -664,32 +672,35 @@ static int map_create(union bpf_attr *attr) · 16 unmodified lines
664	672	atomic64_set(&map->usercnt, 1);
665	673	mutex_init(&map->freeze_mutex);
666	674
667		− if (attr->btf_key_type_id \|\| attr->btf_value_type_id) {
	675	+ map->spin_lock_off = -EINVAL;
	676	+ if (attr->btf_key_type_id \|\| attr->btf_value_type_id \|\|
	677	+ /* Even the map's value is a kernel's struct,
	678	+ * the bpf_prog.o must have BTF to begin with
	679	+ * to figure out the corresponding kernel's
	680	+ * counter part. Thus, attr->btf_fd has
	681	+ * to be valid also.
	682	+ */
	683	+ attr->btf_vmlinux_value_type_id) {
668	684	struct btf *btf;
669	685
670		− if (!attr->btf_value_type_id) {
671		− err = -EINVAL;
672		− goto free_map;
673		− }
674		−
675	686	btf = btf_get_by_fd(attr->btf_fd);
676	687	if (IS_ERR(btf)) {
677	688	err = PTR_ERR(btf);
678	689	goto free_map;
679	690	}
	691	+ map->btf = btf;
680	692
681		− err = map_check_btf(map, btf, attr->btf_key_type_id,
682		− attr->btf_value_type_id);
683		− if (err) {
684		− btf_put(btf);
685		− goto free_map;
	693	+ if (attr->btf_value_type_id) {
	694	+ err = map_check_btf(map, btf, attr->btf_key_type_id,
	695	+ attr->btf_value_type_id);
	696	+ if (err)
	697	+ goto free_map;
686	698	}
687	699
688		− map->btf = btf;
689	700	map->btf_key_type_id = attr->btf_key_type_id;
690	701	map->btf_value_type_id = attr->btf_value_type_id;
691		− } else {
692		− map->spin_lock_off = -EINVAL;
	702	+ map->btf_vmlinux_value_type_id =
	703	+ attr->btf_vmlinux_value_type_id;
693	704	}
694	705
695	706	err = security_bpf_map_alloc(map);
@@ -888,6 +899,9 @@ static int map_lookup_elem(union bpf_attr *attr) · 192 unmodified lines
888	899	} else if (map->map_type == BPF_MAP_TYPE_QUEUE \|\|
889	900	map->map_type == BPF_MAP_TYPE_STACK) {
890	901	err = map->ops->map_peek_elem(map, value);
	902	+ } else if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
	903	+ /* struct_ops map requires directly updating "value" */
	904	+ err = bpf_struct_ops_map_sys_lookup_elem(map, key, value);
891	905	} else {
892	906	rcu_read_lock();
893	907	if (map->ops->map_lookup_elem_sys_only)
@@ -1003,7 +1017,8 @@ static int map_update_elem(union bpf_attr *attr) · 109 unmodified lines
1003	1017	goto out;
1004	1018	} else if (map->map_type == BPF_MAP_TYPE_CPUMAP \|\|
1005	1019	map->map_type == BPF_MAP_TYPE_SOCKHASH \|\|
1006		− map->map_type == BPF_MAP_TYPE_SOCKMAP) {
	1020	+ map->map_type == BPF_MAP_TYPE_SOCKMAP \|\|
	1021	+ map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
1007	1022	err = map->ops->map_update_elem(map, key, value, attr->flags);
1008	1023	goto out;
1009	1024	} else if (IS_FD_PROG_ARRAY(map)) {
@@ -1092,7 +1107,9 @@ static int map_delete_elem(union bpf_attr *attr) · 82 unmodified lines
1092	1107	if (bpf_map_is_dev_bound(map)) {
1093	1108	err = bpf_map_offload_delete_elem(map, key);
1094	1109	goto out;
1095		− } else if (IS_FD_PROG_ARRAY(map)) {
	1110	+ } else if (IS_FD_PROG_ARRAY(map) \|\|
	1111	+ map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
	1112	+ /* These maps require sleepable context */
1096	1113	err = map->ops->map_delete_elem(map, key);
1097	1114	goto out;
1098	1115	}
@@ -1672,17 +1689,22 @@ bpf_prog_load_check_attach(enum bpf_prog_type prog_type, · 573 unmodified lines
1672	1689	enum bpf_attach_type expected_attach_type,
1673	1690	u32 btf_id, u32 prog_fd)
1674	1691	{
1675		− switch (prog_type) {
1676		− case BPF_PROG_TYPE_TRACING:
	1692	+ if (btf_id) {
1677	1693	if (btf_id > BTF_MAX_TYPE)
1678	1694	return -EINVAL;
1679		− break;
1680		− default:
1681		− if (btf_id \|\| prog_fd)
	1695	+
	1696	+ switch (prog_type) {
	1697	+ case BPF_PROG_TYPE_TRACING:
	1698	+ case BPF_PROG_TYPE_STRUCT_OPS:
	1699	+ break;
	1700	+ default:
1682	1701	return -EINVAL;
1683		− break;
	1702	+ }
1684	1703	}
1685	1704
	1705	+ if (prog_fd && prog_type != BPF_PROG_TYPE_TRACING)
	1706	+ return -EINVAL;
	1707	+
1686	1708	switch (prog_type) {
1687	1709	case BPF_PROG_TYPE_CGROUP_SOCK:
1688	1710	switch (expected_attach_type) {
@@ -2817,6 +2839,7 @@ static int bpf_map_get_info_by_fd(struct bpf_map *map, · 1128 unmodified lines
2817	2839	info.btf_key_type_id = map->btf_key_type_id;
2818	2840	info.btf_value_type_id = map->btf_value_type_id;
2819	2841	}
	2842	+ info.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
2820	2843
2821	2844	if (bpf_map_is_dev_bound(map)) {
2822	2845	err = bpf_map_offload_info_fill(&info, map);

kernel/bpf/trampoline.c+5−3

@@ -160,11 +160,12 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr) · 159 unmodified lines
160	160	if (fexit_cnt)
161	161	flags = BPF_TRAMP_F_CALL_ORIG \| BPF_TRAMP_F_SKIP_FRAME;
162	162
163		− err = arch_prepare_bpf_trampoline(new_image, &tr->func.model, flags,
	163	+ err = arch_prepare_bpf_trampoline(new_image, new_image + PAGE_SIZE / 2,
	164	+ &tr->func.model, flags,
164	165	fentry, fentry_cnt,
165	166	fexit, fexit_cnt,
166	167	tr->func.addr);
167		− if (err)
	168	+ if (err < 0)
168	169	goto out;
169	170
170	171	if (tr->selector)
@@ -296,7 +297,8 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start) · 125 unmodified lines
296	297	}
297	298
298	299	int __weak
299		−arch_prepare_bpf_trampoline(void image, struct btf_func_model m, u32 flags,
	300	+arch_prepare_bpf_trampoline(void image, void image_end,
	301	+ const struct btf_func_model *m, u32 flags,
300	302	struct bpf_prog **fentry_progs, int fentry_cnt,
301	303	struct bpf_prog **fexit_progs, int fexit_cnt,
302	304	void *orig_call)

kernel/bpf/verifier.c+109−30

@@ -2859,11 +2859,6 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 2858 unmodified lines
2859	2859	u32 btf_id;
2860	2860	int ret;
2861	2861
2862		− if (atype != BPF_READ) {
2863		− verbose(env, "only read is supported\n");
2864		− return -EACCES;
2865		− }
2866		−
2867	2862	if (off < 0) {
2868	2863	verbose(env,
2869	2864	"R%d is ptr_%s invalid negative access: off=%d\n",
@@ -2880,17 +2875,32 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env, · 10 unmodified lines
2880	2875	return -EACCES;
2881	2876	}
2882	2877
2883		− ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id);
	2878	+ if (env->ops->btf_struct_access) {
	2879	+ ret = env->ops->btf_struct_access(&env->log, t, off, size,
	2880	+ atype, &btf_id);
	2881	+ } else {
	2882	+ if (atype != BPF_READ) {
	2883	+ verbose(env, "only read is supported\n");
	2884	+ return -EACCES;
	2885	+ }
	2886	+
	2887	+ ret = btf_struct_access(&env->log, t, off, size, atype,
	2888	+ &btf_id);
	2889	+ }
	2890	+
2884	2891	if (ret < 0)
2885	2892	return ret;
2886	2893
2887		− if (ret == SCALAR_VALUE) {
2888		− mark_reg_unknown(env, regs, value_regno);
2889		− return 0;
	2894	+ if (atype == BPF_READ) {
	2895	+ if (ret == SCALAR_VALUE) {
	2896	+ mark_reg_unknown(env, regs, value_regno);
	2897	+ return 0;
	2898	+ }
	2899	+ mark_reg_known_zero(env, regs, value_regno);
	2900	+ regs[value_regno].type = PTR_TO_BTF_ID;
	2901	+ regs[value_regno].btf_id = btf_id;
2890	2902	}
2891		− mark_reg_known_zero(env, regs, value_regno);
2892		− regs[value_regno].type = PTR_TO_BTF_ID;
2893		− regs[value_regno].btf_id = btf_id;
	2903	+
2894	2904	return 0;
2895	2905	}
2896	2906
@@ -6349,8 +6359,30 @@ static int check_ld_abs(struct bpf_verifier_env env, struct bpf_insn insn) · 3452 unmodified lines
6349	6359	static int check_return_code(struct bpf_verifier_env *env)
6350	6360	{
6351	6361	struct tnum enforce_attach_type_range = tnum_unknown;
	6362	+ const struct bpf_prog *prog = env->prog;
6352	6363	struct bpf_reg_state *reg;
6353	6364	struct tnum range = tnum_range(0, 1);
	6365	+ int err;
	6366	+
	6367	+ /* The struct_ops func-ptr's return type could be "void" */
	6368	+ if (env->prog->type == BPF_PROG_TYPE_STRUCT_OPS &&
	6369	+ !prog->aux->attach_func_proto->type)
	6370	+ return 0;
	6371	+
	6372	+ /* eBPF calling convetion is such that R0 is used
	6373	+ * to return the value from eBPF program.
	6374	+ * Make sure that it's readable at this time
	6375	+ * of bpf_exit, which means that program wrote
	6376	+ * something into it earlier
	6377	+ */
	6378	+ err = check_reg_arg(env, BPF_REG_0, SRC_OP);
	6379	+ if (err)
	6380	+ return err;
	6381	+
	6382	+ if (is_pointer_value(env, BPF_REG_0)) {
	6383	+ verbose(env, "R0 leaks addr as return value\n");
	6384	+ return -EACCES;
	6385	+ }
6354	6386
6355	6387	switch (env->prog->type) {
6356	6388	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
@@ -8016,21 +8048,6 @@ static int do_check(struct bpf_verifier_env *env) · 1659 unmodified lines
8016	8048	if (err)
8017	8049	return err;
8018	8050
8019		− /* eBPF calling convetion is such that R0 is used
8020		− * to return the value from eBPF program.
8021		− * Make sure that it's readable at this time
8022		− * of bpf_exit, which means that program wrote
8023		− * something into it earlier
8024		− */
8025		− err = check_reg_arg(env, BPF_REG_0, SRC_OP);
8026		− if (err)
8027		− return err;
8028		−
8029		− if (is_pointer_value(env, BPF_REG_0)) {
8030		− verbose(env, "R0 leaks addr as return value\n");
8031		− return -EACCES;
8032		− }
8033		−
8034	8051	err = check_return_code(env);
8035	8052	if (err)
8036	8053	return err;
@@ -8138,6 +8155,11 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env, · 101 unmodified lines
8138	8155	return -EINVAL;
8139	8156	}
8140	8157
	8158	+ if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
	8159	+ verbose(env, "bpf_struct_ops map cannot be used in prog\n");
	8160	+ return -EINVAL;
	8161	+ }
	8162	+
8141	8163	return 0;
8142	8164	}
8143	8165
@@ -8829,12 +8851,14 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env) · 685 unmodified lines
8829	8851	convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
8830	8852	break;
8831	8853	case PTR_TO_BTF_ID:
8832		− if (type == BPF_WRITE) {
	8854	+ if (type == BPF_READ) {
	8855	+ insn->code = BPF_LDX \| BPF_PROBE_MEM \|
	8856	+ BPF_SIZE((insn)->code);
	8857	+ env->prog->aux->num_exentries++;
	8858	+ } else if (env->prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
8833	8859	verbose(env, "Writes through BTF pointers are not allowed\n");
8834	8860	return -EINVAL;
8835	8861	}
8836		− insn->code = BPF_LDX \| BPF_PROBE_MEM \| BPF_SIZE((insn)->code);
8837		− env->prog->aux->num_exentries++;
8838	8862	continue;
8839	8863	default:
8840	8864	continue;
@@ -9502,6 +9526,58 @@ static void print_verification_stats(struct bpf_verifier_env *env) · 661 unmodified lines
9502	9526	env->peak_states, env->longest_mark_read_walk);
9503	9527	}
9504	9528
	9529	+static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
	9530	+{
	9531	+ const struct btf_type t, func_proto;
	9532	+ const struct bpf_struct_ops *st_ops;
	9533	+ const struct btf_member *member;
	9534	+ struct bpf_prog *prog = env->prog;
	9535	+ u32 btf_id, member_idx;
	9536	+ const char *mname;
	9537	+
	9538	+ btf_id = prog->aux->attach_btf_id;
	9539	+ st_ops = bpf_struct_ops_find(btf_id);
	9540	+ if (!st_ops) {
	9541	+ verbose(env, "attach_btf_id %u is not a supported struct\n",
	9542	+ btf_id);
	9543	+ return -ENOTSUPP;
	9544	+ }
	9545	+
	9546	+ t = st_ops->type;
	9547	+ member_idx = prog->expected_attach_type;
	9548	+ if (member_idx >= btf_type_vlen(t)) {
	9549	+ verbose(env, "attach to invalid member idx %u of struct %s\n",
	9550	+ member_idx, st_ops->name);
	9551	+ return -EINVAL;
	9552	+ }
	9553	+
	9554	+ member = &btf_type_member(t)[member_idx];
	9555	+ mname = btf_name_by_offset(btf_vmlinux, member->name_off);
	9556	+ func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type,
	9557	+ NULL);
	9558	+ if (!func_proto) {
	9559	+ verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
	9560	+ mname, member_idx, st_ops->name);
	9561	+ return -EINVAL;
	9562	+ }
	9563	+
	9564	+ if (st_ops->check_member) {
	9565	+ int err = st_ops->check_member(t, member);
	9566	+
	9567	+ if (err) {
	9568	+ verbose(env, "attach to unsupported member %s of struct %s\n",
	9569	+ mname, st_ops->name);
	9570	+ return err;
	9571	+ }
	9572	+ }
	9573	+
	9574	+ prog->aux->attach_func_proto = func_proto;
	9575	+ prog->aux->attach_func_name = mname;
	9576	+ env->ops = st_ops->verifier_ops;
	9577	+
	9578	+ return 0;
	9579	+}
	9580	+
9505	9581	static int check_attach_btf_id(struct bpf_verifier_env *env)
9506	9582	{
9507	9583	struct bpf_prog *prog = env->prog;
@@ -9517,6 +9593,9 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) · 9 unmodified lines
9517	9593	long addr;
9518	9594	u64 key;
9519	9595
	9596	+ if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
	9597	+ return check_struct_ops_btf_id(env);
	9598	+
9520	9599	if (prog->type != BPF_PROG_TYPE_TRACING)
9521	9600	return 0;
9522	9601

net/core/filter.c+1−1

@@ -5935,7 +5935,7 @@ bool bpf_helper_changes_pkt_data(void *func) · 5934 unmodified lines
5935	5935	return false;
5936	5936	}
5937	5937
5938		−static const struct bpf_func_proto *
	5938	+const struct bpf_func_proto *
5939	5939	bpf_base_func_proto(enum bpf_func_id func_id)
5940	5940	{
5941	5941	switch (func_id) {

net/ipv4/Makefile+4−0

@@ -65,3 +65,7 @@ obj-$(CONFIG_NETLABEL) += cipso_ipv4.o · 64 unmodified lines
65	65
66	66	obj-$(CONFIG_XFRM) += xfrm4_policy.o xfrm4_state.o xfrm4_input.o \
67	67	xfrm4_output.o xfrm4_protocol.o
	68	+
	69	+ifeq ($(CONFIG_BPF_JIT),y)
	70	+obj-$(CONFIG_BPF_SYSCALL) += bpf_tcp_ca.o
	71	+endif

net/ipv4/bpf_tcp_ca.c+252−0

@@ -0,0 +1,252 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2019 Facebook */
	3	+
	4	+#include <linux/types.h>
	5	+#include <linux/bpf_verifier.h>
	6	+#include <linux/bpf.h>
	7	+#include <linux/btf.h>
	8	+#include <linux/filter.h>
	9	+#include <net/tcp.h>
	10	+
	11	+static u32 optional_ops[] = {
	12	+ offsetof(struct tcp_congestion_ops, init),
	13	+ offsetof(struct tcp_congestion_ops, release),
	14	+ offsetof(struct tcp_congestion_ops, set_state),
	15	+ offsetof(struct tcp_congestion_ops, cwnd_event),
	16	+ offsetof(struct tcp_congestion_ops, in_ack_event),
	17	+ offsetof(struct tcp_congestion_ops, pkts_acked),
	18	+ offsetof(struct tcp_congestion_ops, min_tso_segs),
	19	+ offsetof(struct tcp_congestion_ops, sndbuf_expand),
	20	+ offsetof(struct tcp_congestion_ops, cong_control),
	21	+};
	22	+
	23	+static u32 unsupported_ops[] = {
	24	+ offsetof(struct tcp_congestion_ops, get_info),
	25	+};
	26	+
	27	+static const struct btf_type *tcp_sock_type;
	28	+static u32 tcp_sock_id, sock_id;
	29	+
	30	+static int bpf_tcp_ca_init(struct btf *btf)
	31	+{
	32	+ s32 type_id;
	33	+
	34	+ type_id = btf_find_by_name_kind(btf, "sock", BTF_KIND_STRUCT);
	35	+ if (type_id < 0)
	36	+ return -EINVAL;
	37	+ sock_id = type_id;
	38	+
	39	+ type_id = btf_find_by_name_kind(btf, "tcp_sock", BTF_KIND_STRUCT);
	40	+ if (type_id < 0)
	41	+ return -EINVAL;
	42	+ tcp_sock_id = type_id;
	43	+ tcp_sock_type = btf_type_by_id(btf, tcp_sock_id);
	44	+
	45	+ return 0;
	46	+}
	47	+
	48	+static bool is_optional(u32 member_offset)
	49	+{
	50	+ unsigned int i;
	51	+
	52	+ for (i = 0; i < ARRAY_SIZE(optional_ops); i++) {
	53	+ if (member_offset == optional_ops[i])
	54	+ return true;
	55	+ }
	56	+
	57	+ return false;
	58	+}
	59	+
	60	+static bool is_unsupported(u32 member_offset)
	61	+{
	62	+ unsigned int i;
	63	+
	64	+ for (i = 0; i < ARRAY_SIZE(unsupported_ops); i++) {
	65	+ if (member_offset == unsupported_ops[i])
	66	+ return true;
	67	+ }
	68	+
	69	+ return false;
	70	+}
	71	+
	72	+extern struct btf *btf_vmlinux;
	73	+
	74	+static bool bpf_tcp_ca_is_valid_access(int off, int size,
	75	+ enum bpf_access_type type,
	76	+ const struct bpf_prog *prog,
	77	+ struct bpf_insn_access_aux *info)
	78	+{
	79	+ if (off < 0 \|\| off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
	80	+ return false;
	81	+ if (type != BPF_READ)
	82	+ return false;
	83	+ if (off % size != 0)
	84	+ return false;
	85	+
	86	+ if (!btf_ctx_access(off, size, type, prog, info))
	87	+ return false;
	88	+
	89	+ if (info->reg_type == PTR_TO_BTF_ID && info->btf_id == sock_id)
	90	+ /* promote it to tcp_sock */
	91	+ info->btf_id = tcp_sock_id;
	92	+
	93	+ return true;
	94	+}
	95	+
	96	+static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log,
	97	+ const struct btf_type *t, int off,
	98	+ int size, enum bpf_access_type atype,
	99	+ u32 *next_btf_id)
	100	+{
	101	+ size_t end;
	102	+
	103	+ if (atype == BPF_READ)
	104	+ return btf_struct_access(log, t, off, size, atype, next_btf_id);
	105	+
	106	+ if (t != tcp_sock_type) {
	107	+ bpf_log(log, "only read is supported\n");
	108	+ return -EACCES;
	109	+ }
	110	+
	111	+ switch (off) {
	112	+ case bpf_ctx_range(struct inet_connection_sock, icsk_ca_priv):
	113	+ end = offsetofend(struct inet_connection_sock, icsk_ca_priv);
	114	+ break;
	115	+ case offsetof(struct inet_connection_sock, icsk_ack.pending):
	116	+ end = offsetofend(struct inet_connection_sock,
	117	+ icsk_ack.pending);
	118	+ break;
	119	+ case offsetof(struct tcp_sock, snd_cwnd):
	120	+ end = offsetofend(struct tcp_sock, snd_cwnd);
	121	+ break;
	122	+ case offsetof(struct tcp_sock, snd_cwnd_cnt):
	123	+ end = offsetofend(struct tcp_sock, snd_cwnd_cnt);
	124	+ break;
	125	+ case offsetof(struct tcp_sock, snd_ssthresh):
	126	+ end = offsetofend(struct tcp_sock, snd_ssthresh);
	127	+ break;
	128	+ case offsetof(struct tcp_sock, ecn_flags):
	129	+ end = offsetofend(struct tcp_sock, ecn_flags);
	130	+ break;
	131	+ default:
	132	+ bpf_log(log, "no write support to tcp_sock at off %d\n", off);
	133	+ return -EACCES;
	134	+ }
	135	+
	136	+ if (off + size > end) {
	137	+ bpf_log(log,
	138	+ "write access at off %d with size %d beyond the member of tcp_sock ended at %zu\n",
	139	+ off, size, end);
	140	+ return -EACCES;
	141	+ }
	142	+
	143	+ return NOT_INIT;
	144	+}
	145	+
	146	+BPF_CALL_2(bpf_tcp_send_ack, struct tcp_sock *, tp, u32, rcv_nxt)
	147	+{
	148	+ /* bpf_tcp_ca prog cannot have NULL tp */
	149	+ __tcp_send_ack((struct sock *)tp, rcv_nxt);
	150	+ return 0;
	151	+}
	152	+
	153	+static const struct bpf_func_proto bpf_tcp_send_ack_proto = {
	154	+ .func = bpf_tcp_send_ack,
	155	+ .gpl_only = false,
	156	+ /* In case we want to report error later */
	157	+ .ret_type = RET_INTEGER,
	158	+ .arg1_type = ARG_PTR_TO_BTF_ID,
	159	+ .arg2_type = ARG_ANYTHING,
	160	+ .btf_id = &tcp_sock_id,
	161	+};
	162	+
	163	+static const struct bpf_func_proto *
	164	+bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id,
	165	+ const struct bpf_prog *prog)
	166	+{
	167	+ switch (func_id) {
	168	+ case BPF_FUNC_tcp_send_ack:
	169	+ return &bpf_tcp_send_ack_proto;
	170	+ default:
	171	+ return bpf_base_func_proto(func_id);
	172	+ }
	173	+}
	174	+
	175	+static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {
	176	+ .get_func_proto = bpf_tcp_ca_get_func_proto,
	177	+ .is_valid_access = bpf_tcp_ca_is_valid_access,
	178	+ .btf_struct_access = bpf_tcp_ca_btf_struct_access,
	179	+};
	180	+
	181	+static int bpf_tcp_ca_init_member(const struct btf_type *t,
	182	+ const struct btf_member *member,
	183	+ void kdata, const void udata)
	184	+{
	185	+ const struct tcp_congestion_ops *utcp_ca;
	186	+ struct tcp_congestion_ops *tcp_ca;
	187	+ size_t tcp_ca_name_len;
	188	+ int prog_fd;
	189	+ u32 moff;
	190	+
	191	+ utcp_ca = (const struct tcp_congestion_ops *)udata;
	192	+ tcp_ca = (struct tcp_congestion_ops *)kdata;
	193	+
	194	+ moff = btf_member_bit_offset(t, member) / 8;
	195	+ switch (moff) {
	196	+ case offsetof(struct tcp_congestion_ops, flags):
	197	+ if (utcp_ca->flags & ~TCP_CONG_MASK)
	198	+ return -EINVAL;
	199	+ tcp_ca->flags = utcp_ca->flags;
	200	+ return 1;
	201	+ case offsetof(struct tcp_congestion_ops, name):
	202	+ tcp_ca_name_len = strnlen(utcp_ca->name, sizeof(utcp_ca->name));
	203	+ if (!tcp_ca_name_len \|\|
	204	+ tcp_ca_name_len == sizeof(utcp_ca->name))
	205	+ return -EINVAL;
	206	+ if (tcp_ca_find(utcp_ca->name))
	207	+ return -EEXIST;
	208	+ memcpy(tcp_ca->name, utcp_ca->name, sizeof(tcp_ca->name));
	209	+ return 1;
	210	+ }
	211	+
	212	+ if (!btf_type_resolve_func_ptr(btf_vmlinux, member->type, NULL))
	213	+ return 0;
	214	+
	215	+ /* Ensure bpf_prog is provided for compulsory func ptr */
	216	+ prog_fd = (int)((unsigned long )(udata + moff));
	217	+ if (!prog_fd && !is_optional(moff) && !is_unsupported(moff))
	218	+ return -EINVAL;
	219	+
	220	+ return 0;
	221	+}
	222	+
	223	+static int bpf_tcp_ca_check_member(const struct btf_type *t,
	224	+ const struct btf_member *member)
	225	+{
	226	+ if (is_unsupported(btf_member_bit_offset(t, member) / 8))
	227	+ return -ENOTSUPP;
	228	+ return 0;
	229	+}
	230	+
	231	+static int bpf_tcp_ca_reg(void *kdata)
	232	+{
	233	+ return tcp_register_congestion_control(kdata);
	234	+}
	235	+
	236	+static void bpf_tcp_ca_unreg(void *kdata)
	237	+{
	238	+ tcp_unregister_congestion_control(kdata);
	239	+}
	240	+
	241	+/* Avoid sparse warning. It is only used in bpf_struct_ops.c. */
	242	+extern struct bpf_struct_ops bpf_tcp_congestion_ops;
	243	+
	244	+struct bpf_struct_ops bpf_tcp_congestion_ops = {
	245	+ .verifier_ops = &bpf_tcp_ca_verifier_ops,
	246	+ .reg = bpf_tcp_ca_reg,
	247	+ .unreg = bpf_tcp_ca_unreg,
	248	+ .check_member = bpf_tcp_ca_check_member,
	249	+ .init_member = bpf_tcp_ca_init_member,
	250	+ .init = bpf_tcp_ca_init,
	251	+ .name = "tcp_congestion_ops",
	252	+};

net/ipv4/tcp_cong.c+8−8

@@ -21,7 +21,7 @@ static DEFINE_SPINLOCK(tcp_cong_list_lock); · 20 unmodified lines
21	21	static LIST_HEAD(tcp_cong_list);
22	22
23	23	/* Simple linear search, don't expect many entries! */
24		−static struct tcp_congestion_ops tcp_ca_find(const char name)
	24	+struct tcp_congestion_ops tcp_ca_find(const char name)
25	25	{
26	26	struct tcp_congestion_ops *e;
27	27
@@ -162,7 +162,7 @@ void tcp_assign_congestion_control(struct sock *sk) · 134 unmodified lines
162	162
163	163	rcu_read_lock();
164	164	ca = rcu_dereference(net->ipv4.tcp_congestion_control);
165		− if (unlikely(!try_module_get(ca->owner)))
	165	+ if (unlikely(!bpf_try_module_get(ca, ca->owner)))
166	166	ca = &tcp_reno;
167	167	icsk->icsk_ca_ops = ca;
168	168	rcu_read_unlock();
@@ -208,7 +208,7 @@ void tcp_cleanup_congestion_control(struct sock *sk) · 39 unmodified lines
208	208
209	209	if (icsk->icsk_ca_ops->release)
210	210	icsk->icsk_ca_ops->release(sk);
211		− module_put(icsk->icsk_ca_ops->owner);
	211	+ bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
212	212	}
213	213
214	214	/* Used by sysctl to change default congestion control */
@@ -222,12 +222,12 @@ int tcp_set_default_congestion_control(struct net net, const char name) · 7 unmodified lines
222	222	ca = tcp_ca_find_autoload(net, name);
223	223	if (!ca) {
224	224	ret = -ENOENT;
225		− } else if (!try_module_get(ca->owner)) {
	225	+ } else if (!bpf_try_module_get(ca, ca->owner)) {
226	226	ret = -EBUSY;
227	227	} else {
228	228	prev = xchg(&net->ipv4.tcp_congestion_control, ca);
229	229	if (prev)
230		− module_put(prev->owner);
	230	+ bpf_module_put(prev, prev->owner);
231	231
232	232	ca->flags \|= TCP_CONG_NON_RESTRICTED;
233	233	ret = 0;
@@ -366,19 +366,19 @@ int tcp_set_congestion_control(struct sock sk, const char name, bool load, · 132 unmodified lines
366	366	} else if (!load) {
367	367	const struct tcp_congestion_ops *old_ca = icsk->icsk_ca_ops;
368	368
369		− if (try_module_get(ca->owner)) {
	369	+ if (bpf_try_module_get(ca, ca->owner)) {
370	370	if (reinit) {
371	371	tcp_reinit_congestion_control(sk, ca);
372	372	} else {
373	373	icsk->icsk_ca_ops = ca;
374		− module_put(old_ca->owner);
	374	+ bpf_module_put(old_ca, old_ca->owner);
375	375	}
376	376	} else {
377	377	err = -EBUSY;
378	378	}
379	379	} else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) \|\| cap_net_admin)) {
380	380	err = -EPERM;
381		− } else if (!try_module_get(ca->owner)) {
	381	+ } else if (!bpf_try_module_get(ca, ca->owner)) {
382	382	err = -EBUSY;
383	383	} else {
384	384	tcp_reinit_congestion_control(sk, ca);

net/ipv4/tcp_ipv4.c+4−2

@@ -2678,7 +2678,8 @@ static void __net_exit tcp_sk_exit(struct net *net) · 2677 unmodified lines
2678	2678	int cpu;
2679	2679
2680	2680	if (net->ipv4.tcp_congestion_control)
2681		− module_put(net->ipv4.tcp_congestion_control->owner);
	2681	+ bpf_module_put(net->ipv4.tcp_congestion_control,
	2682	+ net->ipv4.tcp_congestion_control->owner);
2682	2683
2683	2684	for_each_possible_cpu(cpu)
2684	2685	inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
@@ -2785,7 +2786,8 @@ static int __net_init tcp_sk_init(struct net *net) · 100 unmodified lines
2785	2786
2786	2787	/* Reno is always built in */
2787	2788	if (!net_eq(net, &init_net) &&
2788		− try_module_get(init_net.ipv4.tcp_congestion_control->owner))
	2789	+ bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
	2790	+ init_net.ipv4.tcp_congestion_control->owner))
2789	2791	net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
2790	2792	else
2791	2793	net->ipv4.tcp_congestion_control = &tcp_reno;

net/ipv4/tcp_minisocks.c+2−2

@@ -414,7 +414,7 @@ void tcp_ca_openreq_child(struct sock sk, const struct dst_entry dst) · 413 unmodified lines
414	414
415	415	rcu_read_lock();
416	416	ca = tcp_ca_find_key(ca_key);
417		− if (likely(ca && try_module_get(ca->owner))) {
	417	+ if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
418	418	icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
419	419	icsk->icsk_ca_ops = ca;
420	420	ca_got_dst = true;
@@ -425,7 +425,7 @@ void tcp_ca_openreq_child(struct sock sk, const struct dst_entry dst) · 4 unmodified lines
425	425	/* If no valid choice made yet, assign current system default ca. */
426	426	if (!ca_got_dst &&
427	427	(!icsk->icsk_ca_setsockopt \|\|
428		− !try_module_get(icsk->icsk_ca_ops->owner)))
	428	+ !bpf_try_module_get(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner)))
429	429	tcp_assign_congestion_control(sk);
430	430
431	431	tcp_set_ca_state(sk, TCP_CA_Open);

net/ipv4/tcp_output.c+2−2

@@ -3368,8 +3368,8 @@ static void tcp_ca_dst_init(struct sock sk, const struct dst_entry dst) · 3367 unmodified lines
3368	3368
3369	3369	rcu_read_lock();
3370	3370	ca = tcp_ca_find_key(ca_key);
3371		− if (likely(ca && try_module_get(ca->owner))) {
3372		− module_put(icsk->icsk_ca_ops->owner);
	3371	+ if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
	3372	+ bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
3373	3373	icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
3374	3374	icsk->icsk_ca_ops = ca;
3375	3375	}

tools/include/uapi/linux/bpf.h+17−2

@@ -136,6 +136,7 @@ enum bpf_map_type { · 135 unmodified lines
136	136	BPF_MAP_TYPE_STACK,
137	137	BPF_MAP_TYPE_SK_STORAGE,
138	138	BPF_MAP_TYPE_DEVMAP_HASH,
	139	+ BPF_MAP_TYPE_STRUCT_OPS,
139	140	};
140	141
141	142	/* Note that tracing related programs such as
@@ -174,6 +175,7 @@ enum bpf_prog_type { · 32 unmodified lines
174	175	BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE,
175	176	BPF_PROG_TYPE_CGROUP_SOCKOPT,
176	177	BPF_PROG_TYPE_TRACING,
	178	+ BPF_PROG_TYPE_STRUCT_OPS,
177	179	};
178	180
179	181	enum bpf_attach_type {
@@ -397,6 +399,10 @@ union bpf_attr { · 217 unmodified lines
397	399	__u32 btf_fd; /* fd pointing to a BTF type data */
398	400	__u32 btf_key_type_id; /* BTF type_id of the key */
399	401	__u32 btf_value_type_id; /* BTF type_id of the value */
	402	+ __u32 btf_vmlinux_value_type_id;/* BTF type_id of a kernel-
	403	+ * struct stored as the
	404	+ * map value
	405	+ */
400	406	};
401	407
402	408	struct { /* anonymous struct used by BPF_MAP__ELEM commands /
@@ -2831,6 +2837,14 @@ union bpf_attr { · 2428 unmodified lines
2831	2837	* Return
2832	2838	* On success, the strictly positive length of the string, including
2833	2839	* the trailing NUL character. On error, a negative value.
	2840	+ *
	2841	+ * int bpf_tcp_send_ack(void *tp, u32 rcv_nxt)
	2842	+ * Description
	2843	+ * Send out a tcp-ack. tp is the in-kernel struct tcp_sock.
	2844	+ * rcv_nxt is the ack_seq to be sent out.
	2845	+ * Return
	2846	+ * 0 on success, or a negative error in case of failure.
	2847	+ *
2834	2848	*/
2835	2849	#define __BPF_FUNC_MAPPER(FN) \
2836	2850	FN(unspec), \
@@ -2948,7 +2962,8 @@ union bpf_attr { · 111 unmodified lines
2948	2962	FN(probe_read_user), \
2949	2963	FN(probe_read_kernel), \
2950	2964	FN(probe_read_user_str), \
2951		− FN(probe_read_kernel_str),
	2965	+ FN(probe_read_kernel_str), \
	2966	+ FN(tcp_send_ack),
2952	2967
2953	2968	/* integer value in 'imm' field of BPF_CALL instruction selects which helper
2954	2969	* function eBPF program intends to call
@@ -3349,7 +3364,7 @@ struct bpf_map_info { · 394 unmodified lines
3349	3364	__u32 map_flags;
3350	3365	char name[BPF_OBJ_NAME_LEN];
3351	3366	__u32 ifindex;
3352		− __u32 :32;
	3367	+ __u32 btf_vmlinux_value_type_id;
3353	3368	__u64 netns_dev;
3354	3369	__u64 netns_ino;
3355	3370	__u32 btf_id;

tools/lib/bpf/bpf.c+8−2

@@ -95,7 +95,11 @@ int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr) · 94 unmodified lines
95	95	attr.btf_key_type_id = create_attr->btf_key_type_id;
96	96	attr.btf_value_type_id = create_attr->btf_value_type_id;
97	97	attr.map_ifindex = create_attr->map_ifindex;
98		− attr.inner_map_fd = create_attr->inner_map_fd;
	98	+ if (attr.map_type == BPF_MAP_TYPE_STRUCT_OPS)
	99	+ attr.btf_vmlinux_value_type_id =
	100	+ create_attr->btf_vmlinux_value_type_id;
	101	+ else
	102	+ attr.inner_map_fd = create_attr->inner_map_fd;
99	103
100	104	return sys_bpf(BPF_MAP_CREATE, &attr, sizeof(attr));
101	105	}
@@ -228,7 +232,9 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, · 126 unmodified lines
228	232	memset(&attr, 0, sizeof(attr));
229	233	attr.prog_type = load_attr->prog_type;
230	234	attr.expected_attach_type = load_attr->expected_attach_type;
231		− if (attr.prog_type == BPF_PROG_TYPE_TRACING) {
	235	+ if (attr.prog_type == BPF_PROG_TYPE_STRUCT_OPS) {
	236	+ attr.attach_btf_id = load_attr->attach_btf_id;
	237	+ } else if (attr.prog_type == BPF_PROG_TYPE_TRACING) {
232	238	attr.attach_btf_id = load_attr->attach_btf_id;
233	239	attr.attach_prog_fd = load_attr->attach_prog_fd;
234	240	} else {

tools/lib/bpf/bpf.h+4−1

@@ -46,7 +46,10 @@ struct bpf_create_map_attr { · 45 unmodified lines
46	46	__u32 btf_key_type_id;
47	47	__u32 btf_value_type_id;
48	48	__u32 map_ifindex;
49		− __u32 inner_map_fd;
	49	+ union {
	50	+ __u32 inner_map_fd;
	51	+ __u32 btf_vmlinux_value_type_id;
	52	+ };
50	53	};
51	54
52	55	LIBBPF_API int

tools/lib/bpf/libbpf.c+640−9

@@ -70,6 +70,13 @@ · 69 unmodified lines
70	70
71	71	#define __printf(a, b) __attribute__((format(printf, a, b)))
72	72
	73	+static struct btf *bpf_find_kernel_btf(void);
	74	+static struct bpf_map bpf_object__add_map(struct bpf_object obj);
	75	+static struct bpf_program bpf_object__find_prog_by_idx(struct bpf_object obj,
	76	+ int idx);
	77	+static const struct btf_type *
	78	+skip_mods_and_typedefs(const struct btf btf, __u32 id, __u32 res_id);
	79	+
73	80	static int __base_pr(enum libbpf_print_level level, const char *format,
74	81	va_list args)
75	82	{
@@ -229,10 +236,32 @@ struct bpf_program { · 153 unmodified lines
229	236	__u32 prog_flags;
230	237	};
231	238
	239	+struct bpf_struct_ops {
	240	+ const char *tname;
	241	+ const struct btf_type *type;
	242	+ struct bpf_program **progs;
	243	+ __u32 *kern_func_off;
	244	+ /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
	245	+ void *data;
	246	+ /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
	247	+ * btf_vmlinux's format.
	248	+ * struct bpf_struct_ops_tcp_congestion_ops {
	249	+ * [... some other kernel fields ...]
	250	+ * struct tcp_congestion_ops data;
	251	+ * }
	252	+ * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
	253	+ * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
	254	+ * from "data".
	255	+ */
	256	+ void *kern_vdata;
	257	+ __u32 type_id;
	258	+};
	259	+
232	260	#define DATA_SEC ".data"
233	261	#define BSS_SEC ".bss"
234	262	#define RODATA_SEC ".rodata"
235	263	#define KCONFIG_SEC ".kconfig"
	264	+#define STRUCT_OPS_SEC ".struct_ops"
236	265
237	266	enum libbpf_map_type {
238	267	LIBBPF_MAP_UNSPEC,
@@ -259,10 +288,12 @@ struct bpf_map { · 20 unmodified lines
259	288	struct bpf_map_def def;
260	289	__u32 btf_key_type_id;
261	290	__u32 btf_value_type_id;
	291	+ __u32 btf_vmlinux_value_type_id;
262	292	void *priv;
263	293	bpf_map_clear_priv_t clear_priv;
264	294	enum libbpf_map_type libbpf_type;
265	295	void *mmaped;
	296	+ struct bpf_struct_ops *st_ops;
266	297	char *pin_path;
267	298	bool pinned;
268	299	bool reused;
@@ -326,6 +357,7 @@ struct bpf_object { · 57 unmodified lines
326	357	Elf_Data *data;
327	358	Elf_Data *rodata;
328	359	Elf_Data *bss;
	360	+ Elf_Data *st_ops_data;
329	361	size_t strtabidx;
330	362	struct {
331	363	GElf_Shdr shdr;
@@ -339,6 +371,7 @@ struct bpf_object { · 7 unmodified lines
339	371	int data_shndx;
340	372	int rodata_shndx;
341	373	int bss_shndx;
	374	+ int st_ops_shndx;
342	375	} efile;
343	376	/*
344	377	* All loaded bpf_object is linked in a list, which is
@@ -566,6 +599,359 @@ static __u32 get_kernel_version(void) · 221 unmodified lines
566	599	return KERNEL_VERSION(major, minor, patch);
567	600	}
568	601
	602	+static const struct btf_member *
	603	+find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
	604	+{
	605	+ struct btf_member *m;
	606	+ int i;
	607	+
	608	+ for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
	609	+ if (btf_member_bit_offset(t, i) == bit_offset)
	610	+ return m;
	611	+ }
	612	+
	613	+ return NULL;
	614	+}
	615	+
	616	+static const struct btf_member *
	617	+find_member_by_name(const struct btf btf, const struct btf_type t,
	618	+ const char *name)
	619	+{
	620	+ struct btf_member *m;
	621	+ int i;
	622	+
	623	+ for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
	624	+ if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
	625	+ return m;
	626	+ }
	627	+
	628	+ return NULL;
	629	+}
	630	+
	631	+#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
	632	+#define STRUCT_OPS_VALUE_PREFIX_LEN (sizeof(STRUCT_OPS_VALUE_PREFIX) - 1)
	633	+
	634	+static int
	635	+find_struct_ops_kern_types(const struct btf btf, const char tname,
	636	+ const struct btf_type *type, __u32 type_id,
	637	+ const struct btf_type *vtype, __u32 vtype_id,
	638	+ const struct btf_member **data_member)
	639	+{
	640	+ const struct btf_type kern_type, kern_vtype;
	641	+ const struct btf_member *kern_data_member;
	642	+ __s32 kern_vtype_id, kern_type_id;
	643	+ char vtname[128] = STRUCT_OPS_VALUE_PREFIX;
	644	+ __u32 i;
	645	+
	646	+ kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
	647	+ if (kern_type_id < 0) {
	648	+ pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
	649	+ tname);
	650	+ return kern_type_id;
	651	+ }
	652	+ kern_type = btf__type_by_id(btf, kern_type_id);
	653	+
	654	+ /* Find the corresponding "map_value" type that will be used
	655	+ * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example,
	656	+ * find "struct bpf_struct_ops_tcp_congestion_ops" from the
	657	+ * btf_vmlinux.
	658	+ */
	659	+ strncat(vtname + STRUCT_OPS_VALUE_PREFIX_LEN, tname,
	660	+ sizeof(vtname) - STRUCT_OPS_VALUE_PREFIX_LEN - 1);
	661	+ kern_vtype_id = btf__find_by_name_kind(btf, vtname,
	662	+ BTF_KIND_STRUCT);
	663	+ if (kern_vtype_id < 0) {
	664	+ pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
	665	+ vtname);
	666	+ return kern_vtype_id;
	667	+ }
	668	+ kern_vtype = btf__type_by_id(btf, kern_vtype_id);
	669	+
	670	+ /* Find "struct tcp_congestion_ops" from
	671	+ * struct bpf_struct_ops_tcp_congestion_ops {
	672	+ * [ ... ]
	673	+ * struct tcp_congestion_ops data;
	674	+ * }
	675	+ */
	676	+ kern_data_member = btf_members(kern_vtype);
	677	+ for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
	678	+ if (kern_data_member->type == kern_type_id)
	679	+ break;
	680	+ }
	681	+ if (i == btf_vlen(kern_vtype)) {
	682	+ pr_warn("struct_ops init_kern: struct %s data is not found in struct %s\n",
	683	+ tname, vtname);
	684	+ return -EINVAL;
	685	+ }
	686	+
	687	+ *type = kern_type;
	688	+ *type_id = kern_type_id;
	689	+ *vtype = kern_vtype;
	690	+ *vtype_id = kern_vtype_id;
	691	+ *data_member = kern_data_member;
	692	+
	693	+ return 0;
	694	+}
	695	+
	696	+static bool bpf_map__is_struct_ops(const struct bpf_map *map)
	697	+{
	698	+ return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
	699	+}
	700	+
	701	+/* Init the map's fields that depend on kern_btf */
	702	+static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
	703	+ const struct btf *btf,
	704	+ const struct btf *kern_btf)
	705	+{
	706	+ const struct btf_member member, kern_member, *kern_data_member;
	707	+ const struct btf_type type, kern_type, *kern_vtype;
	708	+ __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
	709	+ struct bpf_struct_ops *st_ops;
	710	+ void data, kern_data;
	711	+ const char *tname;
	712	+ int err;
	713	+
	714	+ st_ops = map->st_ops;
	715	+ type = st_ops->type;
	716	+ tname = st_ops->tname;
	717	+ err = find_struct_ops_kern_types(kern_btf, tname,
	718	+ &kern_type, &kern_type_id,
	719	+ &kern_vtype, &kern_vtype_id,
	720	+ &kern_data_member);
	721	+ if (err)
	722	+ return err;
	723	+
	724	+ pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
	725	+ map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
	726	+
	727	+ map->def.value_size = kern_vtype->size;
	728	+ map->btf_vmlinux_value_type_id = kern_vtype_id;
	729	+
	730	+ st_ops->kern_vdata = calloc(1, kern_vtype->size);
	731	+ if (!st_ops->kern_vdata)
	732	+ return -ENOMEM;
	733	+
	734	+ data = st_ops->data;
	735	+ kern_data_off = kern_data_member->offset / 8;
	736	+ kern_data = st_ops->kern_vdata + kern_data_off;
	737	+
	738	+ member = btf_members(type);
	739	+ for (i = 0; i < btf_vlen(type); i++, member++) {
	740	+ const struct btf_type mtype, kern_mtype;
	741	+ __u32 mtype_id, kern_mtype_id;
	742	+ void mdata, kern_mdata;
	743	+ __s64 msize, kern_msize;
	744	+ __u32 moff, kern_moff;
	745	+ __u32 kern_member_idx;
	746	+ const char *mname;
	747	+
	748	+ mname = btf__name_by_offset(btf, member->name_off);
	749	+ kern_member = find_member_by_name(kern_btf, kern_type, mname);
	750	+ if (!kern_member) {
	751	+ pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
	752	+ map->name, mname);
	753	+ return -ENOTSUP;
	754	+ }
	755	+
	756	+ kern_member_idx = kern_member - btf_members(kern_type);
	757	+ if (btf_member_bitfield_size(type, i) \|\|
	758	+ btf_member_bitfield_size(kern_type, kern_member_idx)) {
	759	+ pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
	760	+ map->name, mname);
	761	+ return -ENOTSUP;
	762	+ }
	763	+
	764	+ moff = member->offset / 8;
	765	+ kern_moff = kern_member->offset / 8;
	766	+
	767	+ mdata = data + moff;
	768	+ kern_mdata = kern_data + kern_moff;
	769	+
	770	+ mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
	771	+ kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
	772	+ &kern_mtype_id);
	773	+ if (BTF_INFO_KIND(mtype->info) !=
	774	+ BTF_INFO_KIND(kern_mtype->info)) {
	775	+ pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
	776	+ map->name, mname, BTF_INFO_KIND(mtype->info),
	777	+ BTF_INFO_KIND(kern_mtype->info));
	778	+ return -ENOTSUP;
	779	+ }
	780	+
	781	+ if (btf_is_ptr(mtype)) {
	782	+ struct bpf_program *prog;
	783	+
	784	+ mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id);
	785	+ kern_mtype = skip_mods_and_typedefs(kern_btf,
	786	+ kern_mtype->type,
	787	+ &kern_mtype_id);
	788	+ if (!btf_is_func_proto(mtype) \|\|
	789	+ !btf_is_func_proto(kern_mtype)) {
	790	+ pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n",
	791	+ map->name, mname);
	792	+ return -ENOTSUP;
	793	+ }
	794	+
	795	+ prog = st_ops->progs[i];
	796	+ if (!prog) {
	797	+ pr_debug("struct_ops init_kern %s: func ptr %s is not set\n",
	798	+ map->name, mname);
	799	+ continue;
	800	+ }
	801	+
	802	+ prog->attach_btf_id = kern_type_id;
	803	+ prog->expected_attach_type = kern_member_idx;
	804	+
	805	+ st_ops->kern_func_off[i] = kern_data_off + kern_moff;
	806	+
	807	+ pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
	808	+ map->name, mname, prog->name, moff,
	809	+ kern_moff);
	810	+
	811	+ continue;
	812	+ }
	813	+
	814	+ msize = btf__resolve_size(btf, mtype_id);
	815	+ kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
	816	+ if (msize < 0 \|\| kern_msize < 0 \|\| msize != kern_msize) {
	817	+ pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
	818	+ map->name, mname, (ssize_t)msize,
	819	+ (ssize_t)kern_msize);
	820	+ return -ENOTSUP;
	821	+ }
	822	+
	823	+ pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
	824	+ map->name, mname, (unsigned int)msize,
	825	+ moff, kern_moff);
	826	+ memcpy(kern_mdata, mdata, msize);
	827	+ }
	828	+
	829	+ return 0;
	830	+}
	831	+
	832	+static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
	833	+{
	834	+ struct btf *kern_btf = NULL;
	835	+ struct bpf_map *map;
	836	+ size_t i;
	837	+ int err;
	838	+
	839	+ for (i = 0; i < obj->nr_maps; i++) {
	840	+ map = &obj->maps[i];
	841	+
	842	+ if (!bpf_map__is_struct_ops(map))
	843	+ continue;
	844	+
	845	+ if (!kern_btf) {
	846	+ kern_btf = bpf_find_kernel_btf();
	847	+ if (IS_ERR(kern_btf))
	848	+ return PTR_ERR(kern_btf);
	849	+ }
	850	+
	851	+ err = bpf_map__init_kern_struct_ops(map, obj->btf, kern_btf);
	852	+ if (err) {
	853	+ btf__free(kern_btf);
	854	+ return err;
	855	+ }
	856	+ }
	857	+
	858	+ btf__free(kern_btf);
	859	+ return 0;
	860	+}
	861	+
	862	+static int bpf_object__init_struct_ops_maps(struct bpf_object *obj)
	863	+{
	864	+ const struct btf_type type, datasec;
	865	+ const struct btf_var_secinfo *vsi;
	866	+ struct bpf_struct_ops *st_ops;
	867	+ const char tname, var_name;
	868	+ __s32 type_id, datasec_id;
	869	+ const struct btf *btf;
	870	+ struct bpf_map *map;
	871	+ __u32 i;
	872	+
	873	+ if (obj->efile.st_ops_shndx == -1)
	874	+ return 0;
	875	+
	876	+ btf = obj->btf;
	877	+ datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC,
	878	+ BTF_KIND_DATASEC);
	879	+ if (datasec_id < 0) {
	880	+ pr_warn("struct_ops init: DATASEC %s not found\n",
	881	+ STRUCT_OPS_SEC);
	882	+ return -EINVAL;
	883	+ }
	884	+
	885	+ datasec = btf__type_by_id(btf, datasec_id);
	886	+ vsi = btf_var_secinfos(datasec);
	887	+ for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
	888	+ type = btf__type_by_id(obj->btf, vsi->type);
	889	+ var_name = btf__name_by_offset(obj->btf, type->name_off);
	890	+
	891	+ type_id = btf__resolve_type(obj->btf, vsi->type);
	892	+ if (type_id < 0) {
	893	+ pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
	894	+ vsi->type, STRUCT_OPS_SEC);
	895	+ return -EINVAL;
	896	+ }
	897	+
	898	+ type = btf__type_by_id(obj->btf, type_id);
	899	+ tname = btf__name_by_offset(obj->btf, type->name_off);
	900	+ if (!tname[0]) {
	901	+ pr_warn("struct_ops init: anonymous type is not supported\n");
	902	+ return -ENOTSUP;
	903	+ }
	904	+ if (!btf_is_struct(type)) {
	905	+ pr_warn("struct_ops init: %s is not a struct\n", tname);
	906	+ return -EINVAL;
	907	+ }
	908	+
	909	+ map = bpf_object__add_map(obj);
	910	+ if (IS_ERR(map))
	911	+ return PTR_ERR(map);
	912	+
	913	+ map->sec_idx = obj->efile.st_ops_shndx;
	914	+ map->sec_offset = vsi->offset;
	915	+ map->name = strdup(var_name);
	916	+ if (!map->name)
	917	+ return -ENOMEM;
	918	+
	919	+ map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
	920	+ map->def.key_size = sizeof(int);
	921	+ map->def.value_size = type->size;
	922	+ map->def.max_entries = 1;
	923	+
	924	+ map->st_ops = calloc(1, sizeof(*map->st_ops));
	925	+ if (!map->st_ops)
	926	+ return -ENOMEM;
	927	+ st_ops = map->st_ops;
	928	+ st_ops->data = malloc(type->size);
	929	+ st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
	930	+ st_ops->kern_func_off = malloc(btf_vlen(type) *
	931	+ sizeof(*st_ops->kern_func_off));
	932	+ if (!st_ops->data \|\| !st_ops->progs \|\| !st_ops->kern_func_off)
	933	+ return -ENOMEM;
	934	+
	935	+ if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) {
	936	+ pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
	937	+ var_name, STRUCT_OPS_SEC);
	938	+ return -EINVAL;
	939	+ }
	940	+
	941	+ memcpy(st_ops->data,
	942	+ obj->efile.st_ops_data->d_buf + vsi->offset,
	943	+ type->size);
	944	+ st_ops->tname = tname;
	945	+ st_ops->type = type;
	946	+ st_ops->type_id = type_id;
	947	+
	948	+ pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
	949	+ tname, type_id, var_name, vsi->offset);
	950	+ }
	951	+
	952	+ return 0;
	953	+}
	954	+
569	955	static struct bpf_object bpf_object__new(const char path,
570	956	const void *obj_buf,
571	957	size_t obj_buf_sz,
@@ -607,6 +993,7 @@ static struct bpf_object bpf_object__new(const char path, · 35 unmodified lines
607	993	obj->efile.data_shndx = -1;
608	994	obj->efile.rodata_shndx = -1;
609	995	obj->efile.bss_shndx = -1;
	996	+ obj->efile.st_ops_shndx = -1;
610	997	obj->kconfig_map_idx = -1;
611	998
612	999	obj->kern_version = get_kernel_version();
@@ -630,6 +1017,7 @@ static void bpf_object__elf_finish(struct bpf_object *obj) · 17 unmodified lines
630	1017	obj->efile.data = NULL;
631	1018	obj->efile.rodata = NULL;
632	1019	obj->efile.bss = NULL;
	1020	+ obj->efile.st_ops_data = NULL;
633	1021
634	1022	zfree(&obj->efile.reloc_sects);
635	1023	obj->efile.nr_reloc_sects = 0;
@@ -815,6 +1203,9 @@ int bpf_object__section_size(const struct bpf_object obj, const char name, · 179 unmodified lines
815	1203	} else if (!strcmp(name, RODATA_SEC)) {
816	1204	if (obj->efile.rodata)
817	1205	*size = obj->efile.rodata->d_size;
	1206	+ } else if (!strcmp(name, STRUCT_OPS_SEC)) {
	1207	+ if (obj->efile.st_ops_data)
	1208	+ *size = obj->efile.st_ops_data->d_size;
818	1209	} else {
819	1210	ret = bpf_object_search_section_size(obj, name, &d_size);
820	1211	if (!ret)
@@ -1440,6 +1831,20 @@ skip_mods_and_typedefs(const struct btf btf, __u32 id, __u32 res_id) · 619 unmodified lines
1440	1831	return t;
1441	1832	}
1442	1833
	1834	+static const struct btf_type *
	1835	+resolve_func_ptr(const struct btf btf, __u32 id, __u32 res_id)
	1836	+{
	1837	+ const struct btf_type *t;
	1838	+
	1839	+ t = skip_mods_and_typedefs(btf, id, NULL);
	1840	+ if (!btf_is_ptr(t))
	1841	+ return NULL;
	1842	+
	1843	+ t = skip_mods_and_typedefs(btf, t->type, res_id);
	1844	+
	1845	+ return btf_is_func_proto(t) ? t : NULL;
	1846	+}
	1847	+
1443	1848	/*
1444	1849	* Fetch integer attribute of BTF map definition. Such attributes are
1445	1850	* represented using a pointer to an array, in which dimensionality of array
@@ -1787,6 +2192,7 @@ static int bpf_object__init_maps(struct bpf_object *obj, · 341 unmodified lines
1787	2192	err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
1788	2193	err = err ?: bpf_object__init_global_data_maps(obj);
1789	2194	err = err ?: bpf_object__init_kconfig_map(obj);
	2195	+ err = err ?: bpf_object__init_struct_ops_maps(obj);
1790	2196	if (err)
1791	2197	return err;
1792	2198
@@ -1889,7 +2295,8 @@ static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) · 96 unmodified lines
1889	2295	static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj)
1890	2296	{
1891	2297	return obj->efile.btf_maps_shndx >= 0 \|\|
1892		− obj->nr_extern > 0;
	2298	+ obj->efile.st_ops_shndx >= 0 \|\|
	2299	+ obj->nr_extern > 0;
1893	2300	}
1894	2301
1895	2302	static int bpf_object__init_btf(struct bpf_object *obj,
@@ -2088,6 +2495,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj) · 192 unmodified lines
2088	2495	} else if (strcmp(name, RODATA_SEC) == 0) {
2089	2496	obj->efile.rodata = data;
2090	2497	obj->efile.rodata_shndx = idx;
	2498	+ } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
	2499	+ obj->efile.st_ops_data = data;
	2500	+ obj->efile.st_ops_shndx = idx;
2091	2501	} else {
2092	2502	pr_debug("skip section(%d) %s\n", idx, name);
2093	2503	}
@@ -2097,7 +2507,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj) · 3 unmodified lines
2097	2507	int sec = sh.sh_info; /* points to other section */
2098	2508
2099	2509	/* Only do relo for section with exec instructions */
2100		− if (!section_have_execinstr(obj, sec)) {
	2510	+ if (!section_have_execinstr(obj, sec) &&
	2511	+ strcmp(name, ".rel" STRUCT_OPS_SEC)) {
2101	2512	pr_debug("skip relo %s(%d) for section(%d)\n",
2102	2513	name, idx, sec);
2103	2514	continue;
@@ -2599,8 +3010,12 @@ static int bpf_map_find_btf_info(struct bpf_object obj, struct bpf_map map) · 495 unmodified lines
2599	3010	__u32 key_type_id = 0, value_type_id = 0;
2600	3011	int ret;
2601	3012
2602		− /* if it's BTF-defined map, we don't need to search for type IDs */
2603		− if (map->sec_idx == obj->efile.btf_maps_shndx)
	3013	+ /* if it's BTF-defined map, we don't need to search for type IDs.
	3014	+ * For struct_ops map, it does not need btf_key_type_id and
	3015	+ * btf_value_type_id.
	3016	+ */
	3017	+ if (map->sec_idx == obj->efile.btf_maps_shndx \|\|
	3018	+ bpf_map__is_struct_ops(map))
2604	3019	return 0;
2605	3020
2606	3021	if (!bpf_map__is_internal(map)) {
@@ -3025,6 +3440,9 @@ bpf_object__create_maps(struct bpf_object *obj) · 418 unmodified lines
3025	3440	if (bpf_map_type__is_map_in_map(def->type) &&
3026	3441	map->inner_map_fd >= 0)
3027	3442	create_attr.inner_map_fd = map->inner_map_fd;
	3443	+ if (bpf_map__is_struct_ops(map))
	3444	+ create_attr.btf_vmlinux_value_type_id =
	3445	+ map->btf_vmlinux_value_type_id;
3028	3446
3029	3447	if (obj->btf && !bpf_map_find_btf_info(obj, map)) {
3030	3448	create_attr.btf_fd = btf__fd(obj->btf);
@@ -3899,7 +4317,7 @@ cleanup: · 868 unmodified lines
3899	4317	* Probe few well-known locations for vmlinux kernel image and try to load BTF
3900	4318	* data out of it to use for target BTF.
3901	4319	*/
3902		−static struct btf *bpf_core_find_kernel_btf(void)
	4320	+static struct btf *bpf_find_kernel_btf(void)
3903	4321	{
3904	4322	struct {
3905	4323	const char *path_fmt;
@@ -4180,7 +4598,7 @@ bpf_core_reloc_fields(struct bpf_object obj, const char targ_btf_path) · 274 unmodified lines
4180	4598	if (targ_btf_path)
4181	4599	targ_btf = btf__parse_elf(targ_btf_path, NULL);
4182	4600	else
4183		− targ_btf = bpf_core_find_kernel_btf();
	4601	+ targ_btf = bpf_find_kernel_btf();
4184	4602	if (IS_ERR(targ_btf)) {
4185	4603	pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf));
4186	4604	return PTR_ERR(targ_btf);
@@ -4379,6 +4797,10 @@ bpf_object__relocate(struct bpf_object obj, const char targ_btf_path) · 192 unmodified lines
4379	4797	return 0;
4380	4798	}
4381	4799
	4800	+static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
	4801	+ GElf_Shdr *shdr,
	4802	+ Elf_Data *data);
	4803	+
4382	4804	static int bpf_object__collect_reloc(struct bpf_object *obj)
4383	4805	{
4384	4806	int i, err;
@@ -4399,6 +4821,15 @@ static int bpf_object__collect_reloc(struct bpf_object *obj) · 14 unmodified lines
4399	4821	return -LIBBPF_ERRNO__INTERNAL;
4400	4822	}
4401	4823
	4824	+ if (idx == obj->efile.st_ops_shndx) {
	4825	+ err = bpf_object__collect_struct_ops_map_reloc(obj,
	4826	+ shdr,
	4827	+ data);
	4828	+ if (err)
	4829	+ return err;
	4830	+ continue;
	4831	+ }
	4832	+
4402	4833	prog = bpf_object__find_prog_by_idx(obj, idx);
4403	4834	if (!prog) {
4404	4835	pr_warn("relocation failed: no section(%d)\n", idx);
@@ -4433,7 +4864,9 @@ load_program(struct bpf_program prog, struct bpf_insn insns, int insns_cnt, · 28 unmodified lines
4433	4864	load_attr.insns = insns;
4434	4865	load_attr.insns_cnt = insns_cnt;
4435	4866	load_attr.license = license;
4436		− if (prog->type == BPF_PROG_TYPE_TRACING) {
	4867	+ if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
	4868	+ load_attr.attach_btf_id = prog->attach_btf_id;
	4869	+ } else if (prog->type == BPF_PROG_TYPE_TRACING) {
4437	4870	load_attr.attach_prog_fd = prog->attach_prog_fd;
4438	4871	load_attr.attach_btf_id = prog->attach_btf_id;
4439	4872	} else {
@@ -4679,6 +5112,9 @@ __bpf_object__open(const char path, const void obj_buf, size_t obj_buf_sz, · 239 unmodified lines
4679	5112	enum bpf_prog_type prog_type;
4680	5113	enum bpf_attach_type attach_type;
4681	5114
	5115	+ if (prog->type != BPF_PROG_TYPE_UNSPEC)
	5116	+ continue;
	5117	+
4682	5118	err = libbpf_prog_type_by_name(prog->section_name, &prog_type,
4683	5119	&attach_type);
4684	5120	if (err == -ESRCH)
@@ -4774,8 +5210,11 @@ int bpf_object__unload(struct bpf_object *obj) · 89 unmodified lines
4774	5210	if (!obj)
4775	5211	return -EINVAL;
4776	5212
4777		− for (i = 0; i < obj->nr_maps; i++)
	5213	+ for (i = 0; i < obj->nr_maps; i++) {
4778	5214	zclose(obj->maps[i].fd);
	5215	+ if (obj->maps[i].st_ops)
	5216	+ zfree(&obj->maps[i].st_ops->kern_vdata);
	5217	+ }
4779	5218
4780	5219	for (i = 0; i < obj->nr_programs; i++)
4781	5220	bpf_program__unload(&obj->programs[i]);
@@ -4891,6 +5330,7 @@ int bpf_object__load_xattr(struct bpf_object_load_attr *attr) · 109 unmodified lines
4891	5330	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
4892	5331	err = err ? : bpf_object__sanitize_and_load_btf(obj);
4893	5332	err = err ? : bpf_object__sanitize_maps(obj);
	5333	+ err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
4894	5334	err = err ? : bpf_object__create_maps(obj);
4895	5335	err = err ? : bpf_object__relocate(obj, attr->target_btf_path);
4896	5336	err = err ? : bpf_object__load_progs(obj, attr->log_level);
@@ -5478,6 +5918,13 @@ void bpf_object__close(struct bpf_object *obj) · 581 unmodified lines
5478	5918	map->mmaped = NULL;
5479	5919	}
5480	5920
	5921	+ if (map->st_ops) {
	5922	+ zfree(&map->st_ops->data);
	5923	+ zfree(&map->st_ops->progs);
	5924	+ zfree(&map->st_ops->kern_func_off);
	5925	+ zfree(&map->st_ops);
	5926	+ }
	5927	+
5481	5928	zfree(&map->name);
5482	5929	zfree(&map->pin_path);
5483	5930	}
@@ -5746,6 +6193,7 @@ BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); · 262 unmodified lines
5746	6193	BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
5747	6194	BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
5748	6195	BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING);
	6196	+BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS);
5749	6197
5750	6198	enum bpf_attach_type
5751	6199	bpf_program__get_expected_attach_type(struct bpf_program *prog)
@@ -5899,6 +6347,7 @@ static const struct bpf_sec_def section_defs[] = { · 147 unmodified lines
5899	6347	BPF_CGROUP_GETSOCKOPT),
5900	6348	BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT,
5901	6349	BPF_CGROUP_SETSOCKOPT),
	6350	+ BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS),
5902	6351	};
5903	6352
5904	6353	#undef BPF_PROG_SEC_IMPL
@@ -5975,11 +6424,141 @@ int libbpf_prog_type_by_name(const char name, enum bpf_prog_type prog_type, · 70 unmodified lines
5975	6424	return -ESRCH;
5976	6425	}
5977	6426
	6427	+static struct bpf_map find_struct_ops_map_by_offset(struct bpf_object obj,
	6428	+ size_t offset)
	6429	+{
	6430	+ struct bpf_map *map;
	6431	+ size_t i;
	6432	+
	6433	+ for (i = 0; i < obj->nr_maps; i++) {
	6434	+ map = &obj->maps[i];
	6435	+ if (!bpf_map__is_struct_ops(map))
	6436	+ continue;
	6437	+ if (map->sec_offset <= offset &&
	6438	+ offset - map->sec_offset < map->def.value_size)
	6439	+ return map;
	6440	+ }
	6441	+
	6442	+ return NULL;
	6443	+}
	6444	+
	6445	+/* Collect the reloc from ELF and populate the st_ops->progs[] */
	6446	+static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj,
	6447	+ GElf_Shdr *shdr,
	6448	+ Elf_Data *data)
	6449	+{
	6450	+ const struct btf_member *member;
	6451	+ struct bpf_struct_ops *st_ops;
	6452	+ struct bpf_program *prog;
	6453	+ unsigned int shdr_idx;
	6454	+ const struct btf *btf;
	6455	+ struct bpf_map *map;
	6456	+ Elf_Data *symbols;
	6457	+ unsigned int moff;
	6458	+ const char *name;
	6459	+ u32 member_idx;
	6460	+ GElf_Sym sym;
	6461	+ GElf_Rel rel;
	6462	+ int i, nrels;
	6463	+
	6464	+ symbols = obj->efile.symbols;
	6465	+ btf = obj->btf;
	6466	+ nrels = shdr->sh_size / shdr->sh_entsize;
	6467	+ for (i = 0; i < nrels; i++) {
	6468	+ if (!gelf_getrel(data, i, &rel)) {
	6469	+ pr_warn("struct_ops reloc: failed to get %d reloc\n", i);
	6470	+ return -LIBBPF_ERRNO__FORMAT;
	6471	+ }
	6472	+
	6473	+ if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) {
	6474	+ pr_warn("struct_ops reloc: symbol %zx not found\n",
	6475	+ (size_t)GELF_R_SYM(rel.r_info));
	6476	+ return -LIBBPF_ERRNO__FORMAT;
	6477	+ }
	6478	+
	6479	+ name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
	6480	+ sym.st_name) ? : "<?>";
	6481	+ map = find_struct_ops_map_by_offset(obj, rel.r_offset);
	6482	+ if (!map) {
	6483	+ pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n",
	6484	+ (size_t)rel.r_offset);
	6485	+ return -EINVAL;
	6486	+ }
	6487	+
	6488	+ moff = rel.r_offset - map->sec_offset;
	6489	+ shdr_idx = sym.st_shndx;
	6490	+ st_ops = map->st_ops;
	6491	+ pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n",
	6492	+ map->name,
	6493	+ (long long)(rel.r_info >> 32),
	6494	+ (long long)sym.st_value,
	6495	+ shdr_idx, (size_t)rel.r_offset,
	6496	+ map->sec_offset, sym.st_name, name);
	6497	+
	6498	+ if (shdr_idx >= SHN_LORESERVE) {
	6499	+ pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n",
	6500	+ map->name, (size_t)rel.r_offset, shdr_idx);
	6501	+ return -LIBBPF_ERRNO__RELOC;
	6502	+ }
	6503	+
	6504	+ member = find_member_by_offset(st_ops->type, moff * 8);
	6505	+ if (!member) {
	6506	+ pr_warn("struct_ops reloc %s: cannot find member at moff %u\n",
	6507	+ map->name, moff);
	6508	+ return -EINVAL;
	6509	+ }
	6510	+ member_idx = member - btf_members(st_ops->type);
	6511	+ name = btf__name_by_offset(btf, member->name_off);
	6512	+
	6513	+ if (!resolve_func_ptr(btf, member->type, NULL)) {
	6514	+ pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n",
	6515	+ map->name, name);
	6516	+ return -EINVAL;
	6517	+ }
	6518	+
	6519	+ prog = bpf_object__find_prog_by_idx(obj, shdr_idx);
	6520	+ if (!prog) {
	6521	+ pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
	6522	+ map->name, shdr_idx, name);
	6523	+ return -EINVAL;
	6524	+ }
	6525	+
	6526	+ if (prog->type == BPF_PROG_TYPE_UNSPEC) {
	6527	+ const struct bpf_sec_def *sec_def;
	6528	+
	6529	+ sec_def = find_sec_def(prog->section_name);
	6530	+ if (sec_def &&
	6531	+ sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) {
	6532	+ /* for pr_warn */
	6533	+ prog->type = sec_def->prog_type;
	6534	+ goto invalid_prog;
	6535	+ }
	6536	+
	6537	+ prog->type = BPF_PROG_TYPE_STRUCT_OPS;
	6538	+ prog->attach_btf_id = st_ops->type_id;
	6539	+ prog->expected_attach_type = member_idx;
	6540	+ } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS \|\|
	6541	+ prog->attach_btf_id != st_ops->type_id \|\|
	6542	+ prog->expected_attach_type != member_idx) {
	6543	+ goto invalid_prog;
	6544	+ }
	6545	+ st_ops->progs[member_idx] = prog;
	6546	+ }
	6547	+
	6548	+ return 0;
	6549	+
	6550	+invalid_prog:
	6551	+ pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
	6552	+ map->name, prog->name, prog->section_name, prog->type,
	6553	+ prog->attach_btf_id, prog->expected_attach_type, name);
	6554	+ return -EINVAL;
	6555	+}
	6556	+
5978	6557	#define BTF_PREFIX "btf_trace_"
5979	6558	int libbpf_find_vmlinux_btf_id(const char *name,
5980	6559	enum bpf_attach_type attach_type)
5981	6560	{
5982		− struct btf *btf = bpf_core_find_kernel_btf();
	6561	+ struct btf *btf = bpf_find_kernel_btf();
5983	6562	char raw_tp_btf[128] = BTF_PREFIX;
5984	6563	char *dst = raw_tp_btf + sizeof(BTF_PREFIX) - 1;
5985	6564	const char *btf_name;
@@ -6805,6 +7384,58 @@ struct bpf_link bpf_program__attach(struct bpf_program prog) · 819 unmodified lines
6805	7384	return sec_def->attach_fn(sec_def, prog);
6806	7385	}
6807	7386
	7387	+static int bpf_link__detach_struct_ops(struct bpf_link *link)
	7388	+{
	7389	+ struct bpf_link_fd l = (void )link;
	7390	+ __u32 zero = 0;
	7391	+
	7392	+ if (bpf_map_delete_elem(l->fd, &zero))
	7393	+ return -errno;
	7394	+
	7395	+ return 0;
	7396	+}
	7397	+
	7398	+struct bpf_link bpf_map__attach_struct_ops(struct bpf_map map)
	7399	+{
	7400	+ struct bpf_struct_ops *st_ops;
	7401	+ struct bpf_link_fd *link;
	7402	+ __u32 i, zero = 0;
	7403	+ int err;
	7404	+
	7405	+ if (!bpf_map__is_struct_ops(map) \|\| map->fd == -1)
	7406	+ return ERR_PTR(-EINVAL);
	7407	+
	7408	+ link = calloc(1, sizeof(*link));
	7409	+ if (!link)
	7410	+ return ERR_PTR(-EINVAL);
	7411	+
	7412	+ st_ops = map->st_ops;
	7413	+ for (i = 0; i < btf_vlen(st_ops->type); i++) {
	7414	+ struct bpf_program *prog = st_ops->progs[i];
	7415	+ void *kern_data;
	7416	+ int prog_fd;
	7417	+
	7418	+ if (!prog)
	7419	+ continue;
	7420	+
	7421	+ prog_fd = bpf_program__fd(prog);
	7422	+ kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];
	7423	+ (unsigned long )kern_data = prog_fd;
	7424	+ }
	7425	+
	7426	+ err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0);
	7427	+ if (err) {
	7428	+ err = -errno;
	7429	+ free(link);
	7430	+ return ERR_PTR(err);
	7431	+ }
	7432	+
	7433	+ link->link.detach = bpf_link__detach_struct_ops;
	7434	+ link->fd = map->fd;
	7435	+
	7436	+ return (struct bpf_link *)link;
	7437	+}
	7438	+
6808	7439	enum bpf_perf_event_ret
6809	7440	bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
6810	7441	void *copy_mem, size_t copy_size,

tools/lib/bpf/libbpf.h+4−1

@@ -239,6 +239,8 @@ bpf_program__attach_raw_tracepoint(struct bpf_program *prog, · 238 unmodified lines
239	239
240	240	LIBBPF_API struct bpf_link *
241	241	bpf_program__attach_trace(struct bpf_program *prog);
	242	+struct bpf_map;
	243	+LIBBPF_API struct bpf_link bpf_map__attach_struct_ops(struct bpf_map map);
242	244	struct bpf_insn;
243	245
244	246	/*
@@ -315,6 +317,7 @@ LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog); · 70 unmodified lines
315	317	LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog);
316	318	LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog);
317	319	LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog);
	320	+LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog);
318	321
319	322	LIBBPF_API enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog);
320	323	LIBBPF_API void bpf_program__set_type(struct bpf_program *prog,
@@ -335,6 +338,7 @@ LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog); · 14 unmodified lines
335	338	LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog);
336	339	LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog);
337	340	LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog);
	341	+LIBBPF_API bool bpf_program__is_struct_ops(const struct bpf_program *prog);
338	342
339	343	/*
340	344	* No need for __attribute__((packed)), all members of 'bpf_map_def'
@@ -354,7 +358,6 @@ struct bpf_map_def { · 13 unmodified lines
354	358	* The 'struct bpf_map' in include/linux/bpf.h is internal to the kernel,
355	359	* so no need to worry about a name clash.
356	360	*/
357		−struct bpf_map;
358	361	LIBBPF_API struct bpf_map *
359	362	bpf_object__find_map_by_name(const struct bpf_object obj, const char name);
360	363

tools/lib/bpf/libbpf.map+3−0

@@ -213,6 +213,7 @@ LIBBPF_0.0.7 { · 212 unmodified lines
213	213	global:
214	214	btf_dump__emit_type_decl;
215	215	bpf_link__disconnect;
	216	+ bpf_map__attach_struct_ops;
216	217	bpf_object__find_program_by_name;
217	218	bpf_object__attach_skeleton;
218	219	bpf_object__destroy_skeleton;
@@ -223,5 +224,7 @@ LIBBPF_0.0.7 { · 4 unmodified lines
223	224	bpf_prog_attach_xattr;
224	225	bpf_program__attach;
225	226	bpf_program__name;
	227	+ bpf_program__is_struct_ops;
	228	+ bpf_program__set_struct_ops;
226	229	btf__align_of;
227	230	} LIBBPF_0.0.6;

tools/lib/bpf/libbpf_probes.c+2−0

@@ -103,6 +103,7 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, · 102 unmodified lines
103	103	case BPF_PROG_TYPE_CGROUP_SYSCTL:
104	104	case BPF_PROG_TYPE_CGROUP_SOCKOPT:
105	105	case BPF_PROG_TYPE_TRACING:
	106	+ case BPF_PROG_TYPE_STRUCT_OPS:
106	107	default:
107	108	break;
108	109	}
@@ -251,6 +252,7 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) · 142 unmodified lines
251	252	case BPF_MAP_TYPE_XSKMAP:
252	253	case BPF_MAP_TYPE_SOCKHASH:
253	254	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
	255	+ case BPF_MAP_TYPE_STRUCT_OPS:
254	256	default:
255	257	break;
256	258	}

tools/testing/selftests/bpf/bpf_tcp_helpers.h+228−0

@@ -0,0 +1,228 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __BPF_TCP_HELPERS_H
	3	+#define __BPF_TCP_HELPERS_H
	4	+
	5	+#include <stdbool.h>
	6	+#include <linux/types.h>
	7	+#include <bpf_helpers.h>
	8	+#include <bpf_core_read.h>
	9	+#include "bpf_trace_helpers.h"
	10	+
	11	+/* "struct_ops/" is only a convention. not a requirement. */
	12	+#define BPF_TCP_OPS_0(fname, ret_type, ...) BPF_TRACE_x(0, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	13	+#define BPF_TCP_OPS_1(fname, ret_type, ...) BPF_TRACE_x(1, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	14	+#define BPF_TCP_OPS_2(fname, ret_type, ...) BPF_TRACE_x(2, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	15	+#define BPF_TCP_OPS_3(fname, ret_type, ...) BPF_TRACE_x(3, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	16	+#define BPF_TCP_OPS_4(fname, ret_type, ...) BPF_TRACE_x(4, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	17	+#define BPF_TCP_OPS_5(fname, ret_type, ...) BPF_TRACE_x(5, "struct_ops/"#fname, fname, ret_type, __VA_ARGS__)
	18	+
	19	+struct sock_common {
	20	+ unsigned char skc_state;
	21	+} __attribute__((preserve_access_index));
	22	+
	23	+struct sock {
	24	+ struct sock_common __sk_common;
	25	+} __attribute__((preserve_access_index));
	26	+
	27	+struct inet_sock {
	28	+ struct sock sk;
	29	+} __attribute__((preserve_access_index));
	30	+
	31	+struct inet_connection_sock {
	32	+ struct inet_sock icsk_inet;
	33	+ __u8 icsk_ca_state:6,
	34	+ icsk_ca_setsockopt:1,
	35	+ icsk_ca_dst_locked:1;
	36	+ struct {
	37	+ __u8 pending;
	38	+ } icsk_ack;
	39	+ __u64 icsk_ca_priv[104 / sizeof(__u64)];
	40	+} __attribute__((preserve_access_index));
	41	+
	42	+struct tcp_sock {
	43	+ struct inet_connection_sock inet_conn;
	44	+
	45	+ __u32 rcv_nxt;
	46	+ __u32 snd_nxt;
	47	+ __u32 snd_una;
	48	+ __u8 ecn_flags;
	49	+ __u32 delivered;
	50	+ __u32 delivered_ce;
	51	+ __u32 snd_cwnd;
	52	+ __u32 snd_cwnd_cnt;
	53	+ __u32 snd_cwnd_clamp;
	54	+ __u32 snd_ssthresh;
	55	+ __u8 syn_data:1, /* SYN includes data */
	56	+ syn_fastopen:1, /* SYN includes Fast Open option */
	57	+ syn_fastopen_exp:1,/* SYN includes Fast Open exp. option */
	58	+ syn_fastopen_ch:1, /* Active TFO re-enabling probe */
	59	+ syn_data_acked:1,/* data in SYN is acked by SYN-ACK */
	60	+ save_syn:1, /* Save headers of SYN packet */
	61	+ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */
	62	+ syn_smc:1; /* SYN includes SMC */
	63	+ __u32 max_packets_out;
	64	+ __u32 lsndtime;
	65	+ __u32 prior_cwnd;
	66	+} __attribute__((preserve_access_index));
	67	+
	68	+static __always_inline struct inet_connection_sock inet_csk(const struct sock sk)
	69	+{
	70	+ return (struct inet_connection_sock *)sk;
	71	+}
	72	+
	73	+static __always_inline void inet_csk_ca(const struct sock sk)
	74	+{
	75	+ return (void *)inet_csk(sk)->icsk_ca_priv;
	76	+}
	77	+
	78	+static __always_inline struct tcp_sock tcp_sk(const struct sock sk)
	79	+{
	80	+ return (struct tcp_sock *)sk;
	81	+}
	82	+
	83	+static __always_inline bool before(__u32 seq1, __u32 seq2)
	84	+{
	85	+ return (__s32)(seq1-seq2) < 0;
	86	+}
	87	+#define after(seq2, seq1) before(seq1, seq2)
	88	+
	89	+#define TCP_ECN_OK 1
	90	+#define TCP_ECN_QUEUE_CWR 2
	91	+#define TCP_ECN_DEMAND_CWR 4
	92	+#define TCP_ECN_SEEN 8
	93	+
	94	+enum inet_csk_ack_state_t {
	95	+ ICSK_ACK_SCHED = 1,
	96	+ ICSK_ACK_TIMER = 2,
	97	+ ICSK_ACK_PUSHED = 4,
	98	+ ICSK_ACK_PUSHED2 = 8,
	99	+ ICSK_ACK_NOW = 16 /* Send the next ACK immediately (once) */
	100	+};
	101	+
	102	+enum tcp_ca_event {
	103	+ CA_EVENT_TX_START = 0,
	104	+ CA_EVENT_CWND_RESTART = 1,
	105	+ CA_EVENT_COMPLETE_CWR = 2,
	106	+ CA_EVENT_LOSS = 3,
	107	+ CA_EVENT_ECN_NO_CE = 4,
	108	+ CA_EVENT_ECN_IS_CE = 5,
	109	+};
	110	+
	111	+enum tcp_ca_state {
	112	+ TCP_CA_Open = 0,
	113	+ TCP_CA_Disorder = 1,
	114	+ TCP_CA_CWR = 2,
	115	+ TCP_CA_Recovery = 3,
	116	+ TCP_CA_Loss = 4
	117	+};
	118	+
	119	+struct ack_sample {
	120	+ __u32 pkts_acked;
	121	+ __s32 rtt_us;
	122	+ __u32 in_flight;
	123	+} __attribute__((preserve_access_index));
	124	+
	125	+struct rate_sample {
	126	+ __u64 prior_mstamp; /* starting timestamp for interval */
	127	+ __u32 prior_delivered; /* tp->delivered at "prior_mstamp" */
	128	+ __s32 delivered; /* number of packets delivered over interval */
	129	+ long interval_us; /* time for tp->delivered to incr "delivered" */
	130	+ __u32 snd_interval_us; /* snd interval for delivered packets */
	131	+ __u32 rcv_interval_us; /* rcv interval for delivered packets */
	132	+ long rtt_us; /* RTT of last (S)ACKed packet (or -1) */
	133	+ int losses; /* number of packets marked lost upon ACK */
	134	+ __u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */
	135	+ __u32 prior_in_flight; /* in flight before this ACK */
	136	+ bool is_app_limited; /* is sample from packet with bubble in pipe? */
	137	+ bool is_retrans; /* is sample from retransmission? */
	138	+ bool is_ack_delayed; /* is this (likely) a delayed ACK? */
	139	+} __attribute__((preserve_access_index));
	140	+
	141	+#define TCP_CA_NAME_MAX 16
	142	+#define TCP_CONG_NEEDS_ECN 0x2
	143	+
	144	+struct tcp_congestion_ops {
	145	+ char name[TCP_CA_NAME_MAX];
	146	+ __u32 flags;
	147	+
	148	+ /* initialize private data (optional) */
	149	+ void (init)(struct sock sk);
	150	+ /* cleanup private data (optional) */
	151	+ void (release)(struct sock sk);
	152	+
	153	+ /* return slow start threshold (required) */
	154	+ __u32 (ssthresh)(struct sock sk);
	155	+ /* do new cwnd calculation (required) */
	156	+ void (cong_avoid)(struct sock sk, __u32 ack, __u32 acked);
	157	+ /* call before changing ca_state (optional) */
	158	+ void (set_state)(struct sock sk, __u8 new_state);
	159	+ /* call when cwnd event occurs (optional) */
	160	+ void (cwnd_event)(struct sock sk, enum tcp_ca_event ev);
	161	+ /* call when ack arrives (optional) */
	162	+ void (in_ack_event)(struct sock sk, __u32 flags);
	163	+ /* new value of cwnd after loss (required) */
	164	+ __u32 (undo_cwnd)(struct sock sk);
	165	+ /* hook for packet ack accounting (optional) */
	166	+ void (pkts_acked)(struct sock sk, const struct ack_sample *sample);
	167	+ /* override sysctl_tcp_min_tso_segs */
	168	+ __u32 (min_tso_segs)(struct sock sk);
	169	+ /* returns the multiplier used in tcp_sndbuf_expand (optional) */
	170	+ __u32 (sndbuf_expand)(struct sock sk);
	171	+ /* call when packets are delivered to update cwnd and pacing rate,
	172	+ * after all the ca_state processing. (optional)
	173	+ */
	174	+ void (cong_control)(struct sock sk, const struct rate_sample *rs);
	175	+};
	176	+
	177	+#define min(a, b) ((a) < (b) ? (a) : (b))
	178	+#define max(a, b) ((a) > (b) ? (a) : (b))
	179	+#define min_not_zero(x, y) ({ \
	180	+ typeof(x) __x = (x); \
	181	+ typeof(y) __y = (y); \
	182	+ __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
	183	+
	184	+static __always_inline __u32 tcp_slow_start(struct tcp_sock *tp, __u32 acked)
	185	+{
	186	+ __u32 cwnd = min(tp->snd_cwnd + acked, tp->snd_ssthresh);
	187	+
	188	+ acked -= cwnd - tp->snd_cwnd;
	189	+ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp);
	190	+
	191	+ return acked;
	192	+}
	193	+
	194	+static __always_inline bool tcp_in_slow_start(const struct tcp_sock *tp)
	195	+{
	196	+ return tp->snd_cwnd < tp->snd_ssthresh;
	197	+}
	198	+
	199	+static __always_inline bool tcp_is_cwnd_limited(const struct sock *sk)
	200	+{
	201	+ const struct tcp_sock *tp = tcp_sk(sk);
	202	+
	203	+ /* If in slow start, ensure cwnd grows to twice what was ACKed. */
	204	+ if (tcp_in_slow_start(tp))
	205	+ return tp->snd_cwnd < 2 * tp->max_packets_out;
	206	+
	207	+ return !!BPF_CORE_READ_BITFIELD(tp, is_cwnd_limited);
	208	+}
	209	+
	210	+static __always_inline void tcp_cong_avoid_ai(struct tcp_sock *tp, __u32 w, __u32 acked)
	211	+{
	212	+ /* If credits accumulated at a higher w, apply them gently now. */
	213	+ if (tp->snd_cwnd_cnt >= w) {
	214	+ tp->snd_cwnd_cnt = 0;
	215	+ tp->snd_cwnd++;
	216	+ }
	217	+
	218	+ tp->snd_cwnd_cnt += acked;
	219	+ if (tp->snd_cwnd_cnt >= w) {
	220	+ __u32 delta = tp->snd_cwnd_cnt / w;
	221	+
	222	+ tp->snd_cwnd_cnt -= delta * w;
	223	+ tp->snd_cwnd += delta;
	224	+ }
	225	+ tp->snd_cwnd = min(tp->snd_cwnd, tp->snd_cwnd_clamp);
	226	+}
	227	+
	228	+#endif

tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c+187−0

@@ -0,0 +1,187 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2019 Facebook */
	3	+
	4	+#include <linux/err.h>
	5	+#include <test_progs.h>
	6	+#include "bpf_dctcp.skel.h"
	7	+
	8	+#define min(a, b) ((a) < (b) ? (a) : (b))
	9	+
	10	+static const unsigned int total_bytes = 10 * 1024 * 1024;
	11	+static const struct timeval timeo_sec = { .tv_sec = 10 };
	12	+static const size_t timeo_optlen = sizeof(timeo_sec);
	13	+static int stop, duration;
	14	+
	15	+static int settimeo(int fd)
	16	+{
	17	+ int err;
	18	+
	19	+ err = setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeo_sec,
	20	+ timeo_optlen);
	21	+ if (CHECK(err == -1, "setsockopt(fd, SO_RCVTIMEO)", "errno:%d\n",
	22	+ errno))
	23	+ return -1;
	24	+
	25	+ err = setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeo_sec,
	26	+ timeo_optlen);
	27	+ if (CHECK(err == -1, "setsockopt(fd, SO_SNDTIMEO)", "errno:%d\n",
	28	+ errno))
	29	+ return -1;
	30	+
	31	+ return 0;
	32	+}
	33	+
	34	+static int settcpca(int fd, const char *tcp_ca)
	35	+{
	36	+ int err;
	37	+
	38	+ err = setsockopt(fd, IPPROTO_TCP, TCP_CONGESTION, tcp_ca, strlen(tcp_ca));
	39	+ if (CHECK(err == -1, "setsockopt(fd, TCP_CONGESTION)", "errno:%d\n",
	40	+ errno))
	41	+ return -1;
	42	+
	43	+ return 0;
	44	+}
	45	+
	46	+static void server(void arg)
	47	+{
	48	+ int lfd = (int)(long)arg, err = 0, fd;
	49	+ ssize_t nr_sent = 0, bytes = 0;
	50	+ char batch[1500];
	51	+
	52	+ fd = accept(lfd, NULL, NULL);
	53	+ while (fd == -1) {
	54	+ if (errno == EINTR)
	55	+ continue;
	56	+ err = -errno;
	57	+ goto done;
	58	+ }
	59	+
	60	+ if (settimeo(fd)) {
	61	+ err = -errno;
	62	+ goto done;
	63	+ }
	64	+
	65	+ while (bytes < total_bytes && !READ_ONCE(stop)) {
	66	+ nr_sent = send(fd, &batch,
	67	+ min(total_bytes - bytes, sizeof(batch)), 0);
	68	+ if (nr_sent == -1 && errno == EINTR)
	69	+ continue;
	70	+ if (nr_sent == -1) {
	71	+ err = -errno;
	72	+ break;
	73	+ }
	74	+ bytes += nr_sent;
	75	+ }
	76	+
	77	+ CHECK(bytes != total_bytes, "send", "%zd != %u nr_sent:%zd errno:%d\n",
	78	+ bytes, total_bytes, nr_sent, errno);
	79	+
	80	+done:
	81	+ if (fd != -1)
	82	+ close(fd);
	83	+ if (err) {
	84	+ WRITE_ONCE(stop, 1);
	85	+ return ERR_PTR(err);
	86	+ }
	87	+ return NULL;
	88	+}
	89	+
	90	+static void do_test(const char *tcp_ca)
	91	+{
	92	+ struct sockaddr_in6 sa6 = {};
	93	+ ssize_t nr_recv = 0, bytes = 0;
	94	+ int lfd = -1, fd = -1;
	95	+ pthread_t srv_thread;
	96	+ socklen_t addrlen = sizeof(sa6);
	97	+ void *thread_ret;
	98	+ char batch[1500];
	99	+ int err;
	100	+
	101	+ WRITE_ONCE(stop, 0);
	102	+
	103	+ lfd = socket(AF_INET6, SOCK_STREAM, 0);
	104	+ if (CHECK(lfd == -1, "socket", "errno:%d\n", errno))
	105	+ return;
	106	+ fd = socket(AF_INET6, SOCK_STREAM, 0);
	107	+ if (CHECK(fd == -1, "socket", "errno:%d\n", errno)) {
	108	+ close(lfd);
	109	+ return;
	110	+ }
	111	+
	112	+ if (settcpca(lfd, tcp_ca) \|\| settcpca(fd, tcp_ca) \|\|
	113	+ settimeo(lfd) \|\| settimeo(fd))
	114	+ goto done;
	115	+
	116	+ /* bind, listen and start server thread to accept */
	117	+ sa6.sin6_family = AF_INET6;
	118	+ sa6.sin6_addr = in6addr_loopback;
	119	+ err = bind(lfd, (struct sockaddr *)&sa6, addrlen);
	120	+ if (CHECK(err == -1, "bind", "errno:%d\n", errno))
	121	+ goto done;
	122	+ err = getsockname(lfd, (struct sockaddr *)&sa6, &addrlen);
	123	+ if (CHECK(err == -1, "getsockname", "errno:%d\n", errno))
	124	+ goto done;
	125	+ err = listen(lfd, 1);
	126	+ if (CHECK(err == -1, "listen", "errno:%d\n", errno))
	127	+ goto done;
	128	+ err = pthread_create(&srv_thread, NULL, server, (void *)(long)lfd);
	129	+ if (CHECK(err != 0, "pthread_create", "err:%d\n", err))
	130	+ goto done;
	131	+
	132	+ /* connect to server */
	133	+ err = connect(fd, (struct sockaddr *)&sa6, addrlen);
	134	+ if (CHECK(err == -1, "connect", "errno:%d\n", errno))
	135	+ goto wait_thread;
	136	+
	137	+ /* recv total_bytes */
	138	+ while (bytes < total_bytes && !READ_ONCE(stop)) {
	139	+ nr_recv = recv(fd, &batch,
	140	+ min(total_bytes - bytes, sizeof(batch)), 0);
	141	+ if (nr_recv == -1 && errno == EINTR)
	142	+ continue;
	143	+ if (nr_recv == -1)
	144	+ break;
	145	+ bytes += nr_recv;
	146	+ }
	147	+
	148	+ CHECK(bytes != total_bytes, "recv", "%zd != %u nr_recv:%zd errno:%d\n",
	149	+ bytes, total_bytes, nr_recv, errno);
	150	+
	151	+wait_thread:
	152	+ WRITE_ONCE(stop, 1);
	153	+ pthread_join(srv_thread, &thread_ret);
	154	+ CHECK(IS_ERR(thread_ret), "pthread_join", "thread_ret:%ld",
	155	+ PTR_ERR(thread_ret));
	156	+done:
	157	+ close(lfd);
	158	+ close(fd);
	159	+}
	160	+
	161	+static void test_dctcp(void)
	162	+{
	163	+ struct bpf_dctcp *dctcp_skel;
	164	+ struct bpf_link *link;
	165	+
	166	+ dctcp_skel = bpf_dctcp__open_and_load();
	167	+ if (CHECK(!dctcp_skel, "bpf_dctcp__open_and_load", "failed\n"))
	168	+ return;
	169	+
	170	+ link = bpf_map__attach_struct_ops(dctcp_skel->maps.dctcp);
	171	+ if (CHECK(IS_ERR(link), "bpf_map__attach_struct_ops", "err:%ld\n",
	172	+ PTR_ERR(link))) {
	173	+ bpf_dctcp__destroy(dctcp_skel);
	174	+ return;
	175	+ }
	176	+
	177	+ do_test("bpf_dctcp");
	178	+
	179	+ bpf_link__destroy(link);
	180	+ bpf_dctcp__destroy(dctcp_skel);
	181	+}
	182	+
	183	+void test_bpf_tcp_ca(void)
	184	+{
	185	+ if (test__start_subtest("dctcp"))
	186	+ test_dctcp();
	187	+}

tools/testing/selftests/bpf/progs/bpf_dctcp.c+210−0

@@ -0,0 +1,210 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+/* Copyright (c) 2019 Facebook */
	3	+
	4	+/* WARNING: This implemenation is not necessarily the same
	5	+ * as the tcp_dctcp.c. The purpose is mainly for testing
	6	+ * the kernel BPF logic.
	7	+ */
	8	+
	9	+#include <linux/bpf.h>
	10	+#include <linux/types.h>
	11	+#include "bpf_tcp_helpers.h"
	12	+
	13	+char _license[] SEC("license") = "GPL";
	14	+
	15	+#define DCTCP_MAX_ALPHA 1024U
	16	+
	17	+struct dctcp {
	18	+ __u32 old_delivered;
	19	+ __u32 old_delivered_ce;
	20	+ __u32 prior_rcv_nxt;
	21	+ __u32 dctcp_alpha;
	22	+ __u32 next_seq;
	23	+ __u32 ce_state;
	24	+ __u32 loss_cwnd;
	25	+};
	26	+
	27	+static unsigned int dctcp_shift_g = 4; /* g = 1/2^4 */
	28	+static unsigned int dctcp_alpha_on_init = DCTCP_MAX_ALPHA;
	29	+
	30	+static __always_inline void dctcp_reset(const struct tcp_sock *tp,
	31	+ struct dctcp *ca)
	32	+{
	33	+ ca->next_seq = tp->snd_nxt;
	34	+
	35	+ ca->old_delivered = tp->delivered;
	36	+ ca->old_delivered_ce = tp->delivered_ce;
	37	+}
	38	+
	39	+BPF_TCP_OPS_1(dctcp_init, void, struct sock *, sk)
	40	+{
	41	+ const struct tcp_sock *tp = tcp_sk(sk);
	42	+ struct dctcp *ca = inet_csk_ca(sk);
	43	+
	44	+ ca->prior_rcv_nxt = tp->rcv_nxt;
	45	+ ca->dctcp_alpha = min(dctcp_alpha_on_init, DCTCP_MAX_ALPHA);
	46	+ ca->loss_cwnd = 0;
	47	+ ca->ce_state = 0;
	48	+
	49	+ dctcp_reset(tp, ca);
	50	+}
	51	+
	52	+BPF_TCP_OPS_1(dctcp_ssthresh, __u32, struct sock *, sk)
	53	+{
	54	+ struct dctcp *ca = inet_csk_ca(sk);
	55	+ struct tcp_sock *tp = tcp_sk(sk);
	56	+
	57	+ ca->loss_cwnd = tp->snd_cwnd;
	58	+ return max(tp->snd_cwnd - ((tp->snd_cwnd * ca->dctcp_alpha) >> 11U), 2U);
	59	+}
	60	+
	61	+BPF_TCP_OPS_2(dctcp_update_alpha, void,
	62	+ struct sock *, sk, __u32, flags)
	63	+{
	64	+ const struct tcp_sock *tp = tcp_sk(sk);
	65	+ struct dctcp *ca = inet_csk_ca(sk);
	66	+
	67	+ /* Expired RTT */
	68	+ if (!before(tp->snd_una, ca->next_seq)) {
	69	+ __u32 delivered_ce = tp->delivered_ce - ca->old_delivered_ce;
	70	+ __u32 alpha = ca->dctcp_alpha;
	71	+
	72	+ /* alpha = (1 - g) * alpha + g * F */
	73	+
	74	+ alpha -= min_not_zero(alpha, alpha >> dctcp_shift_g);
	75	+ if (delivered_ce) {
	76	+ __u32 delivered = tp->delivered - ca->old_delivered;
	77	+
	78	+ /* If dctcp_shift_g == 1, a 32bit value would overflow
	79	+ * after 8 M packets.
	80	+ */
	81	+ delivered_ce <<= (10 - dctcp_shift_g);
	82	+ delivered_ce /= max(1U, delivered);
	83	+
	84	+ alpha = min(alpha + delivered_ce, DCTCP_MAX_ALPHA);
	85	+ }
	86	+ ca->dctcp_alpha = alpha;
	87	+ dctcp_reset(tp, ca);
	88	+ }
	89	+}
	90	+
	91	+static __always_inline void dctcp_react_to_loss(struct sock *sk)
	92	+{
	93	+ struct dctcp *ca = inet_csk_ca(sk);
	94	+ struct tcp_sock *tp = tcp_sk(sk);
	95	+
	96	+ ca->loss_cwnd = tp->snd_cwnd;
	97	+ tp->snd_ssthresh = max(tp->snd_cwnd >> 1U, 2U);
	98	+}
	99	+
	100	+BPF_TCP_OPS_2(dctcp_state, void, struct sock *, sk, __u8, new_state)
	101	+{
	102	+ if (new_state == TCP_CA_Recovery &&
	103	+ new_state != BPF_CORE_READ_BITFIELD(inet_csk(sk), icsk_ca_state))
	104	+ dctcp_react_to_loss(sk);
	105	+ /* We handle RTO in dctcp_cwnd_event to ensure that we perform only
	106	+ * one loss-adjustment per RTT.
	107	+ */
	108	+}
	109	+
	110	+static __always_inline void dctcp_ece_ack_cwr(struct sock *sk, __u32 ce_state)
	111	+{
	112	+ struct tcp_sock *tp = tcp_sk(sk);
	113	+
	114	+ if (ce_state == 1)
	115	+ tp->ecn_flags \|= TCP_ECN_DEMAND_CWR;
	116	+ else
	117	+ tp->ecn_flags &= ~TCP_ECN_DEMAND_CWR;
	118	+}
	119	+
	120	+/* Minimal DCTP CE state machine:
	121	+ *
	122	+ * S: 0 <- last pkt was non-CE
	123	+ * 1 <- last pkt was CE
	124	+ */
	125	+static __always_inline
	126	+void dctcp_ece_ack_update(struct sock *sk, enum tcp_ca_event evt,
	127	+ __u32 prior_rcv_nxt, __u32 ce_state)
	128	+{
	129	+ __u32 new_ce_state = (evt == CA_EVENT_ECN_IS_CE) ? 1 : 0;
	130	+
	131	+ if (*ce_state != new_ce_state) {
	132	+ /* CE state has changed, force an immediate ACK to
	133	+ * reflect the new CE state. If an ACK was delayed,
	134	+ * send that first to reflect the prior CE state.
	135	+ */
	136	+ if (inet_csk(sk)->icsk_ack.pending & ICSK_ACK_TIMER) {
	137	+ dctcp_ece_ack_cwr(sk, *ce_state);
	138	+ bpf_tcp_send_ack(sk, *prior_rcv_nxt);
	139	+ }
	140	+ inet_csk(sk)->icsk_ack.pending \|= ICSK_ACK_NOW;
	141	+ }
	142	+ *prior_rcv_nxt = tcp_sk(sk)->rcv_nxt;
	143	+ *ce_state = new_ce_state;
	144	+ dctcp_ece_ack_cwr(sk, new_ce_state);
	145	+}
	146	+
	147	+BPF_TCP_OPS_2(dctcp_cwnd_event, void,
	148	+ struct sock *, sk, enum tcp_ca_event, ev)
	149	+{
	150	+ struct dctcp *ca = inet_csk_ca(sk);
	151	+
	152	+ switch (ev) {
	153	+ case CA_EVENT_ECN_IS_CE:
	154	+ case CA_EVENT_ECN_NO_CE:
	155	+ dctcp_ece_ack_update(sk, ev, &ca->prior_rcv_nxt, &ca->ce_state);
	156	+ break;
	157	+ case CA_EVENT_LOSS:
	158	+ dctcp_react_to_loss(sk);
	159	+ break;
	160	+ default:
	161	+ /* Don't care for the rest. */
	162	+ break;
	163	+ }
	164	+}
	165	+
	166	+BPF_TCP_OPS_1(dctcp_cwnd_undo, __u32, struct sock *, sk)
	167	+{
	168	+ const struct dctcp *ca = inet_csk_ca(sk);
	169	+
	170	+ return max(tcp_sk(sk)->snd_cwnd, ca->loss_cwnd);
	171	+}
	172	+
	173	+BPF_TCP_OPS_3(tcp_reno_cong_avoid, void,
	174	+ struct sock *, sk, __u32, ack, __u32, acked)
	175	+{
	176	+ struct tcp_sock *tp = tcp_sk(sk);
	177	+
	178	+ if (!tcp_is_cwnd_limited(sk))
	179	+ return;
	180	+
	181	+ /* In "safe" area, increase. */
	182	+ if (tcp_in_slow_start(tp)) {
	183	+ acked = tcp_slow_start(tp, acked);
	184	+ if (!acked)
	185	+ return;
	186	+ }
	187	+ /* In dangerous area, increase slowly. */
	188	+ tcp_cong_avoid_ai(tp, tp->snd_cwnd, acked);
	189	+}
	190	+
	191	+SEC(".struct_ops")
	192	+struct tcp_congestion_ops dctcp_nouse = {
	193	+ .init = (void *)dctcp_init,
	194	+ .set_state = (void *)dctcp_state,
	195	+ .flags = TCP_CONG_NEEDS_ECN,
	196	+ .name = "bpf_dctcp_nouse",
	197	+};
	198	+
	199	+SEC(".struct_ops")
	200	+struct tcp_congestion_ops dctcp = {
	201	+ .init = (void *)dctcp_init,
	202	+ .in_ack_event = (void *)dctcp_update_alpha,
	203	+ .cwnd_event = (void *)dctcp_cwnd_event,
	204	+ .ssthresh = (void *)dctcp_ssthresh,
	205	+ .cong_avoid = (void *)tcp_reno_cong_avoid,
	206	+ .undo_cwnd = (void *)dctcp_cwnd_undo,
	207	+ .set_state = (void *)dctcp_state,
	208	+ .flags = TCP_CONG_NEEDS_ECN,
	209	+ .name = "bpf_dctcp",
	210	+};