History

bpf: tcp: Support tcp_congestion_ops in bpf

This patch makes "struct tcp_congestion_ops" to be the first user of BPF STRUCT_OPS. It allows implementing a tcp_congestion_ops in bpf. The BPF implemented tcp_congestion_ops can be used like regular kernel tcp-cc through sysctl and setsockopt. e.g. [root@arch-fb-vm1 bpf]# sysctl -a | egrep congestion net.ipv4.tcp_allowed_congestion_control = reno cubic bpf_cubic net.ipv4.tcp_available_congestion_control = reno bic cubic bpf_cubic net.ipv4.tcp_congestion_control = bpf_cubic There has been attempt to move the TCP CC to the user space (e.g. CCP in TCP). The common arguments are faster turn around, get away from long-tail kernel versions in production...etc, which are legit points. BPF has been the continuous effort to join both kernel and userspace upsides together (e.g. XDP to gain the performance advantage without bypassing the kernel). The recent BPF advancements (in particular BTF-aware verifier, BPF trampoline, BPF CO-RE...) made implementing kernel struct ops (e.g. tcp cc) possible in BPF. It allows a faster turnaround for testing algorithm in the production while leveraging the existing (and continue growing) BPF feature/framework instead of building one specifically for userspace TCP CC. This patch allows write access to a few fields in tcp-sock (in bpf_tcp_ca_btf_struct_access()). The optional "get_info" is unsupported now. It can be added later. One possible way is to output the info with a btf-id to describe the content.

Martin KaFai Lau committed 6 years ago

Commit 0baf26b1 parent 85d33df10 files · +261 −16Browse files

Unified · Split

10 files
  1. include/linux/filter.h +2 −0
  2. include/net/tcp.h +2 −0
  3. kernel/bpf/bpf_struct_ops_types.h +6 −1
  4. net/core/filter.c +1 −1
  5. net/ipv4/Makefile +4 −0
  6. net/ipv4/bpf_tcp_ca.c +230 −0
  7. net/ipv4/tcp_cong.c +8 −8
  8. net/ipv4/tcp_ipv4.c +4 −2
  9. net/ipv4/tcp_minisocks.c +2 −2
  10. net/ipv4/tcp_output.c +2 −2
include/linux/filter.h+2−0
@@ -843,6 +843,8 @@ int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog); · 842 unmodified lines
843843 int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
844844 bpf_aux_classic_check_t trans, bool save_orig);
845845 void bpf_prog_destroy(struct bpf_prog *fp);
846+const struct bpf_func_proto *
847+bpf_base_func_proto(enum bpf_func_id func_id);
846848
847849 int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk);
848850 int sk_attach_bpf(u32 ufd, struct sock *sk);
include/net/tcp.h+2−0
@@ -1007,6 +1007,7 @@ enum tcp_ca_ack_event_flags { · 1006 unmodified lines
10071007 #define TCP_CONG_NON_RESTRICTED 0x1
10081008 /* Requires ECN/ECT set on all packets */
10091009 #define TCP_CONG_NEEDS_ECN 0x2
1010+#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
10101011
10111012 union tcp_cc_info;
10121013
@@ -1101,6 +1102,7 @@ u32 tcp_reno_undo_cwnd(struct sock *sk); · 88 unmodified lines
11011102 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
11021103 extern struct tcp_congestion_ops tcp_reno;
11031104
1105+struct tcp_congestion_ops *tcp_ca_find(const char *name);
11041106 struct tcp_congestion_ops *tcp_ca_find_key(u32 key);
11051107 u32 tcp_ca_get_key_by_name(struct net *net, const char *name, bool *ecn_ca);
11061108 #ifdef CONFIG_INET
kernel/bpf/bpf_struct_ops_types.h+6−1
@@ -1,4 +1,9 @@
11 /* SPDX-License-Identifier: GPL-2.0 */
22 /* internal file - do not include directly */
33
4−/* To be filled in a later patch */
4+#ifdef CONFIG_BPF_JIT
5+#ifdef CONFIG_INET
6+#include <net/tcp.h>
7+BPF_STRUCT_OPS_TYPE(tcp_congestion_ops)
8+#endif
9+#endif
net/core/filter.c+1−1
@@ -5935,7 +5935,7 @@ bool bpf_helper_changes_pkt_data(void *func) · 5934 unmodified lines
59355935 return false;
59365936 }
59375937
5938−static const struct bpf_func_proto *
5938+const struct bpf_func_proto *
59395939 bpf_base_func_proto(enum bpf_func_id func_id)
59405940 {
59415941 switch (func_id) {
net/ipv4/Makefile+4−0
@@ -65,3 +65,7 @@ obj-$(CONFIG_NETLABEL) += cipso_ipv4.o · 64 unmodified lines
6565
6666 obj-$(CONFIG_XFRM) += xfrm4_policy.o xfrm4_state.o xfrm4_input.o \
6767 xfrm4_output.o xfrm4_protocol.o
68+
69+ifeq ($(CONFIG_BPF_JIT),y)
70+obj-$(CONFIG_BPF_SYSCALL) += bpf_tcp_ca.o
71+endif
net/ipv4/bpf_tcp_ca.c+230−0
@@ -0,0 +1,230 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/* Copyright (c) 2019 Facebook */
3+
4+#include <linux/types.h>
5+#include <linux/bpf_verifier.h>
6+#include <linux/bpf.h>
7+#include <linux/btf.h>
8+#include <linux/filter.h>
9+#include <net/tcp.h>
10+
11+static u32 optional_ops[] = {
12+ offsetof(struct tcp_congestion_ops, init),
13+ offsetof(struct tcp_congestion_ops, release),
14+ offsetof(struct tcp_congestion_ops, set_state),
15+ offsetof(struct tcp_congestion_ops, cwnd_event),
16+ offsetof(struct tcp_congestion_ops, in_ack_event),
17+ offsetof(struct tcp_congestion_ops, pkts_acked),
18+ offsetof(struct tcp_congestion_ops, min_tso_segs),
19+ offsetof(struct tcp_congestion_ops, sndbuf_expand),
20+ offsetof(struct tcp_congestion_ops, cong_control),
21+};
22+
23+static u32 unsupported_ops[] = {
24+ offsetof(struct tcp_congestion_ops, get_info),
25+};
26+
27+static const struct btf_type *tcp_sock_type;
28+static u32 tcp_sock_id, sock_id;
29+
30+static int bpf_tcp_ca_init(struct btf *btf)
31+{
32+ s32 type_id;
33+
34+ type_id = btf_find_by_name_kind(btf, "sock", BTF_KIND_STRUCT);
35+ if (type_id < 0)
36+ return -EINVAL;
37+ sock_id = type_id;
38+
39+ type_id = btf_find_by_name_kind(btf, "tcp_sock", BTF_KIND_STRUCT);
40+ if (type_id < 0)
41+ return -EINVAL;
42+ tcp_sock_id = type_id;
43+ tcp_sock_type = btf_type_by_id(btf, tcp_sock_id);
44+
45+ return 0;
46+}
47+
48+static bool is_optional(u32 member_offset)
49+{
50+ unsigned int i;
51+
52+ for (i = 0; i < ARRAY_SIZE(optional_ops); i++) {
53+ if (member_offset == optional_ops[i])
54+ return true;
55+ }
56+
57+ return false;
58+}
59+
60+static bool is_unsupported(u32 member_offset)
61+{
62+ unsigned int i;
63+
64+ for (i = 0; i < ARRAY_SIZE(unsupported_ops); i++) {
65+ if (member_offset == unsupported_ops[i])
66+ return true;
67+ }
68+
69+ return false;
70+}
71+
72+extern struct btf *btf_vmlinux;
73+
74+static bool bpf_tcp_ca_is_valid_access(int off, int size,
75+ enum bpf_access_type type,
76+ const struct bpf_prog *prog,
77+ struct bpf_insn_access_aux *info)
78+{
79+ if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
80+ return false;
81+ if (type != BPF_READ)
82+ return false;
83+ if (off % size != 0)
84+ return false;
85+
86+ if (!btf_ctx_access(off, size, type, prog, info))
87+ return false;
88+
89+ if (info->reg_type == PTR_TO_BTF_ID && info->btf_id == sock_id)
90+ /* promote it to tcp_sock */
91+ info->btf_id = tcp_sock_id;
92+
93+ return true;
94+}
95+
96+static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log,
97+ const struct btf_type *t, int off,
98+ int size, enum bpf_access_type atype,
99+ u32 *next_btf_id)
100+{
101+ size_t end;
102+
103+ if (atype == BPF_READ)
104+ return btf_struct_access(log, t, off, size, atype, next_btf_id);
105+
106+ if (t != tcp_sock_type) {
107+ bpf_log(log, "only read is supported\n");
108+ return -EACCES;
109+ }
110+
111+ switch (off) {
112+ case bpf_ctx_range(struct inet_connection_sock, icsk_ca_priv):
113+ end = offsetofend(struct inet_connection_sock, icsk_ca_priv);
114+ break;
115+ case offsetof(struct inet_connection_sock, icsk_ack.pending):
116+ end = offsetofend(struct inet_connection_sock,
117+ icsk_ack.pending);
118+ break;
119+ case offsetof(struct tcp_sock, snd_cwnd):
120+ end = offsetofend(struct tcp_sock, snd_cwnd);
121+ break;
122+ case offsetof(struct tcp_sock, snd_cwnd_cnt):
123+ end = offsetofend(struct tcp_sock, snd_cwnd_cnt);
124+ break;
125+ case offsetof(struct tcp_sock, snd_ssthresh):
126+ end = offsetofend(struct tcp_sock, snd_ssthresh);
127+ break;
128+ case offsetof(struct tcp_sock, ecn_flags):
129+ end = offsetofend(struct tcp_sock, ecn_flags);
130+ break;
131+ default:
132+ bpf_log(log, "no write support to tcp_sock at off %d\n", off);
133+ return -EACCES;
134+ }
135+
136+ if (off + size > end) {
137+ bpf_log(log,
138+ "write access at off %d with size %d beyond the member of tcp_sock ended at %zu\n",
139+ off, size, end);
140+ return -EACCES;
141+ }
142+
143+ return NOT_INIT;
144+}
145+
146+static const struct bpf_func_proto *
147+bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id,
148+ const struct bpf_prog *prog)
149+{
150+ return bpf_base_func_proto(func_id);
151+}
152+
153+static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = {
154+ .get_func_proto = bpf_tcp_ca_get_func_proto,
155+ .is_valid_access = bpf_tcp_ca_is_valid_access,
156+ .btf_struct_access = bpf_tcp_ca_btf_struct_access,
157+};
158+
159+static int bpf_tcp_ca_init_member(const struct btf_type *t,
160+ const struct btf_member *member,
161+ void *kdata, const void *udata)
162+{
163+ const struct tcp_congestion_ops *utcp_ca;
164+ struct tcp_congestion_ops *tcp_ca;
165+ size_t tcp_ca_name_len;
166+ int prog_fd;
167+ u32 moff;
168+
169+ utcp_ca = (const struct tcp_congestion_ops *)udata;
170+ tcp_ca = (struct tcp_congestion_ops *)kdata;
171+
172+ moff = btf_member_bit_offset(t, member) / 8;
173+ switch (moff) {
174+ case offsetof(struct tcp_congestion_ops, flags):
175+ if (utcp_ca->flags & ~TCP_CONG_MASK)
176+ return -EINVAL;
177+ tcp_ca->flags = utcp_ca->flags;
178+ return 1;
179+ case offsetof(struct tcp_congestion_ops, name):
180+ tcp_ca_name_len = strnlen(utcp_ca->name, sizeof(utcp_ca->name));
181+ if (!tcp_ca_name_len ||
182+ tcp_ca_name_len == sizeof(utcp_ca->name))
183+ return -EINVAL;
184+ if (tcp_ca_find(utcp_ca->name))
185+ return -EEXIST;
186+ memcpy(tcp_ca->name, utcp_ca->name, sizeof(tcp_ca->name));
187+ return 1;
188+ }
189+
190+ if (!btf_type_resolve_func_ptr(btf_vmlinux, member->type, NULL))
191+ return 0;
192+
193+ /* Ensure bpf_prog is provided for compulsory func ptr */
194+ prog_fd = (int)(*(unsigned long *)(udata + moff));
195+ if (!prog_fd && !is_optional(moff) && !is_unsupported(moff))
196+ return -EINVAL;
197+
198+ return 0;
199+}
200+
201+static int bpf_tcp_ca_check_member(const struct btf_type *t,
202+ const struct btf_member *member)
203+{
204+ if (is_unsupported(btf_member_bit_offset(t, member) / 8))
205+ return -ENOTSUPP;
206+ return 0;
207+}
208+
209+static int bpf_tcp_ca_reg(void *kdata)
210+{
211+ return tcp_register_congestion_control(kdata);
212+}
213+
214+static void bpf_tcp_ca_unreg(void *kdata)
215+{
216+ tcp_unregister_congestion_control(kdata);
217+}
218+
219+/* Avoid sparse warning. It is only used in bpf_struct_ops.c. */
220+extern struct bpf_struct_ops bpf_tcp_congestion_ops;
221+
222+struct bpf_struct_ops bpf_tcp_congestion_ops = {
223+ .verifier_ops = &bpf_tcp_ca_verifier_ops,
224+ .reg = bpf_tcp_ca_reg,
225+ .unreg = bpf_tcp_ca_unreg,
226+ .check_member = bpf_tcp_ca_check_member,
227+ .init_member = bpf_tcp_ca_init_member,
228+ .init = bpf_tcp_ca_init,
229+ .name = "tcp_congestion_ops",
230+};
net/ipv4/tcp_cong.c+8−8
@@ -21,7 +21,7 @@ static DEFINE_SPINLOCK(tcp_cong_list_lock); · 20 unmodified lines
2121 static LIST_HEAD(tcp_cong_list);
2222
2323 /* Simple linear search, don't expect many entries! */
24−static struct tcp_congestion_ops *tcp_ca_find(const char *name)
24+struct tcp_congestion_ops *tcp_ca_find(const char *name)
2525 {
2626 struct tcp_congestion_ops *e;
2727
@@ -162,7 +162,7 @@ void tcp_assign_congestion_control(struct sock *sk) · 134 unmodified lines
162162
163163 rcu_read_lock();
164164 ca = rcu_dereference(net->ipv4.tcp_congestion_control);
165− if (unlikely(!try_module_get(ca->owner)))
165+ if (unlikely(!bpf_try_module_get(ca, ca->owner)))
166166 ca = &tcp_reno;
167167 icsk->icsk_ca_ops = ca;
168168 rcu_read_unlock();
@@ -208,7 +208,7 @@ void tcp_cleanup_congestion_control(struct sock *sk) · 39 unmodified lines
208208
209209 if (icsk->icsk_ca_ops->release)
210210 icsk->icsk_ca_ops->release(sk);
211− module_put(icsk->icsk_ca_ops->owner);
211+ bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
212212 }
213213
214214 /* Used by sysctl to change default congestion control */
@@ -222,12 +222,12 @@ int tcp_set_default_congestion_control(struct net *net, const char *name) · 7 unmodified lines
222222 ca = tcp_ca_find_autoload(net, name);
223223 if (!ca) {
224224 ret = -ENOENT;
225− } else if (!try_module_get(ca->owner)) {
225+ } else if (!bpf_try_module_get(ca, ca->owner)) {
226226 ret = -EBUSY;
227227 } else {
228228 prev = xchg(&net->ipv4.tcp_congestion_control, ca);
229229 if (prev)
230− module_put(prev->owner);
230+ bpf_module_put(prev, prev->owner);
231231
232232 ca->flags |= TCP_CONG_NON_RESTRICTED;
233233 ret = 0;
@@ -366,19 +366,19 @@ int tcp_set_congestion_control(struct sock *sk, const char *name, bool load, · 132 unmodified lines
366366 } else if (!load) {
367367 const struct tcp_congestion_ops *old_ca = icsk->icsk_ca_ops;
368368
369− if (try_module_get(ca->owner)) {
369+ if (bpf_try_module_get(ca, ca->owner)) {
370370 if (reinit) {
371371 tcp_reinit_congestion_control(sk, ca);
372372 } else {
373373 icsk->icsk_ca_ops = ca;
374− module_put(old_ca->owner);
374+ bpf_module_put(old_ca, old_ca->owner);
375375 }
376376 } else {
377377 err = -EBUSY;
378378 }
379379 } else if (!((ca->flags & TCP_CONG_NON_RESTRICTED) || cap_net_admin)) {
380380 err = -EPERM;
381− } else if (!try_module_get(ca->owner)) {
381+ } else if (!bpf_try_module_get(ca, ca->owner)) {
382382 err = -EBUSY;
383383 } else {
384384 tcp_reinit_congestion_control(sk, ca);
net/ipv4/tcp_ipv4.c+4−2
@@ -2678,7 +2678,8 @@ static void __net_exit tcp_sk_exit(struct net *net) · 2677 unmodified lines
26782678 int cpu;
26792679
26802680 if (net->ipv4.tcp_congestion_control)
2681− module_put(net->ipv4.tcp_congestion_control->owner);
2681+ bpf_module_put(net->ipv4.tcp_congestion_control,
2682+ net->ipv4.tcp_congestion_control->owner);
26822683
26832684 for_each_possible_cpu(cpu)
26842685 inet_ctl_sock_destroy(*per_cpu_ptr(net->ipv4.tcp_sk, cpu));
@@ -2785,7 +2786,8 @@ static int __net_init tcp_sk_init(struct net *net) · 100 unmodified lines
27852786
27862787 /* Reno is always built in */
27872788 if (!net_eq(net, &init_net) &&
2788− try_module_get(init_net.ipv4.tcp_congestion_control->owner))
2789+ bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
2790+ init_net.ipv4.tcp_congestion_control->owner))
27892791 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
27902792 else
27912793 net->ipv4.tcp_congestion_control = &tcp_reno;
net/ipv4/tcp_minisocks.c+2−2
@@ -414,7 +414,7 @@ void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst) · 413 unmodified lines
414414
415415 rcu_read_lock();
416416 ca = tcp_ca_find_key(ca_key);
417− if (likely(ca && try_module_get(ca->owner))) {
417+ if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
418418 icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
419419 icsk->icsk_ca_ops = ca;
420420 ca_got_dst = true;
@@ -425,7 +425,7 @@ void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst) · 4 unmodified lines
425425 /* If no valid choice made yet, assign current system default ca. */
426426 if (!ca_got_dst &&
427427 (!icsk->icsk_ca_setsockopt ||
428− !try_module_get(icsk->icsk_ca_ops->owner)))
428+ !bpf_try_module_get(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner)))
429429 tcp_assign_congestion_control(sk);
430430
431431 tcp_set_ca_state(sk, TCP_CA_Open);
net/ipv4/tcp_output.c+2−2
@@ -3368,8 +3368,8 @@ static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst) · 3367 unmodified lines
33683368
33693369 rcu_read_lock();
33703370 ca = tcp_ca_find_key(ca_key);
3371− if (likely(ca && try_module_get(ca->owner))) {
3372− module_put(icsk->icsk_ca_ops->owner);
3371+ if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
3372+ bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
33733373 icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
33743374 icsk->icsk_ca_ops = ca;
33753375 }