History

Merge branch 'udp_tunnel-gro-optimizations'

Paolo Abeni says: ==================== udp_tunnel: GRO optimizations The UDP tunnel GRO stage is source of measurable overhead for workload based on UDP-encapsulated traffic: each incoming packets requires a full UDP socket lookup and an indirect call. In the most common setups a single UDP tunnel device is used. In such case we can optimize both the lookup and the indirect call. Patch 1 tracks per netns the active UDP tunnels and replaces the socket lookup with a single destination port comparison when possible. Patch 2 tracks the different types of UDP tunnels and replaces the indirect call with a static one when there is a single UDP tunnel type active. I measure ~5% performance improvement in TCP over UDP tunnel stream tests on top of this series. v3: https://lore.kernel.org/netdev/cover.1741632298.git.pabeni@redhat.com/ v2: https://lore.kernel.org/netdev/cover.1741338765.git.pabeni@redhat.com/ v1: https://lore.kernel.org/netdev/cover.1741275846.git.pabeni@redhat.com/ ====================

Paolo Abeni committed 1 year ago

Commit aedfbe22 parents f5825e7 311b3659 files · +249 −2Browse files

Unified · Split

9 files
  1. include/linux/udp.h +16 −0
  2. include/net/netns/ipv4.h +11 −0
  3. include/net/udp.h +1 −0
  4. include/net/udp_tunnel.h +22 −0
  5. net/ipv4/udp.c +12 −1
  6. net/ipv4/udp_offload.c +166 −1
  7. net/ipv4/udp_tunnel_core.c +14 −0
  8. net/ipv6/udp.c +2 −0
  9. net/ipv6/udp_offload.c +5 −0
include/linux/udp.h+16−0
@@ -101,6 +101,13 @@ struct udp_sock { · 100 unmodified lines
101101
102102 /* Cache friendly copy of sk->sk_peek_off >= 0 */
103103 bool peeking_with_offset;
104+
105+ /*
106+ * Accounting for the tunnel GRO fastpath.
107+ * Unprotected by compilers guard, as it uses space available in
108+ * the last UDP socket cacheline.
109+ */
110+ struct hlist_node tunnel_list;
104111 };
105112
106113 #define udp_test_bit(nr, sk) \
@@ -219,4 +226,13 @@ static inline void udp_allow_gso(struct sock *sk) · 112 unmodified lines
219226
220227 #define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)
221228
229+static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)
230+{
231+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
232+ return rcu_dereference(net->ipv4.udp_tunnel_gro[is_ipv6].sk);
233+#else
234+ return NULL;
235+#endif
236+}
237+
222238 #endif /* _LINUX_UDP_H */
include/net/netns/ipv4.h+11−0
@@ -47,6 +47,11 @@ struct sysctl_fib_multipath_hash_seed { · 46 unmodified lines
4747 };
4848 #endif
4949
50+struct udp_tunnel_gro {
51+ struct sock __rcu *sk;
52+ struct hlist_head list;
53+};
54+
5055 struct netns_ipv4 {
5156 /* Cacheline organization can be found documented in
5257 * Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst.
@@ -85,6 +90,11 @@ struct netns_ipv4 { · 32 unmodified lines
8590 struct inet_timewait_death_row tcp_death_row;
8691 struct udp_table *udp_table;
8792
93+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
94+ /* Not in a pernet subsys because need to be available at GRO stage */
95+ struct udp_tunnel_gro udp_tunnel_gro[2];
96+#endif
97+
8898 #ifdef CONFIG_SYSCTL
8999 struct ctl_table_header *forw_hdr;
90100 struct ctl_table_header *frags_hdr;
@@ -277,4 +287,5 @@ struct netns_ipv4 { · 186 unmodified lines
277287 struct hlist_head *inet_addr_lst;
278288 struct delayed_work addr_chk_work;
279289 };
290+
280291 #endif
include/net/udp.h+1−0
@@ -290,6 +290,7 @@ static inline void udp_lib_init_sock(struct sock *sk) · 289 unmodified lines
290290 struct udp_sock *up = udp_sk(sk);
291291
292292 skb_queue_head_init(&up->reader_queue);
293+ INIT_HLIST_NODE(&up->tunnel_list);
293294 up->forward_threshold = sk->sk_rcvbuf >> 2;
294295 set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);
295296 }
include/net/udp_tunnel.h+22−0
@@ -203,6 +203,28 @@ static inline void udp_tunnel_encap_enable(struct sock *sk) · 202 unmodified lines
203203 udp_encap_enable();
204204 }
205205
206+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
207+void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add);
208+void udp_tunnel_update_gro_rcv(struct sock *sk, bool add);
209+#else
210+static inline void udp_tunnel_update_gro_lookup(struct net *net,
211+ struct sock *sk, bool add) {}
212+static inline void udp_tunnel_update_gro_rcv(struct sock *sk, bool add) {}
213+#endif
214+
215+static inline void udp_tunnel_cleanup_gro(struct sock *sk)
216+{
217+ struct udp_sock *up = udp_sk(sk);
218+ struct net *net = sock_net(sk);
219+
220+ udp_tunnel_update_gro_rcv(sk, false);
221+
222+ if (!up->tunnel_list.pprev)
223+ return;
224+
225+ udp_tunnel_update_gro_lookup(net, sk, false);
226+}
227+
206228 #define UDP_TUNNEL_NIC_MAX_TABLES 4
207229
208230 enum udp_tunnel_nic_info_flags {
net/ipv4/udp.c+12−1
@@ -2891,8 +2891,10 @@ void udp_destroy_sock(struct sock *sk) · 2890 unmodified lines
28912891 if (encap_destroy)
28922892 encap_destroy(sk);
28932893 }
2894− if (udp_test_bit(ENCAP_ENABLED, sk))
2894+ if (udp_test_bit(ENCAP_ENABLED, sk)) {
28952895 static_branch_dec(&udp_encap_needed_key);
2896+ udp_tunnel_cleanup_gro(sk);
2897+ }
28962898 }
28972899 }
28982900
@@ -3804,6 +3806,15 @@ fallback: · 905 unmodified lines
38043806
38053807 static int __net_init udp_pernet_init(struct net *net)
38063808 {
3809+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
3810+ int i;
3811+
3812+ /* No tunnel is configured */
3813+ for (i = 0; i < ARRAY_SIZE(net->ipv4.udp_tunnel_gro); ++i) {
3814+ INIT_HLIST_HEAD(&net->ipv4.udp_tunnel_gro[i].list);
3815+ RCU_INIT_POINTER(net->ipv4.udp_tunnel_gro[i].sk, NULL);
3816+ }
3817+#endif
38073818 udp_sysctl_init(net);
38083819 udp_set_table(net);
38093820
net/ipv4/udp_offload.c+166−1
@@ -12,6 +12,164 @@ · 11 unmodified lines
1212 #include <net/udp.h>
1313 #include <net/protocol.h>
1414 #include <net/inet_common.h>
15+#include <net/udp_tunnel.h>
16+
17+#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
18+
19+/*
20+ * Dummy GRO tunnel callback, exists mainly to avoid dangling/NULL
21+ * values for the udp tunnel static call.
22+ */
23+static struct sk_buff *dummy_gro_rcv(struct sock *sk,
24+ struct list_head *head,
25+ struct sk_buff *skb)
26+{
27+ NAPI_GRO_CB(skb)->flush = 1;
28+ return NULL;
29+}
30+
31+typedef struct sk_buff *(*udp_tunnel_gro_rcv_t)(struct sock *sk,
32+ struct list_head *head,
33+ struct sk_buff *skb);
34+
35+struct udp_tunnel_type_entry {
36+ udp_tunnel_gro_rcv_t gro_receive;
37+ refcount_t count;
38+};
39+
40+#define UDP_MAX_TUNNEL_TYPES (IS_ENABLED(CONFIG_GENEVE) + \
41+ IS_ENABLED(CONFIG_VXLAN) * 2 + \
42+ IS_ENABLED(CONFIG_NET_FOU) * 2)
43+
44+DEFINE_STATIC_CALL(udp_tunnel_gro_rcv, dummy_gro_rcv);
45+static DEFINE_STATIC_KEY_FALSE(udp_tunnel_static_call);
46+static struct mutex udp_tunnel_gro_type_lock;
47+static struct udp_tunnel_type_entry udp_tunnel_gro_types[UDP_MAX_TUNNEL_TYPES];
48+static unsigned int udp_tunnel_gro_type_nr;
49+static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
50+
51+void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
52+{
53+ bool is_ipv6 = sk->sk_family == AF_INET6;
54+ struct udp_sock *tup, *up = udp_sk(sk);
55+ struct udp_tunnel_gro *udp_tunnel_gro;
56+
57+ spin_lock(&udp_tunnel_gro_lock);
58+ udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];
59+ if (add)
60+ hlist_add_head(&up->tunnel_list, &udp_tunnel_gro->list);
61+ else
62+ hlist_del_init(&up->tunnel_list);
63+
64+ if (udp_tunnel_gro->list.first &&
65+ !udp_tunnel_gro->list.first->next) {
66+ tup = hlist_entry(udp_tunnel_gro->list.first, struct udp_sock,
67+ tunnel_list);
68+
69+ rcu_assign_pointer(udp_tunnel_gro->sk, (struct sock *)tup);
70+ } else {
71+ RCU_INIT_POINTER(udp_tunnel_gro->sk, NULL);
72+ }
73+
74+ spin_unlock(&udp_tunnel_gro_lock);
75+}
76+EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_lookup);
77+
78+void udp_tunnel_update_gro_rcv(struct sock *sk, bool add)
79+{
80+ struct udp_tunnel_type_entry *cur = NULL;
81+ struct udp_sock *up = udp_sk(sk);
82+ int i, old_gro_type_nr;
83+
84+ if (!up->gro_receive)
85+ return;
86+
87+ mutex_lock(&udp_tunnel_gro_type_lock);
88+ for (i = 0; i < udp_tunnel_gro_type_nr; i++)
89+ if (udp_tunnel_gro_types[i].gro_receive == up->gro_receive)
90+ cur = &udp_tunnel_gro_types[i];
91+
92+ old_gro_type_nr = udp_tunnel_gro_type_nr;
93+ if (add) {
94+ /*
95+ * Update the matching entry, if found, or add a new one
96+ * if needed
97+ */
98+ if (cur) {
99+ refcount_inc(&cur->count);
100+ goto out;
101+ }
102+
103+ if (unlikely(udp_tunnel_gro_type_nr == UDP_MAX_TUNNEL_TYPES)) {
104+ pr_err_once("Too many UDP tunnel types, please increase UDP_MAX_TUNNEL_TYPES\n");
105+ /* Ensure static call will never be enabled */
106+ udp_tunnel_gro_type_nr = UDP_MAX_TUNNEL_TYPES + 2;
107+ goto out;
108+ }
109+
110+ cur = &udp_tunnel_gro_types[udp_tunnel_gro_type_nr++];
111+ refcount_set(&cur->count, 1);
112+ cur->gro_receive = up->gro_receive;
113+ } else {
114+ /*
115+ * The stack cleanups only successfully added tunnel, the
116+ * lookup on removal should never fail.
117+ */
118+ if (WARN_ON_ONCE(!cur))
119+ goto out;
120+
121+ if (!refcount_dec_and_test(&cur->count))
122+ goto out;
123+
124+ /* avoid gaps, so that the enable tunnel has always id 0 */
125+ *cur = udp_tunnel_gro_types[--udp_tunnel_gro_type_nr];
126+ }
127+
128+ if (udp_tunnel_gro_type_nr == 1) {
129+ static_call_update(udp_tunnel_gro_rcv,
130+ udp_tunnel_gro_types[0].gro_receive);
131+ static_branch_enable(&udp_tunnel_static_call);
132+ } else if (old_gro_type_nr == 1) {
133+ static_branch_disable(&udp_tunnel_static_call);
134+ static_call_update(udp_tunnel_gro_rcv, dummy_gro_rcv);
135+ }
136+
137+out:
138+ mutex_unlock(&udp_tunnel_gro_type_lock);
139+}
140+EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_rcv);
141+
142+static void udp_tunnel_gro_init(void)
143+{
144+ mutex_init(&udp_tunnel_gro_type_lock);
145+}
146+
147+static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
148+ struct list_head *head,
149+ struct sk_buff *skb)
150+{
151+ if (static_branch_likely(&udp_tunnel_static_call)) {
152+ if (unlikely(gro_recursion_inc_test(skb))) {
153+ NAPI_GRO_CB(skb)->flush |= 1;
154+ return NULL;
155+ }
156+ return static_call(udp_tunnel_gro_rcv)(sk, head, skb);
157+ }
158+ return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
159+}
160+
161+#else
162+
163+static void udp_tunnel_gro_init(void) {}
164+
165+static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
166+ struct list_head *head,
167+ struct sk_buff *skb)
168+{
169+ return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
170+}
171+
172+#endif
15173
16174 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
17175 netdev_features_t features,
@@ -622,7 +780,7 @@ struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, · 604 unmodified lines
622780
623781 skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
624782 skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
625− pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
783+ pp = udp_tunnel_gro_rcv(sk, head, skb);
626784
627785 out:
628786 skb_gro_flush_final(skb, pp, flush);
@@ -635,8 +793,13 @@ static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport, · 6 unmodified lines
635793 {
636794 const struct iphdr *iph = skb_gro_network_header(skb);
637795 struct net *net = dev_net_rcu(skb->dev);
796+ struct sock *sk;
638797 int iif, sdif;
639798
799+ sk = udp_tunnel_sk(net, false);
800+ if (sk && dport == htons(sk->sk_num))
801+ return sk;
802+
640803 inet_get_iif_sdif(skb, &iif, &sdif);
641804
642805 return __udp4_lib_lookup(net, iph->saddr, sport,
@@ -767,5 +930,7 @@ int __init udpv4_offload_init(void) · 124 unmodified lines
767930 .gro_complete = udp4_gro_complete,
768931 },
769932 };
933+
934+ udp_tunnel_gro_init();
770935 return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP);
771936 }
net/ipv4/udp_tunnel_core.c+14−0
@@ -58,6 +58,15 @@ error: · 57 unmodified lines
5858 }
5959 EXPORT_SYMBOL(udp_sock_create4);
6060
61+static bool sk_saddr_any(struct sock *sk)
62+{
63+#if IS_ENABLED(CONFIG_IPV6)
64+ return ipv6_addr_any(&sk->sk_v6_rcv_saddr);
65+#else
66+ return !sk->sk_rcv_saddr;
67+#endif
68+}
69+
6170 void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
6271 struct udp_tunnel_sock_cfg *cfg)
6372 {
@@ -80,6 +89,11 @@ void setup_udp_tunnel_sock(struct net *net, struct socket *sock, · 16 unmodified lines
8089 udp_sk(sk)->gro_complete = cfg->gro_complete;
8190
8291 udp_tunnel_encap_enable(sk);
92+
93+ udp_tunnel_update_gro_rcv(sock->sk, true);
94+
95+ if (!sk->sk_dport && !sk->sk_bound_dev_if && sk_saddr_any(sock->sk))
96+ udp_tunnel_update_gro_lookup(net, sock->sk, true);
8397 }
8498 EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);
8599
net/ipv6/udp.c+2−0
@@ -46,6 +46,7 @@ · 45 unmodified lines
4646 #include <net/tcp_states.h>
4747 #include <net/ip6_checksum.h>
4848 #include <net/ip6_tunnel.h>
49+#include <net/udp_tunnel.h>
4950 #include <net/xfrm.h>
5051 #include <net/inet_hashtables.h>
5152 #include <net/inet6_hashtables.h>
@@ -1825,6 +1826,7 @@ void udpv6_destroy_sock(struct sock *sk) · 1773 unmodified lines
18251826 if (udp_test_bit(ENCAP_ENABLED, sk)) {
18261827 static_branch_dec(&udpv6_encap_needed_key);
18271828 udp_encap_disable();
1829+ udp_tunnel_cleanup_gro(sk);
18281830 }
18291831 }
18301832 }
net/ipv6/udp_offload.c+5−0
@@ -118,8 +118,13 @@ static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport, · 117 unmodified lines
118118 {
119119 const struct ipv6hdr *iph = skb_gro_network_header(skb);
120120 struct net *net = dev_net_rcu(skb->dev);
121+ struct sock *sk;
121122 int iif, sdif;
122123
124+ sk = udp_tunnel_sk(net, true);
125+ if (sk && dport == htons(sk->sk_num))
126+ return sk;
127+
123128 inet6_get_iif_sdif(skb, &iif, &sdif);
124129
125130 return __udp6_lib_lookup(net, &iph->saddr, sport,