History

Merge branch 'udp_tunnel-gro-optimizations'

Paolo Abeni says: ==================== udp_tunnel: GRO optimizations The UDP tunnel GRO stage is source of measurable overhead for workload based on UDP-encapsulated traffic: each incoming packets requires a full UDP socket lookup and an indirect call. In the most common setups a single UDP tunnel device is used. In such case we can optimize both the lookup and the indirect call. Patch 1 tracks per netns the active UDP tunnels and replaces the socket lookup with a single destination port comparison when possible. Patch 2 tracks the different types of UDP tunnels and replaces the indirect call with a static one when there is a single UDP tunnel type active. I measure ~5% performance improvement in TCP over UDP tunnel stream tests on top of this series. v3: https://lore.kernel.org/netdev/cover.1741632298.git.pabeni@redhat.com/ v2: https://lore.kernel.org/netdev/cover.1741338765.git.pabeni@redhat.com/ v1: https://lore.kernel.org/netdev/cover.1741275846.git.pabeni@redhat.com/ ====================

Paolo Abeni committed 1 year ago

Commit aedfbe22 parents f5825e7 311b3659 files · +249 −2Browse files

Unified · Split

9 files
  1. include/linux/udp.h +16 −0
  2. include/net/netns/ipv4.h +11 −0
  3. include/net/udp.h +1 −0
  4. include/net/udp_tunnel.h +22 −0
  5. net/ipv4/udp.c +12 −1
  6. net/ipv4/udp_offload.c +166 −1
  7. net/ipv4/udp_tunnel_core.c +14 −0
  8. net/ipv6/udp.c +2 −0
  9. net/ipv6/udp_offload.c +5 −0
include/linux/udp.h+16−0
@@ -101,6 +101,13 @@ struct udp_sock { · 100 unmodified lines
101 101
102 /* Cache friendly copy of sk->sk_peek_off >= 0 */102 /* Cache friendly copy of sk->sk_peek_off >= 0 */
103 bool peeking_with_offset;103 bool peeking_with_offset;
104
105 /*
106 * Accounting for the tunnel GRO fastpath.
107 * Unprotected by compilers guard, as it uses space available in
108 * the last UDP socket cacheline.
109 */
110 struct hlist_node tunnel_list;
104};111};
105 112
106#define udp_test_bit(nr, sk) \113#define udp_test_bit(nr, sk) \
@@ -219,4 +226,13 @@ static inline void udp_allow_gso(struct sock *sk) · 112 unmodified lines
219 226
220#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)227#define IS_UDPLITE(__sk) (__sk->sk_protocol == IPPROTO_UDPLITE)
221 228
229static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6)
230{
231#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
232 return rcu_dereference(net->ipv4.udp_tunnel_gro[is_ipv6].sk);
233#else
234 return NULL;
235#endif
236}
237
222#endif /* _LINUX_UDP_H */238#endif /* _LINUX_UDP_H */
include/net/netns/ipv4.h+11−0
@@ -47,6 +47,11 @@ struct sysctl_fib_multipath_hash_seed { · 46 unmodified lines
47};47};
48#endif48#endif
49 49
50struct udp_tunnel_gro {
51 struct sock __rcu *sk;
52 struct hlist_head list;
53};
54
50struct netns_ipv4 {55struct netns_ipv4 {
51 /* Cacheline organization can be found documented in56 /* Cacheline organization can be found documented in
52 * Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst.57 * Documentation/networking/net_cachelines/netns_ipv4_sysctl.rst.
@@ -85,6 +90,11 @@ struct netns_ipv4 { · 32 unmodified lines
85 struct inet_timewait_death_row tcp_death_row;90 struct inet_timewait_death_row tcp_death_row;
86 struct udp_table *udp_table;91 struct udp_table *udp_table;
87 92
93#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
94 /* Not in a pernet subsys because need to be available at GRO stage */
95 struct udp_tunnel_gro udp_tunnel_gro[2];
96#endif
97
88#ifdef CONFIG_SYSCTL98#ifdef CONFIG_SYSCTL
89 struct ctl_table_header *forw_hdr;99 struct ctl_table_header *forw_hdr;
90 struct ctl_table_header *frags_hdr;100 struct ctl_table_header *frags_hdr;
@@ -277,4 +287,5 @@ struct netns_ipv4 { · 186 unmodified lines
277 struct hlist_head *inet_addr_lst;287 struct hlist_head *inet_addr_lst;
278 struct delayed_work addr_chk_work;288 struct delayed_work addr_chk_work;
279};289};
290
280#endif291#endif
include/net/udp.h+1−0
@@ -290,6 +290,7 @@ static inline void udp_lib_init_sock(struct sock *sk) · 289 unmodified lines
290 struct udp_sock *up = udp_sk(sk);290 struct udp_sock *up = udp_sk(sk);
291 291
292 skb_queue_head_init(&up->reader_queue);292 skb_queue_head_init(&up->reader_queue);
293 INIT_HLIST_NODE(&up->tunnel_list);
293 up->forward_threshold = sk->sk_rcvbuf >> 2;294 up->forward_threshold = sk->sk_rcvbuf >> 2;
294 set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);295 set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);
295}296}
include/net/udp_tunnel.h+22−0
@@ -203,6 +203,28 @@ static inline void udp_tunnel_encap_enable(struct sock *sk) · 202 unmodified lines
203 udp_encap_enable();203 udp_encap_enable();
204}204}
205 205
206#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
207void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add);
208void udp_tunnel_update_gro_rcv(struct sock *sk, bool add);
209#else
210static inline void udp_tunnel_update_gro_lookup(struct net *net,
211 struct sock *sk, bool add) {}
212static inline void udp_tunnel_update_gro_rcv(struct sock *sk, bool add) {}
213#endif
214
215static inline void udp_tunnel_cleanup_gro(struct sock *sk)
216{
217 struct udp_sock *up = udp_sk(sk);
218 struct net *net = sock_net(sk);
219
220 udp_tunnel_update_gro_rcv(sk, false);
221
222 if (!up->tunnel_list.pprev)
223 return;
224
225 udp_tunnel_update_gro_lookup(net, sk, false);
226}
227
206#define UDP_TUNNEL_NIC_MAX_TABLES 4228#define UDP_TUNNEL_NIC_MAX_TABLES 4
207 229
208enum udp_tunnel_nic_info_flags {230enum udp_tunnel_nic_info_flags {
net/ipv4/udp.c+12−1
@@ -2891,8 +2891,10 @@ void udp_destroy_sock(struct sock *sk) · 2890 unmodified lines
2891 if (encap_destroy)2891 if (encap_destroy)
2892 encap_destroy(sk);2892 encap_destroy(sk);
2893 }2893 }
2894 if (udp_test_bit(ENCAP_ENABLED, sk))2894 if (udp_test_bit(ENCAP_ENABLED, sk)) {
2895 static_branch_dec(&udp_encap_needed_key);2895 static_branch_dec(&udp_encap_needed_key);
2896 udp_tunnel_cleanup_gro(sk);
2897 }
2896 }2898 }
2897}2899}
2898 2900
@@ -3804,6 +3806,15 @@ fallback: · 905 unmodified lines
3804 3806
3805static int __net_init udp_pernet_init(struct net *net)3807static int __net_init udp_pernet_init(struct net *net)
3806{3808{
3809#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
3810 int i;
3811
3812 /* No tunnel is configured */
3813 for (i = 0; i < ARRAY_SIZE(net->ipv4.udp_tunnel_gro); ++i) {
3814 INIT_HLIST_HEAD(&net->ipv4.udp_tunnel_gro[i].list);
3815 RCU_INIT_POINTER(net->ipv4.udp_tunnel_gro[i].sk, NULL);
3816 }
3817#endif
3807 udp_sysctl_init(net);3818 udp_sysctl_init(net);
3808 udp_set_table(net);3819 udp_set_table(net);
3809 3820
net/ipv4/udp_offload.c+166−1
@@ -12,6 +12,164 @@ · 11 unmodified lines
12#include <net/udp.h>12#include <net/udp.h>
13#include <net/protocol.h>13#include <net/protocol.h>
14#include <net/inet_common.h>14#include <net/inet_common.h>
15#include <net/udp_tunnel.h>
16
17#if IS_ENABLED(CONFIG_NET_UDP_TUNNEL)
18
19/*
20 * Dummy GRO tunnel callback, exists mainly to avoid dangling/NULL
21 * values for the udp tunnel static call.
22 */
23static struct sk_buff *dummy_gro_rcv(struct sock *sk,
24 struct list_head *head,
25 struct sk_buff *skb)
26{
27 NAPI_GRO_CB(skb)->flush = 1;
28 return NULL;
29}
30
31typedef struct sk_buff *(*udp_tunnel_gro_rcv_t)(struct sock *sk,
32 struct list_head *head,
33 struct sk_buff *skb);
34
35struct udp_tunnel_type_entry {
36 udp_tunnel_gro_rcv_t gro_receive;
37 refcount_t count;
38};
39
40#define UDP_MAX_TUNNEL_TYPES (IS_ENABLED(CONFIG_GENEVE) + \
41 IS_ENABLED(CONFIG_VXLAN) * 2 + \
42 IS_ENABLED(CONFIG_NET_FOU) * 2)
43
44DEFINE_STATIC_CALL(udp_tunnel_gro_rcv, dummy_gro_rcv);
45static DEFINE_STATIC_KEY_FALSE(udp_tunnel_static_call);
46static struct mutex udp_tunnel_gro_type_lock;
47static struct udp_tunnel_type_entry udp_tunnel_gro_types[UDP_MAX_TUNNEL_TYPES];
48static unsigned int udp_tunnel_gro_type_nr;
49static DEFINE_SPINLOCK(udp_tunnel_gro_lock);
50
51void udp_tunnel_update_gro_lookup(struct net *net, struct sock *sk, bool add)
52{
53 bool is_ipv6 = sk->sk_family == AF_INET6;
54 struct udp_sock *tup, *up = udp_sk(sk);
55 struct udp_tunnel_gro *udp_tunnel_gro;
56
57 spin_lock(&udp_tunnel_gro_lock);
58 udp_tunnel_gro = &net->ipv4.udp_tunnel_gro[is_ipv6];
59 if (add)
60 hlist_add_head(&up->tunnel_list, &udp_tunnel_gro->list);
61 else
62 hlist_del_init(&up->tunnel_list);
63
64 if (udp_tunnel_gro->list.first &&
65 !udp_tunnel_gro->list.first->next) {
66 tup = hlist_entry(udp_tunnel_gro->list.first, struct udp_sock,
67 tunnel_list);
68
69 rcu_assign_pointer(udp_tunnel_gro->sk, (struct sock *)tup);
70 } else {
71 RCU_INIT_POINTER(udp_tunnel_gro->sk, NULL);
72 }
73
74 spin_unlock(&udp_tunnel_gro_lock);
75}
76EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_lookup);
77
78void udp_tunnel_update_gro_rcv(struct sock *sk, bool add)
79{
80 struct udp_tunnel_type_entry *cur = NULL;
81 struct udp_sock *up = udp_sk(sk);
82 int i, old_gro_type_nr;
83
84 if (!up->gro_receive)
85 return;
86
87 mutex_lock(&udp_tunnel_gro_type_lock);
88 for (i = 0; i < udp_tunnel_gro_type_nr; i++)
89 if (udp_tunnel_gro_types[i].gro_receive == up->gro_receive)
90 cur = &udp_tunnel_gro_types[i];
91
92 old_gro_type_nr = udp_tunnel_gro_type_nr;
93 if (add) {
94 /*
95 * Update the matching entry, if found, or add a new one
96 * if needed
97 */
98 if (cur) {
99 refcount_inc(&cur->count);
100 goto out;
101 }
102
103 if (unlikely(udp_tunnel_gro_type_nr == UDP_MAX_TUNNEL_TYPES)) {
104 pr_err_once("Too many UDP tunnel types, please increase UDP_MAX_TUNNEL_TYPES\n");
105 /* Ensure static call will never be enabled */
106 udp_tunnel_gro_type_nr = UDP_MAX_TUNNEL_TYPES + 2;
107 goto out;
108 }
109
110 cur = &udp_tunnel_gro_types[udp_tunnel_gro_type_nr++];
111 refcount_set(&cur->count, 1);
112 cur->gro_receive = up->gro_receive;
113 } else {
114 /*
115 * The stack cleanups only successfully added tunnel, the
116 * lookup on removal should never fail.
117 */
118 if (WARN_ON_ONCE(!cur))
119 goto out;
120
121 if (!refcount_dec_and_test(&cur->count))
122 goto out;
123
124 /* avoid gaps, so that the enable tunnel has always id 0 */
125 *cur = udp_tunnel_gro_types[--udp_tunnel_gro_type_nr];
126 }
127
128 if (udp_tunnel_gro_type_nr == 1) {
129 static_call_update(udp_tunnel_gro_rcv,
130 udp_tunnel_gro_types[0].gro_receive);
131 static_branch_enable(&udp_tunnel_static_call);
132 } else if (old_gro_type_nr == 1) {
133 static_branch_disable(&udp_tunnel_static_call);
134 static_call_update(udp_tunnel_gro_rcv, dummy_gro_rcv);
135 }
136
137out:
138 mutex_unlock(&udp_tunnel_gro_type_lock);
139}
140EXPORT_SYMBOL_GPL(udp_tunnel_update_gro_rcv);
141
142static void udp_tunnel_gro_init(void)
143{
144 mutex_init(&udp_tunnel_gro_type_lock);
145}
146
147static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
148 struct list_head *head,
149 struct sk_buff *skb)
150{
151 if (static_branch_likely(&udp_tunnel_static_call)) {
152 if (unlikely(gro_recursion_inc_test(skb))) {
153 NAPI_GRO_CB(skb)->flush |= 1;
154 return NULL;
155 }
156 return static_call(udp_tunnel_gro_rcv)(sk, head, skb);
157 }
158 return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
159}
160
161#else
162
163static void udp_tunnel_gro_init(void) {}
164
165static struct sk_buff *udp_tunnel_gro_rcv(struct sock *sk,
166 struct list_head *head,
167 struct sk_buff *skb)
168{
169 return call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);
170}
171
172#endif
15 173
16static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,174static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,
17 netdev_features_t features,175 netdev_features_t features,
@@ -622,7 +780,7 @@ struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, · 604 unmodified lines
622 780
623 skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */781 skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
624 skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));782 skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
625 pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);783 pp = udp_tunnel_gro_rcv(sk, head, skb);
626 784
627out:785out:
628 skb_gro_flush_final(skb, pp, flush);786 skb_gro_flush_final(skb, pp, flush);
@@ -635,8 +793,13 @@ static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport, · 6 unmodified lines
635{793{
636 const struct iphdr *iph = skb_gro_network_header(skb);794 const struct iphdr *iph = skb_gro_network_header(skb);
637 struct net *net = dev_net_rcu(skb->dev);795 struct net *net = dev_net_rcu(skb->dev);
796 struct sock *sk;
638 int iif, sdif;797 int iif, sdif;
639 798
799 sk = udp_tunnel_sk(net, false);
800 if (sk && dport == htons(sk->sk_num))
801 return sk;
802
640 inet_get_iif_sdif(skb, &iif, &sdif);803 inet_get_iif_sdif(skb, &iif, &sdif);
641 804
642 return __udp4_lib_lookup(net, iph->saddr, sport,805 return __udp4_lib_lookup(net, iph->saddr, sport,
@@ -767,5 +930,7 @@ int __init udpv4_offload_init(void) · 124 unmodified lines
767 .gro_complete = udp4_gro_complete,930 .gro_complete = udp4_gro_complete,
768 },931 },
769 };932 };
933
934 udp_tunnel_gro_init();
770 return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP);935 return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP);
771}936}
net/ipv4/udp_tunnel_core.c+14−0
@@ -58,6 +58,15 @@ error: · 57 unmodified lines
58}58}
59EXPORT_SYMBOL(udp_sock_create4);59EXPORT_SYMBOL(udp_sock_create4);
60 60
61static bool sk_saddr_any(struct sock *sk)
62{
63#if IS_ENABLED(CONFIG_IPV6)
64 return ipv6_addr_any(&sk->sk_v6_rcv_saddr);
65#else
66 return !sk->sk_rcv_saddr;
67#endif
68}
69
61void setup_udp_tunnel_sock(struct net *net, struct socket *sock,70void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
62 struct udp_tunnel_sock_cfg *cfg)71 struct udp_tunnel_sock_cfg *cfg)
63{72{
@@ -80,6 +89,11 @@ void setup_udp_tunnel_sock(struct net *net, struct socket *sock, · 16 unmodified lines
80 udp_sk(sk)->gro_complete = cfg->gro_complete;89 udp_sk(sk)->gro_complete = cfg->gro_complete;
81 90
82 udp_tunnel_encap_enable(sk);91 udp_tunnel_encap_enable(sk);
92
93 udp_tunnel_update_gro_rcv(sock->sk, true);
94
95 if (!sk->sk_dport && !sk->sk_bound_dev_if && sk_saddr_any(sock->sk))
96 udp_tunnel_update_gro_lookup(net, sock->sk, true);
83}97}
84EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);98EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);
85 99
net/ipv6/udp.c+2−0
@@ -46,6 +46,7 @@ · 45 unmodified lines
46#include <net/tcp_states.h>46#include <net/tcp_states.h>
47#include <net/ip6_checksum.h>47#include <net/ip6_checksum.h>
48#include <net/ip6_tunnel.h>48#include <net/ip6_tunnel.h>
49#include <net/udp_tunnel.h>
49#include <net/xfrm.h>50#include <net/xfrm.h>
50#include <net/inet_hashtables.h>51#include <net/inet_hashtables.h>
51#include <net/inet6_hashtables.h>52#include <net/inet6_hashtables.h>
@@ -1825,6 +1826,7 @@ void udpv6_destroy_sock(struct sock *sk) · 1773 unmodified lines
1825 if (udp_test_bit(ENCAP_ENABLED, sk)) {1826 if (udp_test_bit(ENCAP_ENABLED, sk)) {
1826 static_branch_dec(&udpv6_encap_needed_key);1827 static_branch_dec(&udpv6_encap_needed_key);
1827 udp_encap_disable();1828 udp_encap_disable();
1829 udp_tunnel_cleanup_gro(sk);
1828 }1830 }
1829 }1831 }
1830}1832}
net/ipv6/udp_offload.c+5−0
@@ -118,8 +118,13 @@ static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport, · 117 unmodified lines
118{118{
119 const struct ipv6hdr *iph = skb_gro_network_header(skb);119 const struct ipv6hdr *iph = skb_gro_network_header(skb);
120 struct net *net = dev_net_rcu(skb->dev);120 struct net *net = dev_net_rcu(skb->dev);
121 struct sock *sk;
121 int iif, sdif;122 int iif, sdif;
122 123
124 sk = udp_tunnel_sk(net, true);
125 if (sk && dport == htons(sk->sk_num))
126 return sk;
127
123 inet6_get_iif_sdif(skb, &iif, &sdif);128 inet6_get_iif_sdif(skb, &iif, &sdif);
124 129
125 return __udp6_lib_lookup(net, &iph->saddr, sport,130 return __udp6_lib_lookup(net, &iph->saddr, sport,