History

vxlan: Remote checksum offload

Add support for remote checksum offload in VXLAN. This uses a reserved bit to indicate that RCO is being done, and uses the low order reserved eight bits of the VNI to hold the start and offset values in a compressed manner. Start is encoded in the low order seven bits of VNI. This is start >> 1 so that the checksum start offset is 0-254 using even values only. Checksum offset (transport checksum field) is indicated in the high order bit in the low order byte of the VNI. If the bit is set, the checksum field is for UDP (so offset = start + 6), else checksum field is for TCP (so offset = start + 16). Only TCP and UDP are supported in this implementation. Remote checksum offload for VXLAN is described in: https://tools.ietf.org/html/draft-herbert-vxlan-rco-00 Tested by running 200 TCP_STREAM connections with VXLAN (over IPv4). With UDP checksums and Remote Checksum Offload IPv4 Client 11.84% CPU utilization Server 12.96% CPU utilization 9197 Mbps IPv6 Client 12.46% CPU utilization Server 14.48% CPU utilization 8963 Mbps With UDP checksums, no remote checksum offload IPv4 Client 15.67% CPU utilization Server 14.83% CPU utilization 9094 Mbps IPv6 Client 16.21% CPU utilization Server 14.32% CPU utilization 9058 Mbps No UDP checksums IPv4 Client 15.03% CPU utilization Server 23.09% CPU utilization 9089 Mbps IPv6 Client 16.18% CPU utilization Server 26.57% CPU utilization 8954 Mbps

Tom Herbert committed 11 years ago

Commit dfd86451 parent a2b12f33 files · +198 −6Browse files

Unified · Split

3 files
  1. drivers/net/vxlan.c +185 −6
  2. include/net/vxlan.h +11 −0
  3. include/uapi/linux/if_link.h +2 −0
drivers/net/vxlan.c+185−6
@@ -539,6 +539,46 @@ static int vxlan_fdb_append(struct vxlan_fdb *f, · 538 unmodified lines
539539 return 1;
540540 }
541541
542+static struct vxlanhdr *vxlan_gro_remcsum(struct sk_buff *skb,
543+ unsigned int off,
544+ struct vxlanhdr *vh, size_t hdrlen,
545+ u32 data)
546+{
547+ size_t start, offset, plen;
548+ __wsum delta;
549+
550+ if (skb->remcsum_offload)
551+ return vh;
552+
553+ if (!NAPI_GRO_CB(skb)->csum_valid)
554+ return NULL;
555+
556+ start = (data & VXLAN_RCO_MASK) << VXLAN_RCO_SHIFT;
557+ offset = start + ((data & VXLAN_RCO_UDP) ?
558+ offsetof(struct udphdr, check) :
559+ offsetof(struct tcphdr, check));
560+
561+ plen = hdrlen + offset + sizeof(u16);
562+
563+ /* Pull checksum that will be written */
564+ if (skb_gro_header_hard(skb, off + plen)) {
565+ vh = skb_gro_header_slow(skb, off + plen, off);
566+ if (!vh)
567+ return NULL;
568+ }
569+
570+ delta = remcsum_adjust((void *)vh + hdrlen,
571+ NAPI_GRO_CB(skb)->csum, start, offset);
572+
573+ /* Adjust skb->csum since we changed the packet */
574+ skb->csum = csum_add(skb->csum, delta);
575+ NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
576+
577+ skb->remcsum_offload = 1;
578+
579+ return vh;
580+}
581+
542582 static struct sk_buff **vxlan_gro_receive(struct sk_buff **head,
543583 struct sk_buff *skb,
544584 struct udp_offload *uoff)
@@ -547,6 +587,9 @@ static struct sk_buff **vxlan_gro_receive(struct sk_buff **head, · 2 unmodified lines
547587 struct vxlanhdr *vh, *vh2;
548588 unsigned int hlen, off_vx;
549589 int flush = 1;
590+ struct vxlan_sock *vs = container_of(uoff, struct vxlan_sock,
591+ udp_offloads);
592+ u32 flags;
550593
551594 off_vx = skb_gro_offset(skb);
552595 hlen = off_vx + sizeof(*vh);
@@ -557,6 +600,19 @@ static struct sk_buff **vxlan_gro_receive(struct sk_buff **head, · 4 unmodified lines
557600 goto out;
558601 }
559602
603+ skb_gro_pull(skb, sizeof(struct vxlanhdr)); /* pull vxlan header */
604+ skb_gro_postpull_rcsum(skb, vh, sizeof(struct vxlanhdr));
605+
606+ flags = ntohl(vh->vx_flags);
607+
608+ if ((flags & VXLAN_HF_RCO) && (vs->flags & VXLAN_F_REMCSUM_RX)) {
609+ vh = vxlan_gro_remcsum(skb, off_vx, vh, sizeof(struct vxlanhdr),
610+ ntohl(vh->vx_vni));
611+
612+ if (!vh)
613+ goto out;
614+ }
615+
560616 flush = 0;
561617
562618 for (p = *head; p; p = p->next) {
@@ -570,8 +626,6 @@ static struct sk_buff **vxlan_gro_receive(struct sk_buff **head, · 7 unmodified lines
570626 }
571627 }
572628
573− skb_gro_pull(skb, sizeof(struct vxlanhdr));
574− skb_gro_postpull_rcsum(skb, vh, sizeof(struct vxlanhdr));
575629 pp = eth_gro_receive(head, skb);
576630
577631 out:
@@ -1087,6 +1141,42 @@ static void vxlan_igmp_leave(struct work_struct *work) · 509 unmodified lines
10871141 dev_put(vxlan->dev);
10881142 }
10891143
1144+static struct vxlanhdr *vxlan_remcsum(struct sk_buff *skb, struct vxlanhdr *vh,
1145+ size_t hdrlen, u32 data)
1146+{
1147+ size_t start, offset, plen;
1148+ __wsum delta;
1149+
1150+ if (skb->remcsum_offload) {
1151+ /* Already processed in GRO path */
1152+ skb->remcsum_offload = 0;
1153+ return vh;
1154+ }
1155+
1156+ start = (data & VXLAN_RCO_MASK) << VXLAN_RCO_SHIFT;
1157+ offset = start + ((data & VXLAN_RCO_UDP) ?
1158+ offsetof(struct udphdr, check) :
1159+ offsetof(struct tcphdr, check));
1160+
1161+ plen = hdrlen + offset + sizeof(u16);
1162+
1163+ if (!pskb_may_pull(skb, plen))
1164+ return NULL;
1165+
1166+ vh = (struct vxlanhdr *)(udp_hdr(skb) + 1);
1167+
1168+ if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE))
1169+ __skb_checksum_complete(skb);
1170+
1171+ delta = remcsum_adjust((void *)vh + hdrlen,
1172+ skb->csum, start, offset);
1173+
1174+ /* Adjust skb->csum since we changed the packet */
1175+ skb->csum = csum_add(skb->csum, delta);
1176+
1177+ return vh;
1178+}
1179+
10901180 /* Callback from net/ipv4/udp.c to receive packets */
10911181 static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
10921182 {
@@ -1111,12 +1201,22 @@ static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb) · 18 unmodified lines
11111201
11121202 if (iptunnel_pull_header(skb, VXLAN_HLEN, htons(ETH_P_TEB)))
11131203 goto drop;
1204+ vxh = (struct vxlanhdr *)(udp_hdr(skb) + 1);
11141205
11151206 vs = rcu_dereference_sk_user_data(sk);
11161207 if (!vs)
11171208 goto drop;
11181209
1119− if (flags || (vni & 0xff)) {
1210+ if ((flags & VXLAN_HF_RCO) && (vs->flags & VXLAN_F_REMCSUM_RX)) {
1211+ vxh = vxlan_remcsum(skb, vxh, sizeof(struct vxlanhdr), vni);
1212+ if (!vxh)
1213+ goto drop;
1214+
1215+ flags &= ~VXLAN_HF_RCO;
1216+ vni &= VXLAN_VID_MASK;
1217+ }
1218+
1219+ if (flags || (vni & ~VXLAN_VID_MASK)) {
11201220 /* If there are any unprocessed flags remaining treat
11211221 * this as a malformed packet. This behavior diverges from
11221222 * VXLAN RFC (RFC7348) which stipulates that bits in reserved
@@ -1553,8 +1653,23 @@ static int vxlan6_xmit_skb(struct vxlan_sock *vs, · 430 unmodified lines
15531653 int min_headroom;
15541654 int err;
15551655 bool udp_sum = !udp_get_no_check6_tx(vs->sock->sk);
1656+ int type = udp_sum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
1657+ u16 hdrlen = sizeof(struct vxlanhdr);
1658+
1659+ if ((vs->flags & VXLAN_F_REMCSUM_TX) &&
1660+ skb->ip_summed == CHECKSUM_PARTIAL) {
1661+ int csum_start = skb_checksum_start_offset(skb);
1662+
1663+ if (csum_start <= VXLAN_MAX_REMCSUM_START &&
1664+ !(csum_start & VXLAN_RCO_SHIFT_MASK) &&
1665+ (skb->csum_offset == offsetof(struct udphdr, check) ||
1666+ skb->csum_offset == offsetof(struct tcphdr, check))) {
1667+ udp_sum = false;
1668+ type |= SKB_GSO_TUNNEL_REMCSUM;
1669+ }
1670+ }
15561671
1557− skb = udp_tunnel_handle_offloads(skb, udp_sum);
1672+ skb = iptunnel_handle_offloads(skb, udp_sum, type);
15581673 if (IS_ERR(skb)) {
15591674 err = -EINVAL;
15601675 goto err;
@@ -1583,6 +1698,22 @@ static int vxlan6_xmit_skb(struct vxlan_sock *vs, · 22 unmodified lines
15831698 vxh->vx_flags = htonl(VXLAN_HF_VNI);
15841699 vxh->vx_vni = vni;
15851700
1701+ if (type & SKB_GSO_TUNNEL_REMCSUM) {
1702+ u32 data = (skb_checksum_start_offset(skb) - hdrlen) >>
1703+ VXLAN_RCO_SHIFT;
1704+
1705+ if (skb->csum_offset == offsetof(struct udphdr, check))
1706+ data |= VXLAN_RCO_UDP;
1707+
1708+ vxh->vx_vni |= htonl(data);
1709+ vxh->vx_flags |= htonl(VXLAN_HF_RCO);
1710+
1711+ if (!skb_is_gso(skb)) {
1712+ skb->ip_summed = CHECKSUM_NONE;
1713+ skb->encapsulation = 0;
1714+ }
1715+ }
1716+
15861717 skb_set_inner_protocol(skb, htons(ETH_P_TEB));
15871718
15881719 udp_tunnel6_xmit_skb(vs->sock, dst, skb, dev, saddr, daddr, prio,
@@ -1603,8 +1734,23 @@ int vxlan_xmit_skb(struct vxlan_sock *vs, · 14 unmodified lines
16031734 int min_headroom;
16041735 int err;
16051736 bool udp_sum = !vs->sock->sk->sk_no_check_tx;
1737+ int type = udp_sum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
1738+ u16 hdrlen = sizeof(struct vxlanhdr);
1739+
1740+ if ((vs->flags & VXLAN_F_REMCSUM_TX) &&
1741+ skb->ip_summed == CHECKSUM_PARTIAL) {
1742+ int csum_start = skb_checksum_start_offset(skb);
1743+
1744+ if (csum_start <= VXLAN_MAX_REMCSUM_START &&
1745+ !(csum_start & VXLAN_RCO_SHIFT_MASK) &&
1746+ (skb->csum_offset == offsetof(struct udphdr, check) ||
1747+ skb->csum_offset == offsetof(struct tcphdr, check))) {
1748+ udp_sum = false;
1749+ type |= SKB_GSO_TUNNEL_REMCSUM;
1750+ }
1751+ }
16061752
1607− skb = udp_tunnel_handle_offloads(skb, udp_sum);
1753+ skb = iptunnel_handle_offloads(skb, udp_sum, type);
16081754 if (IS_ERR(skb))
16091755 return PTR_ERR(skb);
16101756
@@ -1627,6 +1773,22 @@ int vxlan_xmit_skb(struct vxlan_sock *vs, · 16 unmodified lines
16271773 vxh->vx_flags = htonl(VXLAN_HF_VNI);
16281774 vxh->vx_vni = vni;
16291775
1776+ if (type & SKB_GSO_TUNNEL_REMCSUM) {
1777+ u32 data = (skb_checksum_start_offset(skb) - hdrlen) >>
1778+ VXLAN_RCO_SHIFT;
1779+
1780+ if (skb->csum_offset == offsetof(struct udphdr, check))
1781+ data |= VXLAN_RCO_UDP;
1782+
1783+ vxh->vx_vni |= htonl(data);
1784+ vxh->vx_flags |= htonl(VXLAN_HF_RCO);
1785+
1786+ if (!skb_is_gso(skb)) {
1787+ skb->ip_summed = CHECKSUM_NONE;
1788+ skb->encapsulation = 0;
1789+ }
1790+ }
1791+
16301792 skb_set_inner_protocol(skb, htons(ETH_P_TEB));
16311793
16321794 return udp_tunnel_xmit_skb(vs->sock, rt, skb, src, dst, tos,
@@ -2218,6 +2380,8 @@ static const struct nla_policy vxlan_policy[IFLA_VXLAN_MAX + 1] = { · 585 unmodified lines
22182380 [IFLA_VXLAN_UDP_CSUM] = { .type = NLA_U8 },
22192381 [IFLA_VXLAN_UDP_ZERO_CSUM6_TX] = { .type = NLA_U8 },
22202382 [IFLA_VXLAN_UDP_ZERO_CSUM6_RX] = { .type = NLA_U8 },
2383+ [IFLA_VXLAN_REMCSUM_TX] = { .type = NLA_U8 },
2384+ [IFLA_VXLAN_REMCSUM_RX] = { .type = NLA_U8 },
22212385 };
22222386
22232387 static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[])
@@ -2339,6 +2503,7 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, __be16 port, · 115 unmodified lines
23392503 atomic_set(&vs->refcnt, 1);
23402504 vs->rcv = rcv;
23412505 vs->data = data;
2506+ vs->flags = flags;
23422507
23432508 /* Initialize the vxlan udp offloads structure */
23442509 vs->udp_offloads.port = port;
@@ -2533,6 +2698,14 @@ static int vxlan_newlink(struct net *net, struct net_device *dev, · 188 unmodified lines
25332698 nla_get_u8(data[IFLA_VXLAN_UDP_ZERO_CSUM6_RX]))
25342699 vxlan->flags |= VXLAN_F_UDP_ZERO_CSUM6_RX;
25352700
2701+ if (data[IFLA_VXLAN_REMCSUM_TX] &&
2702+ nla_get_u8(data[IFLA_VXLAN_REMCSUM_TX]))
2703+ vxlan->flags |= VXLAN_F_REMCSUM_TX;
2704+
2705+ if (data[IFLA_VXLAN_REMCSUM_RX] &&
2706+ nla_get_u8(data[IFLA_VXLAN_REMCSUM_RX]))
2707+ vxlan->flags |= VXLAN_F_REMCSUM_RX;
2708+
25362709 if (vxlan_find_vni(net, vni, use_ipv6 ? AF_INET6 : AF_INET,
25372710 vxlan->dst_port)) {
25382711 pr_info("duplicate VNI %u\n", vni);
@@ -2601,6 +2774,8 @@ static size_t vxlan_get_size(const struct net_device *dev) · 62 unmodified lines
26012774 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_CSUM */
26022775 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_TX */
26032776 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_RX */
2777+ nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_REMCSUM_TX */
2778+ nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_REMCSUM_RX */
26042779 0;
26052780 }
26062781
@@ -2666,7 +2841,11 @@ static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev) · 59 unmodified lines
26662841 nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_TX,
26672842 !!(vxlan->flags & VXLAN_F_UDP_ZERO_CSUM6_TX)) ||
26682843 nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_RX,
2669− !!(vxlan->flags & VXLAN_F_UDP_ZERO_CSUM6_RX)))
2844+ !!(vxlan->flags & VXLAN_F_UDP_ZERO_CSUM6_RX)) ||
2845+ nla_put_u8(skb, IFLA_VXLAN_REMCSUM_TX,
2846+ !!(vxlan->flags & VXLAN_F_REMCSUM_TX)) ||
2847+ nla_put_u8(skb, IFLA_VXLAN_REMCSUM_RX,
2848+ !!(vxlan->flags & VXLAN_F_REMCSUM_RX)))
26702849 goto nla_put_failure;
26712850
26722851 if (nla_put(skb, IFLA_VXLAN_PORT_RANGE, sizeof(ports), &ports))
include/net/vxlan.h+11−0
@@ -19,6 +19,14 @@ struct vxlanhdr { · 18 unmodified lines
1919
2020 /* VXLAN header flags. */
2121 #define VXLAN_HF_VNI 0x08000000
22+#define VXLAN_HF_RCO 0x00200000
23+
24+/* Remote checksum offload header option */
25+#define VXLAN_RCO_MASK 0x7f /* Last byte of vni field */
26+#define VXLAN_RCO_UDP 0x80 /* Indicate UDP RCO (TCP when not set *) */
27+#define VXLAN_RCO_SHIFT 1 /* Left shift of start */
28+#define VXLAN_RCO_SHIFT_MASK ((1 << VXLAN_RCO_SHIFT) - 1)
29+#define VXLAN_MAX_REMCSUM_START (VXLAN_RCO_MASK << VXLAN_RCO_SHIFT)
2230
2331 #define VXLAN_N_VID (1u << 24)
2432 #define VXLAN_VID_MASK (VXLAN_N_VID - 1)
@@ -38,6 +46,7 @@ struct vxlan_sock { · 13 unmodified lines
3846 struct hlist_head vni_list[VNI_HASH_SIZE];
3947 atomic_t refcnt;
4048 struct udp_offload udp_offloads;
49+ u32 flags;
4150 };
4251
4352 #define VXLAN_F_LEARN 0x01
@@ -49,6 +58,8 @@ struct vxlan_sock { · 5 unmodified lines
4958 #define VXLAN_F_UDP_CSUM 0x40
5059 #define VXLAN_F_UDP_ZERO_CSUM6_TX 0x80
5160 #define VXLAN_F_UDP_ZERO_CSUM6_RX 0x100
61+#define VXLAN_F_REMCSUM_TX 0x200
62+#define VXLAN_F_REMCSUM_RX 0x400
5263
5364 struct vxlan_sock *vxlan_sock_add(struct net *net, __be16 port,
5465 vxlan_rcv_t *rcv, void *data,
include/uapi/linux/if_link.h+2−0
@@ -370,6 +370,8 @@ enum { · 369 unmodified lines
370370 IFLA_VXLAN_UDP_CSUM,
371371 IFLA_VXLAN_UDP_ZERO_CSUM6_TX,
372372 IFLA_VXLAN_UDP_ZERO_CSUM6_RX,
373+ IFLA_VXLAN_REMCSUM_TX,
374+ IFLA_VXLAN_REMCSUM_RX,
373375 __IFLA_VXLAN_MAX
374376 };
375377 #define IFLA_VXLAN_MAX (__IFLA_VXLAN_MAX - 1)