History

vxlan: Remote checksum offload

Add support for remote checksum offload in VXLAN. This uses a reserved bit to indicate that RCO is being done, and uses the low order reserved eight bits of the VNI to hold the start and offset values in a compressed manner. Start is encoded in the low order seven bits of VNI. This is start >> 1 so that the checksum start offset is 0-254 using even values only. Checksum offset (transport checksum field) is indicated in the high order bit in the low order byte of the VNI. If the bit is set, the checksum field is for UDP (so offset = start + 6), else checksum field is for TCP (so offset = start + 16). Only TCP and UDP are supported in this implementation. Remote checksum offload for VXLAN is described in: https://tools.ietf.org/html/draft-herbert-vxlan-rco-00 Tested by running 200 TCP_STREAM connections with VXLAN (over IPv4). With UDP checksums and Remote Checksum Offload IPv4 Client 11.84% CPU utilization Server 12.96% CPU utilization 9197 Mbps IPv6 Client 12.46% CPU utilization Server 14.48% CPU utilization 8963 Mbps With UDP checksums, no remote checksum offload IPv4 Client 15.67% CPU utilization Server 14.83% CPU utilization 9094 Mbps IPv6 Client 16.21% CPU utilization Server 14.32% CPU utilization 9058 Mbps No UDP checksums IPv4 Client 15.03% CPU utilization Server 23.09% CPU utilization 9089 Mbps IPv6 Client 16.18% CPU utilization Server 26.57% CPU utilization 8954 Mbps

Tom Herbert committed 11 years ago

Commit dfd86451 parent a2b12f33 files · +198 −6Browse files

Unified · Split

3 files
  1. drivers/net/vxlan.c +185 −6
  2. include/net/vxlan.h +11 −0
  3. include/uapi/linux/if_link.h +2 −0
drivers/net/vxlan.c+185−6
@@ -539,6 +539,46 @@ static int vxlan_fdb_append(struct vxlan_fdb *f, · 538 unmodified lines
539 return 1;539 return 1;
540}540}
541 541
542static struct vxlanhdr *vxlan_gro_remcsum(struct sk_buff *skb,
543 unsigned int off,
544 struct vxlanhdr *vh, size_t hdrlen,
545 u32 data)
546{
547 size_t start, offset, plen;
548 __wsum delta;
549
550 if (skb->remcsum_offload)
551 return vh;
552
553 if (!NAPI_GRO_CB(skb)->csum_valid)
554 return NULL;
555
556 start = (data & VXLAN_RCO_MASK) << VXLAN_RCO_SHIFT;
557 offset = start + ((data & VXLAN_RCO_UDP) ?
558 offsetof(struct udphdr, check) :
559 offsetof(struct tcphdr, check));
560
561 plen = hdrlen + offset + sizeof(u16);
562
563 /* Pull checksum that will be written */
564 if (skb_gro_header_hard(skb, off + plen)) {
565 vh = skb_gro_header_slow(skb, off + plen, off);
566 if (!vh)
567 return NULL;
568 }
569
570 delta = remcsum_adjust((void *)vh + hdrlen,
571 NAPI_GRO_CB(skb)->csum, start, offset);
572
573 /* Adjust skb->csum since we changed the packet */
574 skb->csum = csum_add(skb->csum, delta);
575 NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);
576
577 skb->remcsum_offload = 1;
578
579 return vh;
580}
581
542static struct sk_buff **vxlan_gro_receive(struct sk_buff **head,582static struct sk_buff **vxlan_gro_receive(struct sk_buff **head,
543 struct sk_buff *skb,583 struct sk_buff *skb,
544 struct udp_offload *uoff)584 struct udp_offload *uoff)
@@ -547,6 +587,9 @@ static struct sk_buff **vxlan_gro_receive(struct sk_buff **head, · 2 unmodified lines
547 struct vxlanhdr *vh, *vh2;587 struct vxlanhdr *vh, *vh2;
548 unsigned int hlen, off_vx;588 unsigned int hlen, off_vx;
549 int flush = 1;589 int flush = 1;
590 struct vxlan_sock *vs = container_of(uoff, struct vxlan_sock,
591 udp_offloads);
592 u32 flags;
550 593
551 off_vx = skb_gro_offset(skb);594 off_vx = skb_gro_offset(skb);
552 hlen = off_vx + sizeof(*vh);595 hlen = off_vx + sizeof(*vh);
@@ -557,6 +600,19 @@ static struct sk_buff **vxlan_gro_receive(struct sk_buff **head, · 4 unmodified lines
557 goto out;600 goto out;
558 }601 }
559 602
603 skb_gro_pull(skb, sizeof(struct vxlanhdr)); /* pull vxlan header */
604 skb_gro_postpull_rcsum(skb, vh, sizeof(struct vxlanhdr));
605
606 flags = ntohl(vh->vx_flags);
607
608 if ((flags & VXLAN_HF_RCO) && (vs->flags & VXLAN_F_REMCSUM_RX)) {
609 vh = vxlan_gro_remcsum(skb, off_vx, vh, sizeof(struct vxlanhdr),
610 ntohl(vh->vx_vni));
611
612 if (!vh)
613 goto out;
614 }
615
560 flush = 0;616 flush = 0;
561 617
562 for (p = *head; p; p = p->next) {618 for (p = *head; p; p = p->next) {
@@ -570,8 +626,6 @@ static struct sk_buff **vxlan_gro_receive(struct sk_buff **head, · 7 unmodified lines
570 }626 }
571 }627 }
572 628
573 skb_gro_pull(skb, sizeof(struct vxlanhdr));
574 skb_gro_postpull_rcsum(skb, vh, sizeof(struct vxlanhdr));
575 pp = eth_gro_receive(head, skb);629 pp = eth_gro_receive(head, skb);
576 630
577out:631out:
@@ -1087,6 +1141,42 @@ static void vxlan_igmp_leave(struct work_struct *work) · 509 unmodified lines
1087 dev_put(vxlan->dev);1141 dev_put(vxlan->dev);
1088}1142}
1089 1143
1144static struct vxlanhdr *vxlan_remcsum(struct sk_buff *skb, struct vxlanhdr *vh,
1145 size_t hdrlen, u32 data)
1146{
1147 size_t start, offset, plen;
1148 __wsum delta;
1149
1150 if (skb->remcsum_offload) {
1151 /* Already processed in GRO path */
1152 skb->remcsum_offload = 0;
1153 return vh;
1154 }
1155
1156 start = (data & VXLAN_RCO_MASK) << VXLAN_RCO_SHIFT;
1157 offset = start + ((data & VXLAN_RCO_UDP) ?
1158 offsetof(struct udphdr, check) :
1159 offsetof(struct tcphdr, check));
1160
1161 plen = hdrlen + offset + sizeof(u16);
1162
1163 if (!pskb_may_pull(skb, plen))
1164 return NULL;
1165
1166 vh = (struct vxlanhdr *)(udp_hdr(skb) + 1);
1167
1168 if (unlikely(skb->ip_summed != CHECKSUM_COMPLETE))
1169 __skb_checksum_complete(skb);
1170
1171 delta = remcsum_adjust((void *)vh + hdrlen,
1172 skb->csum, start, offset);
1173
1174 /* Adjust skb->csum since we changed the packet */
1175 skb->csum = csum_add(skb->csum, delta);
1176
1177 return vh;
1178}
1179
1090/* Callback from net/ipv4/udp.c to receive packets */1180/* Callback from net/ipv4/udp.c to receive packets */
1091static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb)1181static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
1092{1182{
@@ -1111,12 +1201,22 @@ static int vxlan_udp_encap_recv(struct sock *sk, struct sk_buff *skb) · 18 unmodified lines
1111 1201
1112 if (iptunnel_pull_header(skb, VXLAN_HLEN, htons(ETH_P_TEB)))1202 if (iptunnel_pull_header(skb, VXLAN_HLEN, htons(ETH_P_TEB)))
1113 goto drop;1203 goto drop;
1204 vxh = (struct vxlanhdr *)(udp_hdr(skb) + 1);
1114 1205
1115 vs = rcu_dereference_sk_user_data(sk);1206 vs = rcu_dereference_sk_user_data(sk);
1116 if (!vs)1207 if (!vs)
1117 goto drop;1208 goto drop;
1118 1209
1119 if (flags || (vni & 0xff)) {1210 if ((flags & VXLAN_HF_RCO) && (vs->flags & VXLAN_F_REMCSUM_RX)) {
1211 vxh = vxlan_remcsum(skb, vxh, sizeof(struct vxlanhdr), vni);
1212 if (!vxh)
1213 goto drop;
1214
1215 flags &= ~VXLAN_HF_RCO;
1216 vni &= VXLAN_VID_MASK;
1217 }
1218
1219 if (flags || (vni & ~VXLAN_VID_MASK)) {
1120 /* If there are any unprocessed flags remaining treat1220 /* If there are any unprocessed flags remaining treat
1121 * this as a malformed packet. This behavior diverges from1221 * this as a malformed packet. This behavior diverges from
1122 * VXLAN RFC (RFC7348) which stipulates that bits in reserved1222 * VXLAN RFC (RFC7348) which stipulates that bits in reserved
@@ -1553,8 +1653,23 @@ static int vxlan6_xmit_skb(struct vxlan_sock *vs, · 430 unmodified lines
1553 int min_headroom;1653 int min_headroom;
1554 int err;1654 int err;
1555 bool udp_sum = !udp_get_no_check6_tx(vs->sock->sk);1655 bool udp_sum = !udp_get_no_check6_tx(vs->sock->sk);
1656 int type = udp_sum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
1657 u16 hdrlen = sizeof(struct vxlanhdr);
1658
1659 if ((vs->flags & VXLAN_F_REMCSUM_TX) &&
1660 skb->ip_summed == CHECKSUM_PARTIAL) {
1661 int csum_start = skb_checksum_start_offset(skb);
1662
1663 if (csum_start <= VXLAN_MAX_REMCSUM_START &&
1664 !(csum_start & VXLAN_RCO_SHIFT_MASK) &&
1665 (skb->csum_offset == offsetof(struct udphdr, check) ||
1666 skb->csum_offset == offsetof(struct tcphdr, check))) {
1667 udp_sum = false;
1668 type |= SKB_GSO_TUNNEL_REMCSUM;
1669 }
1670 }
1556 1671
1557 skb = udp_tunnel_handle_offloads(skb, udp_sum);1672 skb = iptunnel_handle_offloads(skb, udp_sum, type);
1558 if (IS_ERR(skb)) {1673 if (IS_ERR(skb)) {
1559 err = -EINVAL;1674 err = -EINVAL;
1560 goto err;1675 goto err;
@@ -1583,6 +1698,22 @@ static int vxlan6_xmit_skb(struct vxlan_sock *vs, · 22 unmodified lines
1583 vxh->vx_flags = htonl(VXLAN_HF_VNI);1698 vxh->vx_flags = htonl(VXLAN_HF_VNI);
1584 vxh->vx_vni = vni;1699 vxh->vx_vni = vni;
1585 1700
1701 if (type & SKB_GSO_TUNNEL_REMCSUM) {
1702 u32 data = (skb_checksum_start_offset(skb) - hdrlen) >>
1703 VXLAN_RCO_SHIFT;
1704
1705 if (skb->csum_offset == offsetof(struct udphdr, check))
1706 data |= VXLAN_RCO_UDP;
1707
1708 vxh->vx_vni |= htonl(data);
1709 vxh->vx_flags |= htonl(VXLAN_HF_RCO);
1710
1711 if (!skb_is_gso(skb)) {
1712 skb->ip_summed = CHECKSUM_NONE;
1713 skb->encapsulation = 0;
1714 }
1715 }
1716
1586 skb_set_inner_protocol(skb, htons(ETH_P_TEB));1717 skb_set_inner_protocol(skb, htons(ETH_P_TEB));
1587 1718
1588 udp_tunnel6_xmit_skb(vs->sock, dst, skb, dev, saddr, daddr, prio,1719 udp_tunnel6_xmit_skb(vs->sock, dst, skb, dev, saddr, daddr, prio,
@@ -1603,8 +1734,23 @@ int vxlan_xmit_skb(struct vxlan_sock *vs, · 14 unmodified lines
1603 int min_headroom;1734 int min_headroom;
1604 int err;1735 int err;
1605 bool udp_sum = !vs->sock->sk->sk_no_check_tx;1736 bool udp_sum = !vs->sock->sk->sk_no_check_tx;
1737 int type = udp_sum ? SKB_GSO_UDP_TUNNEL_CSUM : SKB_GSO_UDP_TUNNEL;
1738 u16 hdrlen = sizeof(struct vxlanhdr);
1739
1740 if ((vs->flags & VXLAN_F_REMCSUM_TX) &&
1741 skb->ip_summed == CHECKSUM_PARTIAL) {
1742 int csum_start = skb_checksum_start_offset(skb);
1743
1744 if (csum_start <= VXLAN_MAX_REMCSUM_START &&
1745 !(csum_start & VXLAN_RCO_SHIFT_MASK) &&
1746 (skb->csum_offset == offsetof(struct udphdr, check) ||
1747 skb->csum_offset == offsetof(struct tcphdr, check))) {
1748 udp_sum = false;
1749 type |= SKB_GSO_TUNNEL_REMCSUM;
1750 }
1751 }
1606 1752
1607 skb = udp_tunnel_handle_offloads(skb, udp_sum);1753 skb = iptunnel_handle_offloads(skb, udp_sum, type);
1608 if (IS_ERR(skb))1754 if (IS_ERR(skb))
1609 return PTR_ERR(skb);1755 return PTR_ERR(skb);
1610 1756
@@ -1627,6 +1773,22 @@ int vxlan_xmit_skb(struct vxlan_sock *vs, · 16 unmodified lines
1627 vxh->vx_flags = htonl(VXLAN_HF_VNI);1773 vxh->vx_flags = htonl(VXLAN_HF_VNI);
1628 vxh->vx_vni = vni;1774 vxh->vx_vni = vni;
1629 1775
1776 if (type & SKB_GSO_TUNNEL_REMCSUM) {
1777 u32 data = (skb_checksum_start_offset(skb) - hdrlen) >>
1778 VXLAN_RCO_SHIFT;
1779
1780 if (skb->csum_offset == offsetof(struct udphdr, check))
1781 data |= VXLAN_RCO_UDP;
1782
1783 vxh->vx_vni |= htonl(data);
1784 vxh->vx_flags |= htonl(VXLAN_HF_RCO);
1785
1786 if (!skb_is_gso(skb)) {
1787 skb->ip_summed = CHECKSUM_NONE;
1788 skb->encapsulation = 0;
1789 }
1790 }
1791
1630 skb_set_inner_protocol(skb, htons(ETH_P_TEB));1792 skb_set_inner_protocol(skb, htons(ETH_P_TEB));
1631 1793
1632 return udp_tunnel_xmit_skb(vs->sock, rt, skb, src, dst, tos,1794 return udp_tunnel_xmit_skb(vs->sock, rt, skb, src, dst, tos,
@@ -2218,6 +2380,8 @@ static const struct nla_policy vxlan_policy[IFLA_VXLAN_MAX + 1] = { · 585 unmodified lines
2218 [IFLA_VXLAN_UDP_CSUM] = { .type = NLA_U8 },2380 [IFLA_VXLAN_UDP_CSUM] = { .type = NLA_U8 },
2219 [IFLA_VXLAN_UDP_ZERO_CSUM6_TX] = { .type = NLA_U8 },2381 [IFLA_VXLAN_UDP_ZERO_CSUM6_TX] = { .type = NLA_U8 },
2220 [IFLA_VXLAN_UDP_ZERO_CSUM6_RX] = { .type = NLA_U8 },2382 [IFLA_VXLAN_UDP_ZERO_CSUM6_RX] = { .type = NLA_U8 },
2383 [IFLA_VXLAN_REMCSUM_TX] = { .type = NLA_U8 },
2384 [IFLA_VXLAN_REMCSUM_RX] = { .type = NLA_U8 },
2221};2385};
2222 2386
2223static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[])2387static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[])
@@ -2339,6 +2503,7 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, __be16 port, · 115 unmodified lines
2339 atomic_set(&vs->refcnt, 1);2503 atomic_set(&vs->refcnt, 1);
2340 vs->rcv = rcv;2504 vs->rcv = rcv;
2341 vs->data = data;2505 vs->data = data;
2506 vs->flags = flags;
2342 2507
2343 /* Initialize the vxlan udp offloads structure */2508 /* Initialize the vxlan udp offloads structure */
2344 vs->udp_offloads.port = port;2509 vs->udp_offloads.port = port;
@@ -2533,6 +2698,14 @@ static int vxlan_newlink(struct net *net, struct net_device *dev, · 188 unmodified lines
2533 nla_get_u8(data[IFLA_VXLAN_UDP_ZERO_CSUM6_RX]))2698 nla_get_u8(data[IFLA_VXLAN_UDP_ZERO_CSUM6_RX]))
2534 vxlan->flags |= VXLAN_F_UDP_ZERO_CSUM6_RX;2699 vxlan->flags |= VXLAN_F_UDP_ZERO_CSUM6_RX;
2535 2700
2701 if (data[IFLA_VXLAN_REMCSUM_TX] &&
2702 nla_get_u8(data[IFLA_VXLAN_REMCSUM_TX]))
2703 vxlan->flags |= VXLAN_F_REMCSUM_TX;
2704
2705 if (data[IFLA_VXLAN_REMCSUM_RX] &&
2706 nla_get_u8(data[IFLA_VXLAN_REMCSUM_RX]))
2707 vxlan->flags |= VXLAN_F_REMCSUM_RX;
2708
2536 if (vxlan_find_vni(net, vni, use_ipv6 ? AF_INET6 : AF_INET,2709 if (vxlan_find_vni(net, vni, use_ipv6 ? AF_INET6 : AF_INET,
2537 vxlan->dst_port)) {2710 vxlan->dst_port)) {
2538 pr_info("duplicate VNI %u\n", vni);2711 pr_info("duplicate VNI %u\n", vni);
@@ -2601,6 +2774,8 @@ static size_t vxlan_get_size(const struct net_device *dev) · 62 unmodified lines
2601 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_CSUM */2774 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_CSUM */
2602 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_TX */2775 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_TX */
2603 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_RX */2776 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_UDP_ZERO_CSUM6_RX */
2777 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_REMCSUM_TX */
2778 nla_total_size(sizeof(__u8)) + /* IFLA_VXLAN_REMCSUM_RX */
2604 0;2779 0;
2605}2780}
2606 2781
@@ -2666,7 +2841,11 @@ static int vxlan_fill_info(struct sk_buff *skb, const struct net_device *dev) · 59 unmodified lines
2666 nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_TX,2841 nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_TX,
2667 !!(vxlan->flags & VXLAN_F_UDP_ZERO_CSUM6_TX)) ||2842 !!(vxlan->flags & VXLAN_F_UDP_ZERO_CSUM6_TX)) ||
2668 nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_RX,2843 nla_put_u8(skb, IFLA_VXLAN_UDP_ZERO_CSUM6_RX,
2669 !!(vxlan->flags & VXLAN_F_UDP_ZERO_CSUM6_RX)))2844 !!(vxlan->flags & VXLAN_F_UDP_ZERO_CSUM6_RX)) ||
2845 nla_put_u8(skb, IFLA_VXLAN_REMCSUM_TX,
2846 !!(vxlan->flags & VXLAN_F_REMCSUM_TX)) ||
2847 nla_put_u8(skb, IFLA_VXLAN_REMCSUM_RX,
2848 !!(vxlan->flags & VXLAN_F_REMCSUM_RX)))
2670 goto nla_put_failure;2849 goto nla_put_failure;
2671 2850
2672 if (nla_put(skb, IFLA_VXLAN_PORT_RANGE, sizeof(ports), &ports))2851 if (nla_put(skb, IFLA_VXLAN_PORT_RANGE, sizeof(ports), &ports))
include/net/vxlan.h+11−0
@@ -19,6 +19,14 @@ struct vxlanhdr { · 18 unmodified lines
19 19
20/* VXLAN header flags. */20/* VXLAN header flags. */
21#define VXLAN_HF_VNI 0x0800000021#define VXLAN_HF_VNI 0x08000000
22#define VXLAN_HF_RCO 0x00200000
23
24/* Remote checksum offload header option */
25#define VXLAN_RCO_MASK 0x7f /* Last byte of vni field */
26#define VXLAN_RCO_UDP 0x80 /* Indicate UDP RCO (TCP when not set *) */
27#define VXLAN_RCO_SHIFT 1 /* Left shift of start */
28#define VXLAN_RCO_SHIFT_MASK ((1 << VXLAN_RCO_SHIFT) - 1)
29#define VXLAN_MAX_REMCSUM_START (VXLAN_RCO_MASK << VXLAN_RCO_SHIFT)
22 30
23#define VXLAN_N_VID (1u << 24)31#define VXLAN_N_VID (1u << 24)
24#define VXLAN_VID_MASK (VXLAN_N_VID - 1)32#define VXLAN_VID_MASK (VXLAN_N_VID - 1)
@@ -38,6 +46,7 @@ struct vxlan_sock { · 13 unmodified lines
38 struct hlist_head vni_list[VNI_HASH_SIZE];46 struct hlist_head vni_list[VNI_HASH_SIZE];
39 atomic_t refcnt;47 atomic_t refcnt;
40 struct udp_offload udp_offloads;48 struct udp_offload udp_offloads;
49 u32 flags;
41};50};
42 51
43#define VXLAN_F_LEARN 0x0152#define VXLAN_F_LEARN 0x01
@@ -49,6 +58,8 @@ struct vxlan_sock { · 5 unmodified lines
49#define VXLAN_F_UDP_CSUM 0x4058#define VXLAN_F_UDP_CSUM 0x40
50#define VXLAN_F_UDP_ZERO_CSUM6_TX 0x8059#define VXLAN_F_UDP_ZERO_CSUM6_TX 0x80
51#define VXLAN_F_UDP_ZERO_CSUM6_RX 0x10060#define VXLAN_F_UDP_ZERO_CSUM6_RX 0x100
61#define VXLAN_F_REMCSUM_TX 0x200
62#define VXLAN_F_REMCSUM_RX 0x400
52 63
53struct vxlan_sock *vxlan_sock_add(struct net *net, __be16 port,64struct vxlan_sock *vxlan_sock_add(struct net *net, __be16 port,
54 vxlan_rcv_t *rcv, void *data,65 vxlan_rcv_t *rcv, void *data,
include/uapi/linux/if_link.h+2−0
@@ -370,6 +370,8 @@ enum { · 369 unmodified lines
370 IFLA_VXLAN_UDP_CSUM,370 IFLA_VXLAN_UDP_CSUM,
371 IFLA_VXLAN_UDP_ZERO_CSUM6_TX,371 IFLA_VXLAN_UDP_ZERO_CSUM6_TX,
372 IFLA_VXLAN_UDP_ZERO_CSUM6_RX,372 IFLA_VXLAN_UDP_ZERO_CSUM6_RX,
373 IFLA_VXLAN_REMCSUM_TX,
374 IFLA_VXLAN_REMCSUM_RX,
373 __IFLA_VXLAN_MAX375 __IFLA_VXLAN_MAX
374};376};
375#define IFLA_VXLAN_MAX (__IFLA_VXLAN_MAX - 1)377#define IFLA_VXLAN_MAX (__IFLA_VXLAN_MAX - 1)