IPv6 needs a cookie in dst_check() call. We need to add rx_dst_cookie and provide a family independent sk_rx_dst_set(sk, skb) method to properly support IPv6 TCP early demux.
Unified · Split
| @@ -369,6 +369,7 @@ struct ipv6_pinfo { · 368 unmodified lines | ||
| 369 | 369 | __u8 rcv_tclass; |
| 370 | 370 | |
| 371 | 371 | __u32 dst_cookie; |
| 372 | + __u32 rx_dst_cookie; | |
| 372 | 373 | |
| 373 | 374 | struct ipv6_mc_socklist __rcu *ipv6_mc_list; |
| 374 | 375 | struct ipv6_ac_socklist *ipv6_ac_list; |
| @@ -39,6 +39,7 @@ struct inet_connection_sock_af_ops { · 38 unmodified lines | ||
| 39 | 39 | int (*queue_xmit)(struct sk_buff *skb, struct flowi *fl); |
| 40 | 40 | void (*send_check)(struct sock *sk, struct sk_buff *skb); |
| 41 | 41 | int (*rebuild_header)(struct sock *sk); |
| 42 | + void (*sk_rx_dst_set)(struct sock *sk, const struct sk_buff *skb); | |
| 42 | 43 | int (*conn_request)(struct sock *sk, struct sk_buff *skb); |
| 43 | 44 | struct sock *(*syn_recv_sock)(struct sock *sk, struct sk_buff *skb, |
| 44 | 45 | struct request_sock *req, |
| @@ -249,13 +249,4 @@ static inline __u8 inet_sk_flowi_flags(const struct sock *sk) · 248 unmodified lines | ||
| 249 | 249 | return flags; |
| 250 | 250 | } |
| 251 | 251 | |
| 252 | −static inline void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) | |
| 253 | −{ | |
| 254 | − struct dst_entry *dst = skb_dst(skb); | |
| 255 | − | |
| 256 | − dst_hold(dst); | |
| 257 | − sk->sk_rx_dst = dst; | |
| 258 | − inet_sk(sk)->rx_dst_ifindex = skb->skb_iif; | |
| 259 | −} | |
| 260 | − | |
| 261 | 252 | #endif /* _INET_SOCK_H */ |
| @@ -5392,6 +5392,8 @@ int tcp_rcv_established(struct sock *sk, struct sk_buff *skb, · 5391 unmodified lines | ||
| 5392 | 5392 | { |
| 5393 | 5393 | struct tcp_sock *tp = tcp_sk(sk); |
| 5394 | 5394 | |
| 5395 | + if (unlikely(sk->sk_rx_dst == NULL)) | |
| 5396 | + inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb); | |
| 5395 | 5397 | /* |
| 5396 | 5398 | * Header prediction. |
| 5397 | 5399 | * The code loosely follows the one in the famous |
| @@ -5605,7 +5607,7 @@ void tcp_finish_connect(struct sock *sk, struct sk_buff *skb) · 207 unmodified lines | ||
| 5605 | 5607 | tcp_set_state(sk, TCP_ESTABLISHED); |
| 5606 | 5608 | |
| 5607 | 5609 | if (skb != NULL) { |
| 5608 | − inet_sk_rx_dst_set(sk, skb); | |
| 5610 | + icsk->icsk_af_ops->sk_rx_dst_set(sk, skb); | |
| 5609 | 5611 | security_inet_conn_established(sk, skb); |
| 5610 | 5612 | } |
| 5611 | 5613 | |
| @@ -1627,9 +1627,6 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb) · 1626 unmodified lines | ||
| 1627 | 1627 | sk->sk_rx_dst = NULL; |
| 1628 | 1628 | } |
| 1629 | 1629 | } |
| 1630 | − if (unlikely(sk->sk_rx_dst == NULL)) | |
| 1631 | − inet_sk_rx_dst_set(sk, skb); | |
| 1632 | − | |
| 1633 | 1630 | if (tcp_rcv_established(sk, skb, tcp_hdr(skb), skb->len)) { |
| 1634 | 1631 | rsk = sk; |
| 1635 | 1632 | goto reset; |
| @@ -1872,10 +1869,20 @@ static struct timewait_sock_ops tcp_timewait_sock_ops = { · 236 unmodified lines | ||
| 1872 | 1869 | .twsk_destructor= tcp_twsk_destructor, |
| 1873 | 1870 | }; |
| 1874 | 1871 | |
| 1872 | +static void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) | |
| 1873 | +{ | |
| 1874 | + struct dst_entry *dst = skb_dst(skb); | |
| 1875 | + | |
| 1876 | + dst_hold(dst); | |
| 1877 | + sk->sk_rx_dst = dst; | |
| 1878 | + inet_sk(sk)->rx_dst_ifindex = skb->skb_iif; | |
| 1879 | +} | |
| 1880 | + | |
| 1875 | 1881 | const struct inet_connection_sock_af_ops ipv4_specific = { |
| 1876 | 1882 | .queue_xmit = ip_queue_xmit, |
| 1877 | 1883 | .send_check = tcp_v4_send_check, |
| 1878 | 1884 | .rebuild_header = inet_sk_rebuild_header, |
| 1885 | + .sk_rx_dst_set = inet_sk_rx_dst_set, | |
| 1879 | 1886 | .conn_request = tcp_v4_conn_request, |
| 1880 | 1887 | .syn_recv_sock = tcp_v4_syn_recv_sock, |
| 1881 | 1888 | .net_header_len = sizeof(struct iphdr), |
| @@ -387,7 +387,7 @@ struct sock *tcp_create_openreq_child(struct sock *sk, struct request_sock *req, · 386 unmodified lines | ||
| 387 | 387 | struct tcp_sock *oldtp = tcp_sk(sk); |
| 388 | 388 | struct tcp_cookie_values *oldcvp = oldtp->cookie_values; |
| 389 | 389 | |
| 390 | − inet_sk_rx_dst_set(newsk, skb); | |
| 390 | + newicsk->icsk_af_ops->sk_rx_dst_set(newsk, skb); | |
| 391 | 391 | |
| 392 | 392 | /* TCP Cookie Transactions require space for the cookie pair, |
| 393 | 393 | * as it differs for each connection. There is no need to |
| @@ -1447,7 +1447,17 @@ static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) · 1446 unmodified lines | ||
| 1447 | 1447 | opt_skb = skb_clone(skb, sk_gfp_atomic(sk, GFP_ATOMIC)); |
| 1448 | 1448 | |
| 1449 | 1449 | if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */ |
| 1450 | + struct dst_entry *dst = sk->sk_rx_dst; | |
| 1451 | + | |
| 1450 | 1452 | sock_rps_save_rxhash(sk, skb); |
| 1453 | + if (dst) { | |
| 1454 | + if (inet_sk(sk)->rx_dst_ifindex != skb->skb_iif || | |
| 1455 | + dst->ops->check(dst, np->rx_dst_cookie) == NULL) { | |
| 1456 | + dst_release(dst); | |
| 1457 | + sk->sk_rx_dst = NULL; | |
| 1458 | + } | |
| 1459 | + } | |
| 1460 | + | |
| 1451 | 1461 | if (tcp_rcv_established(sk, skb, tcp_hdr(skb), skb->len)) |
| 1452 | 1462 | goto reset; |
| 1453 | 1463 | if (opt_skb) |
| @@ -1705,9 +1715,9 @@ static void tcp_v6_early_demux(struct sk_buff *skb) · 251 unmodified lines | ||
| 1705 | 1715 | struct dst_entry *dst = sk->sk_rx_dst; |
| 1706 | 1716 | struct inet_sock *icsk = inet_sk(sk); |
| 1707 | 1717 | if (dst) |
| 1708 | − dst = dst_check(dst, 0); | |
| 1718 | + dst = dst_check(dst, inet6_sk(sk)->rx_dst_cookie); | |
| 1709 | 1719 | if (dst && |
| 1710 | − icsk->rx_dst_ifindex == inet6_iif(skb)) | |
| 1720 | + icsk->rx_dst_ifindex == skb->skb_iif) | |
| 1711 | 1721 | skb_dst_set_noref(skb, dst); |
| 1712 | 1722 | } |
| 1713 | 1723 | } |
| @@ -1719,10 +1729,23 @@ static struct timewait_sock_ops tcp6_timewait_sock_ops = { · 5 unmodified lines | ||
| 1719 | 1729 | .twsk_destructor= tcp_twsk_destructor, |
| 1720 | 1730 | }; |
| 1721 | 1731 | |
| 1732 | +static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) | |
| 1733 | +{ | |
| 1734 | + struct dst_entry *dst = skb_dst(skb); | |
| 1735 | + const struct rt6_info *rt = (const struct rt6_info *)dst; | |
| 1736 | + | |
| 1737 | + dst_hold(dst); | |
| 1738 | + sk->sk_rx_dst = dst; | |
| 1739 | + inet_sk(sk)->rx_dst_ifindex = skb->skb_iif; | |
| 1740 | + if (rt->rt6i_node) | |
| 1741 | + inet6_sk(sk)->rx_dst_cookie = rt->rt6i_node->fn_sernum; | |
| 1742 | +} | |
| 1743 | + | |
| 1722 | 1744 | static const struct inet_connection_sock_af_ops ipv6_specific = { |
| 1723 | 1745 | .queue_xmit = inet6_csk_xmit, |
| 1724 | 1746 | .send_check = tcp_v6_send_check, |
| 1725 | 1747 | .rebuild_header = inet6_sk_rebuild_header, |
| 1748 | + .sk_rx_dst_set = inet6_sk_rx_dst_set, | |
| 1726 | 1749 | .conn_request = tcp_v6_conn_request, |
| 1727 | 1750 | .syn_recv_sock = tcp_v6_syn_recv_sock, |
| 1728 | 1751 | .net_header_len = sizeof(struct ipv6hdr), |