History

Merge branch 'tcp-receive-side-improvements'

Eric Dumazet says: ==================== tcp: receive side improvements We have set tcp_rmem[2] to 15 MB for about 8 years at Google, but had some issues for high speed flows on very small RTT. TCP rx autotuning has a tendency to overestimate the RTT, thus tp->rcvq_space.space and sk->sk_rcvbuf. This makes TCP receive queues much bigger than necessary, to a point cpu caches are evicted before application can copy the data, on cpus using DDIO. This series aims to fix this. - First patch adds tcp_rcvbuf_grow() tracepoint, which was very convenient to study the various issues fixed in this series. - Seven patches fix receiver autotune issues. - Two patches fix sender side issues. - Final patch increases tcp_rmem[2] so that TCP speed over WAN can meet modern needs. Tested on a 200Gbit NIC, average max throughput of a single flow: Before: 73593 Mbit. After: 122514 Mbit. ====================

Jakub Kicinski committed 1 year ago

Commit 2da35e42 parents bebd7b2 572be9b7 files · +134 −66Browse files

Unified · Split

7 files
  1. Documentation/networking/ip-sysctl.rst +2 −2
  2. include/linux/tcp.h +1 −1
  3. include/trace/events/tcp.h +73 −0
  4. net/ipv4/tcp.c +1 −1
  5. net/ipv4/tcp_input.c +53 −57
  6. net/ipv4/tcp_ipv4.c +2 −2
  7. net/ipv4/tcp_output.c +2 −3
Documentation/networking/ip-sysctl.rst+2−2
@@ -735,7 +735,7 @@ tcp_rmem - vector of 3 INTEGERs: min, default, max · 734 unmodified lines
735 net.core.rmem_max. Calling setsockopt() with SO_RCVBUF disables735 net.core.rmem_max. Calling setsockopt() with SO_RCVBUF disables
736 automatic tuning of that socket's receive buffer size, in which736 automatic tuning of that socket's receive buffer size, in which
737 case this value is ignored.737 case this value is ignored.
738 Default: between 131072 and 6MB, depending on RAM size.738 Default: between 131072 and 32MB, depending on RAM size.
739 739
740tcp_sack - BOOLEAN740tcp_sack - BOOLEAN
741 Enable select acknowledgments (SACKS).741 Enable select acknowledgments (SACKS).
@@ -1099,7 +1099,7 @@ tcp_limit_output_bytes - INTEGER · 357 unmodified lines
1099 limits the number of bytes on qdisc or device to reduce artificial1099 limits the number of bytes on qdisc or device to reduce artificial
1100 RTT/cwnd and reduce bufferbloat.1100 RTT/cwnd and reduce bufferbloat.
1101 1101
1102 Default: 1048576 (16 * 65536)1102 Default: 4194304 (4 MB)
1103 1103
1104tcp_challenge_ack_limit - INTEGER1104tcp_challenge_ack_limit - INTEGER
1105 Limits number of Challenge ACK sent per second, as recommended1105 Limits number of Challenge ACK sent per second, as recommended
include/linux/tcp.h+1−1
@@ -340,7 +340,7 @@ struct tcp_sock { · 339 unmodified lines
340 } rcv_rtt_est;340 } rcv_rtt_est;
341/* Receiver queue space */341/* Receiver queue space */
342 struct {342 struct {
343 u32 space;343 int space;
344 u32 seq;344 u32 seq;
345 u64 time;345 u64 time;
346 } rcvq_space;346 } rcvq_space;
include/trace/events/tcp.h+73−0
@@ -213,6 +213,79 @@ DEFINE_EVENT(tcp_event_sk, tcp_rcv_space_adjust, · 212 unmodified lines
213 TP_ARGS(sk)213 TP_ARGS(sk)
214);214);
215 215
216TRACE_EVENT(tcp_rcvbuf_grow,
217
218 TP_PROTO(struct sock *sk, int time),
219
220 TP_ARGS(sk, time),
221
222 TP_STRUCT__entry(
223 __field(int, time)
224 __field(__u32, rtt_us)
225 __field(__u32, copied)
226 __field(__u32, inq)
227 __field(__u32, space)
228 __field(__u32, ooo_space)
229 __field(__u32, rcvbuf)
230 __field(__u8, scaling_ratio)
231 __field(__u16, sport)
232 __field(__u16, dport)
233 __field(__u16, family)
234 __array(__u8, saddr, 4)
235 __array(__u8, daddr, 4)
236 __array(__u8, saddr_v6, 16)
237 __array(__u8, daddr_v6, 16)
238 __field(const void *, skaddr)
239 __field(__u64, sock_cookie)
240 ),
241
242 TP_fast_assign(
243 struct inet_sock *inet = inet_sk(sk);
244 struct tcp_sock *tp = tcp_sk(sk);
245 __be32 *p32;
246
247 __entry->time = time;
248 __entry->rtt_us = tp->rcv_rtt_est.rtt_us >> 3;
249 __entry->copied = tp->copied_seq - tp->rcvq_space.seq;
250 __entry->inq = tp->rcv_nxt - tp->copied_seq;
251 __entry->space = tp->rcvq_space.space;
252 __entry->ooo_space = RB_EMPTY_ROOT(&tp->out_of_order_queue) ? 0 :
253 TCP_SKB_CB(tp->ooo_last_skb)->end_seq -
254 tp->rcv_nxt;
255
256 __entry->rcvbuf = sk->sk_rcvbuf;
257 __entry->scaling_ratio = tp->scaling_ratio;
258 __entry->sport = ntohs(inet->inet_sport);
259 __entry->dport = ntohs(inet->inet_dport);
260 __entry->family = sk->sk_family;
261
262 p32 = (__be32 *) __entry->saddr;
263 *p32 = inet->inet_saddr;
264
265 p32 = (__be32 *) __entry->daddr;
266 *p32 = inet->inet_daddr;
267
268 TP_STORE_ADDRS(__entry, inet->inet_saddr, inet->inet_daddr,
269 sk->sk_v6_rcv_saddr, sk->sk_v6_daddr);
270
271 __entry->skaddr = sk;
272 __entry->sock_cookie = sock_gen_cookie(sk);
273 ),
274
275 TP_printk("time=%u rtt_us=%u copied=%u inq=%u space=%u ooo=%u scaling_ratio=%u rcvbuf=%u "
276 "family=%s sport=%hu dport=%hu saddr=%pI4 daddr=%pI4 "
277 "saddrv6=%pI6c daddrv6=%pI6c skaddr=%p sock_cookie=%llx",
278 __entry->time, __entry->rtt_us, __entry->copied,
279 __entry->inq, __entry->space, __entry->ooo_space,
280 __entry->scaling_ratio, __entry->rcvbuf,
281 show_family_name(__entry->family),
282 __entry->sport, __entry->dport,
283 __entry->saddr, __entry->daddr,
284 __entry->saddr_v6, __entry->daddr_v6,
285 __entry->skaddr,
286 __entry->sock_cookie)
287);
288
216TRACE_EVENT(tcp_retransmit_synack,289TRACE_EVENT(tcp_retransmit_synack,
217 290
218 TP_PROTO(const struct sock *sk, const struct request_sock *req),291 TP_PROTO(const struct sock *sk, const struct request_sock *req),
net/ipv4/tcp.c+1−1
@@ -5231,7 +5231,7 @@ void __init tcp_init(void) · 5230 unmodified lines
5231 /* Set per-socket limits to no more than 1/128 the pressure threshold */5231 /* Set per-socket limits to no more than 1/128 the pressure threshold */
5232 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);5232 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
5233 max_wshare = min(4UL*1024*1024, limit);5233 max_wshare = min(4UL*1024*1024, limit);
5234 max_rshare = min(6UL*1024*1024, limit);5234 max_rshare = min(32UL*1024*1024, limit);
5235 5235
5236 init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE;5236 init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE;
5237 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;5237 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
net/ipv4/tcp_input.c+53−57
@@ -664,10 +664,12 @@ EXPORT_IPV6_MOD(tcp_initialize_rcv_mss); · 663 unmodified lines
664 */664 */
665static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)665static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep)
666{666{
667 u32 new_sample = tp->rcv_rtt_est.rtt_us;667 u32 new_sample, old_sample = tp->rcv_rtt_est.rtt_us;
668 long m = sample;668 long m = sample << 3;
669 669
670 if (new_sample != 0) {670 if (old_sample == 0 || m < old_sample) {
671 new_sample = m;
672 } else {
671 /* If we sample in larger samples in the non-timestamp673 /* If we sample in larger samples in the non-timestamp
672 * case, we could grossly overestimate the RTT especially674 * case, we could grossly overestimate the RTT especially
673 * with chatty applications or bulk transfer apps which675 * with chatty applications or bulk transfer apps which
@@ -678,17 +680,12 @@ static void tcp_rcv_rtt_update(struct tcp_sock *tp, u32 sample, int win_dep) · 4 unmodified lines
678 * else with timestamps disabled convergence takes too680 * else with timestamps disabled convergence takes too
679 * long.681 * long.
680 */682 */
681 if (!win_dep) {683 if (win_dep)
682 m -= (new_sample >> 3);684 return;
683 new_sample += m;685 /* Do not use this sample if receive queue is not empty. */
684 } else {686 if (tp->rcv_nxt != tp->copied_seq)
685 m <<= 3;687 return;
686 if (m < new_sample)688 new_sample = old_sample - (old_sample >> 3) + sample;
687 new_sample = m;
688 }
689 } else {
690 /* No previous measure. */
691 new_sample = m << 3;
692 }689 }
693 690
694 tp->rcv_rtt_est.rtt_us = new_sample;691 tp->rcv_rtt_est.rtt_us = new_sample;
@@ -712,7 +709,7 @@ new_measure: · 17 unmodified lines
712 tp->rcv_rtt_est.time = tp->tcp_mstamp;709 tp->rcv_rtt_est.time = tp->tcp_mstamp;
713}710}
714 711
715static s32 tcp_rtt_tsopt_us(const struct tcp_sock *tp)712static s32 tcp_rtt_tsopt_us(const struct tcp_sock *tp, u32 min_delta)
716{713{
717 u32 delta, delta_us;714 u32 delta, delta_us;
718 715
@@ -722,7 +719,7 @@ static s32 tcp_rtt_tsopt_us(const struct tcp_sock *tp) · 3 unmodified lines
722 719
723 if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {720 if (likely(delta < INT_MAX / (USEC_PER_SEC / TCP_TS_HZ))) {
724 if (!delta)721 if (!delta)
725 delta = 1;722 delta = min_delta;
726 delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);723 delta_us = delta * (USEC_PER_SEC / TCP_TS_HZ);
727 return delta_us;724 return delta_us;
728 }725 }
@@ -740,13 +737,39 @@ static inline void tcp_rcv_rtt_measure_ts(struct sock *sk, · 11 unmodified lines
740 737
741 if (TCP_SKB_CB(skb)->end_seq -738 if (TCP_SKB_CB(skb)->end_seq -
742 TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss) {739 TCP_SKB_CB(skb)->seq >= inet_csk(sk)->icsk_ack.rcv_mss) {
743 s32 delta = tcp_rtt_tsopt_us(tp);740 s32 delta = tcp_rtt_tsopt_us(tp, 0);
744 741
745 if (delta >= 0)742 if (delta > 0)
746 tcp_rcv_rtt_update(tp, delta, 0);743 tcp_rcv_rtt_update(tp, delta, 0);
747 }744 }
748}745}
749 746
747static void tcp_rcvbuf_grow(struct sock *sk)
748{
749 const struct net *net = sock_net(sk);
750 struct tcp_sock *tp = tcp_sk(sk);
751 int rcvwin, rcvbuf, cap;
752
753 if (!READ_ONCE(net->ipv4.sysctl_tcp_moderate_rcvbuf) ||
754 (sk->sk_userlocks & SOCK_RCVBUF_LOCK))
755 return;
756
757 /* slow start: allow the sender to double its rate. */
758 rcvwin = tp->rcvq_space.space << 1;
759
760 if (!RB_EMPTY_ROOT(&tp->out_of_order_queue))
761 rcvwin += TCP_SKB_CB(tp->ooo_last_skb)->end_seq - tp->rcv_nxt;
762
763 cap = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]);
764
765 rcvbuf = min_t(u32, tcp_space_from_win(sk, rcvwin), cap);
766 if (rcvbuf > sk->sk_rcvbuf) {
767 WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
768 /* Make the window clamp follow along. */
769 WRITE_ONCE(tp->window_clamp,
770 tcp_win_from_space(sk, rcvbuf));
771 }
772}
750/*773/*
751 * This function should be called every time data is copied to user space.774 * This function should be called every time data is copied to user space.
752 * It calculates the appropriate TCP receive buffer space.775 * It calculates the appropriate TCP receive buffer space.
@@ -754,8 +777,7 @@ static inline void tcp_rcv_rtt_measure_ts(struct sock *sk, · 1 unmodified line
754void tcp_rcv_space_adjust(struct sock *sk)777void tcp_rcv_space_adjust(struct sock *sk)
755{778{
756 struct tcp_sock *tp = tcp_sk(sk);779 struct tcp_sock *tp = tcp_sk(sk);
757 u32 copied;780 int time, inq, copied;
758 int time;
759 781
760 trace_tcp_rcv_space_adjust(sk);782 trace_tcp_rcv_space_adjust(sk);
761 783
@@ -766,45 +788,18 @@ void tcp_rcv_space_adjust(struct sock *sk) · 4 unmodified lines
766 788
767 /* Number of bytes copied to user in last RTT */789 /* Number of bytes copied to user in last RTT */
768 copied = tp->copied_seq - tp->rcvq_space.seq;790 copied = tp->copied_seq - tp->rcvq_space.seq;
791 /* Number of bytes in receive queue. */
792 inq = tp->rcv_nxt - tp->copied_seq;
793 copied -= inq;
769 if (copied <= tp->rcvq_space.space)794 if (copied <= tp->rcvq_space.space)
770 goto new_measure;795 goto new_measure;
771 796
772 /* A bit of theory :797 trace_tcp_rcvbuf_grow(sk, time);
773 * copied = bytes received in previous RTT, our base window
774 * To cope with packet losses, we need a 2x factor
775 * To cope with slow start, and sender growing its cwin by 100 %
776 * every RTT, we need a 4x factor, because the ACK we are sending
777 * now is for the next RTT, not the current one :
778 * <prev RTT . ><current RTT .. ><next RTT .... >
779 */
780
781 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) &&
782 !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
783 u64 rcvwin, grow;
784 int rcvbuf;
785
786 /* minimal window to cope with packet losses, assuming
787 * steady state. Add some cushion because of small variations.
788 */
789 rcvwin = ((u64)copied << 1) + 16 * tp->advmss;
790 798
791 /* Accommodate for sender rate increase (eg. slow start) */
792 grow = rcvwin * (copied - tp->rcvq_space.space);
793 do_div(grow, tp->rcvq_space.space);
794 rcvwin += (grow << 1);
795
796 rcvbuf = min_t(u64, tcp_space_from_win(sk, rcvwin),
797 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]));
798 if (rcvbuf > sk->sk_rcvbuf) {
799 WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
800
801 /* Make the window clamp follow along. */
802 WRITE_ONCE(tp->window_clamp,
803 tcp_win_from_space(sk, rcvbuf));
804 }
805 }
806 tp->rcvq_space.space = copied;799 tp->rcvq_space.space = copied;
807 800
801 tcp_rcvbuf_grow(sk);
802
808new_measure:803new_measure:
809 tp->rcvq_space.seq = tp->copied_seq;804 tp->rcvq_space.seq = tp->copied_seq;
810 tp->rcvq_space.time = tp->tcp_mstamp;805 tp->rcvq_space.time = tp->tcp_mstamp;
@@ -3226,7 +3221,7 @@ static bool tcp_ack_update_rtt(struct sock *sk, const int flag, · 2415 unmodified lines
3226 */3221 */
3227 if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp &&3222 if (seq_rtt_us < 0 && tp->rx_opt.saw_tstamp &&
3228 tp->rx_opt.rcv_tsecr && flag & FLAG_ACKED)3223 tp->rx_opt.rcv_tsecr && flag & FLAG_ACKED)
3229 seq_rtt_us = ca_rtt_us = tcp_rtt_tsopt_us(tp);3224 seq_rtt_us = ca_rtt_us = tcp_rtt_tsopt_us(tp, 1);
3230 3225
3231 rs->rtt_us = ca_rtt_us; /* RTT of last (S)ACKed packet (or -1) */3226 rs->rtt_us = ca_rtt_us; /* RTT of last (S)ACKed packet (or -1) */
3232 if (seq_rtt_us < 0)3227 if (seq_rtt_us < 0)
@@ -5173,6 +5168,7 @@ end: · 1940 unmodified lines
5173 skb_condense(skb);5168 skb_condense(skb);
5174 skb_set_owner_r(skb, sk);5169 skb_set_owner_r(skb, sk);
5175 }5170 }
5171 tcp_rcvbuf_grow(sk);
5176}5172}
5177 5173
5178static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb,5174static int __must_check tcp_queue_rcv(struct sock *sk, struct sk_buff *skb,
@@ -6873,6 +6869,9 @@ tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) · 1694 unmodified lines
6873 if (!tp->srtt_us)6869 if (!tp->srtt_us)
6874 tcp_synack_rtt_meas(sk, req);6870 tcp_synack_rtt_meas(sk, req);
6875 6871
6872 if (tp->rx_opt.tstamp_ok)
6873 tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
6874
6876 if (req) {6875 if (req) {
6877 tcp_rcv_synrecv_state_fastopen(sk);6876 tcp_rcv_synrecv_state_fastopen(sk);
6878 } else {6877 } else {
@@ -6898,9 +6897,6 @@ tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb) · 19 unmodified lines
6898 tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;6897 tp->snd_wnd = ntohs(th->window) << tp->rx_opt.snd_wscale;
6899 tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);6898 tcp_init_wl(tp, TCP_SKB_CB(skb)->seq);
6900 6899
6901 if (tp->rx_opt.tstamp_ok)
6902 tp->advmss -= TCPOLEN_TSTAMP_ALIGNED;
6903
6904 if (!inet_csk(sk)->icsk_ca_ops->cong_control)6900 if (!inet_csk(sk)->icsk_ca_ops->cong_control)
6905 tcp_update_pacing_rate(sk);6901 tcp_update_pacing_rate(sk);
6906 6902
net/ipv4/tcp_ipv4.c+2−2
@@ -3495,8 +3495,8 @@ static int __net_init tcp_sk_init(struct net *net) · 3494 unmodified lines
3495 * which are too large can cause TCP streams to be bursty.3495 * which are too large can cause TCP streams to be bursty.
3496 */3496 */
3497 net->ipv4.sysctl_tcp_tso_win_divisor = 3;3497 net->ipv4.sysctl_tcp_tso_win_divisor = 3;
3498 /* Default TSQ limit of 16 TSO segments */3498 /* Default TSQ limit of 4 MB */
3499 net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;3499 net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20;
3500 3500
3501 /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */3501 /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
3502 net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;3502 net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
net/ipv4/tcp_output.c+2−3
@@ -2619,9 +2619,8 @@ static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb, · 2618 unmodified lines
2619 limit = max_t(unsigned long,2619 limit = max_t(unsigned long,
2620 2 * skb->truesize,2620 2 * skb->truesize,
2621 READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift));2621 READ_ONCE(sk->sk_pacing_rate) >> READ_ONCE(sk->sk_pacing_shift));
2622 if (sk->sk_pacing_status == SK_PACING_NONE)2622 limit = min_t(unsigned long, limit,
2623 limit = min_t(unsigned long, limit,2623 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes));
2624 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes));
2625 limit <<= factor;2624 limit <<= factor;
2626 2625
2627 if (static_branch_unlikely(&tcp_tx_delay_enabled) &&2626 if (static_branch_unlikely(&tcp_tx_delay_enabled) &&