History

tcp: track application-limited rate samples

This commit adds code to track whether the delivery rate represented by each rate_sample was limited by the application. Upon each transmit, we store in the is_app_limited field in the skb a boolean bit indicating whether there is a known "bubble in the pipe": a point in the rate sample interval where the sender was application-limited, and did not transmit even though the cwnd and pacing rate allowed it. This logic marks the flow app-limited on a write if *all* of the following are true: 1) There is less than 1 MSS of unsent data in the write queue available to transmit. 2) There is no packet in the sender's queues (e.g. in fq or the NIC tx queue). 3) The connection is not limited by cwnd. 4) There are no lost packets to retransmit. The tcp_rate_check_app_limited() code in tcp_rate.c determines whether the connection is application-limited at the moment. If the flow is application-limited, it sets the tp->app_limited field. If the flow is application-limited then that means there is effectively a "bubble" of silence in the pipe now, and this silence will be reflected in a lower bandwidth sample for any rate samples from now until we get an ACK indicating this bubble has exited the pipe: specifically, until we get an ACK for the next packet we transmit. When we send every skb we record in scb->tx.is_app_limited whether the resulting rate sample will be application-limited. The code in tcp_rate_gen() checks to see when it is safe to mark all known application-limited bubbles of silence as having exited the pipe. It does this by checking to see when the delivered count moves past the tp->app_limited marker. At this point it zeroes the tp->app_limited marker, as all known bubbles are out of the pipe. We make room for the tx.is_app_limited bit in the skb by borrowing a bit from the in_flight field used by NV to record the number of bytes in flight. The receive window in the TCP header is 16 bits, and the max receive window scaling shift factor is 14 (RFC 1323). So the max receive window offered by the TCP protocol is 2^(16+14) = 2^30. So we only need 30 bits for the tx.in_flight used by NV.

Soheil Hassas Yeganeh committed 10 years ago

Commit d7722e81 parent b9f64825 files · +45 −2Browse files

Unified · Split

5 files
  1. include/linux/tcp.h +1 −0
  2. include/net/tcp.h +5 −1
  3. net/ipv4/tcp.c +8 −0
  4. net/ipv4/tcp_minisocks.c +3 −0
  5. net/ipv4/tcp_rate.c +28 −1
include/linux/tcp.h+1−0
@@ -268,6 +268,7 @@ struct tcp_sock { · 267 unmodified lines
268 u32 prr_out; /* Total number of pkts sent during Recovery. */268 u32 prr_out; /* Total number of pkts sent during Recovery. */
269 u32 delivered; /* Total data packets delivered incl. rexmits */269 u32 delivered; /* Total data packets delivered incl. rexmits */
270 u32 lost; /* Total data packets lost incl. rexmits */270 u32 lost; /* Total data packets lost incl. rexmits */
271 u32 app_limited; /* limited until "delivered" reaches this val */
271 struct skb_mstamp first_tx_mstamp; /* start of window send phase */272 struct skb_mstamp first_tx_mstamp; /* start of window send phase */
272 struct skb_mstamp delivered_mstamp; /* time we reached "delivered" */273 struct skb_mstamp delivered_mstamp; /* time we reached "delivered" */
273 274
include/net/tcp.h+5−1
@@ -764,7 +764,9 @@ struct tcp_skb_cb { · 763 unmodified lines
764 union {764 union {
765 struct {765 struct {
766 /* There is space for up to 24 bytes */766 /* There is space for up to 24 bytes */
767 __u32 in_flight;/* Bytes in flight when packet sent */767 __u32 in_flight:30,/* Bytes in flight at transmit */
768 is_app_limited:1, /* cwnd not fully used? */
769 unused:1;
768 /* pkts S/ACKed so far upon tx of skb, incl retrans: */770 /* pkts S/ACKed so far upon tx of skb, incl retrans: */
769 __u32 delivered;771 __u32 delivered;
770 /* start of send pipeline phase */772 /* start of send pipeline phase */
@@ -883,6 +885,7 @@ struct rate_sample { · 112 unmodified lines
883 int losses; /* number of packets marked lost upon ACK */885 int losses; /* number of packets marked lost upon ACK */
884 u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */886 u32 acked_sacked; /* number of packets newly (S)ACKed upon ACK */
885 u32 prior_in_flight; /* in flight before this ACK */887 u32 prior_in_flight; /* in flight before this ACK */
888 bool is_app_limited; /* is sample from packet with bubble in pipe? */
886 bool is_retrans; /* is sample from retransmission? */889 bool is_retrans; /* is sample from retransmission? */
887};890};
888 891
@@ -978,6 +981,7 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, · 89 unmodified lines
978 struct rate_sample *rs);981 struct rate_sample *rs);
979void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,982void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
980 struct skb_mstamp *now, struct rate_sample *rs);983 struct skb_mstamp *now, struct rate_sample *rs);
984void tcp_rate_check_app_limited(struct sock *sk);
981 985
982/* These functions determine how the current flow behaves in respect of SACK986/* These functions determine how the current flow behaves in respect of SACK
983 * handling. SACK is negotiated with the peer, and therefore it can vary987 * handling. SACK is negotiated with the peer, and therefore it can vary
net/ipv4/tcp.c+8−0
@@ -396,6 +396,9 @@ void tcp_init_sock(struct sock *sk) · 395 unmodified lines
396 */396 */
397 tp->snd_cwnd = TCP_INIT_CWND;397 tp->snd_cwnd = TCP_INIT_CWND;
398 398
399 /* There's a bubble in the pipe until at least the first ACK. */
400 tp->app_limited = ~0U;
401
399 /* See draft-stevens-tcpca-spec-01 for discussion of the402 /* See draft-stevens-tcpca-spec-01 for discussion of the
400 * initialization of these values.403 * initialization of these values.
401 */404 */
@@ -1014,6 +1017,9 @@ int tcp_sendpage(struct sock *sk, struct page *page, int offset, · 612 unmodified lines
1014 flags);1017 flags);
1015 1018
1016 lock_sock(sk);1019 lock_sock(sk);
1020
1021 tcp_rate_check_app_limited(sk); /* is sending application-limited? */
1022
1017 res = do_tcp_sendpages(sk, page, offset, size, flags);1023 res = do_tcp_sendpages(sk, page, offset, size, flags);
1018 release_sock(sk);1024 release_sock(sk);
1019 return res;1025 return res;
@@ -1115,6 +1121,8 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) · 95 unmodified lines
1115 1121
1116 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);1122 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1117 1123
1124 tcp_rate_check_app_limited(sk); /* is sending application-limited? */
1125
1118 /* Wait for a connection to finish. One exception is TCP Fast Open1126 /* Wait for a connection to finish. One exception is TCP Fast Open
1119 * (passive side) where data is allowed to be sent before a connection1127 * (passive side) where data is allowed to be sent before a connection
1120 * is fully established.1128 * is fully established.
net/ipv4/tcp_minisocks.c+3−0
@@ -487,6 +487,9 @@ struct sock *tcp_create_openreq_child(const struct sock *sk, · 486 unmodified lines
487 newtp->snd_cwnd = TCP_INIT_CWND;487 newtp->snd_cwnd = TCP_INIT_CWND;
488 newtp->snd_cwnd_cnt = 0;488 newtp->snd_cwnd_cnt = 0;
489 489
490 /* There's a bubble in the pipe until at least the first ACK. */
491 newtp->app_limited = ~0U;
492
490 tcp_init_xmit_timers(newsk);493 tcp_init_xmit_timers(newsk);
491 newtp->write_seq = newtp->pushed_seq = treq->snt_isn + 1;494 newtp->write_seq = newtp->pushed_seq = treq->snt_isn + 1;
492 495
net/ipv4/tcp_rate.c+28−1
@@ -26,9 +26,13 @@ · 25 unmodified lines
26 * other factors like applications or receiver window limits. The estimator26 * other factors like applications or receiver window limits. The estimator
27 * deliberately avoids using the inter-packet spacing approach because that27 * deliberately avoids using the inter-packet spacing approach because that
28 * approach requires a large number of samples and sophisticated filtering.28 * approach requires a large number of samples and sophisticated filtering.
29 *
30 * TCP flows can often be application-limited in request/response workloads.
31 * The estimator marks a bandwidth sample as application-limited if there
32 * was some moment during the sampled window of packets when there was no data
33 * ready to send in the write queue.
29 */34 */
30 35
31
32/* Snapshot the current delivery information in the skb, to generate36/* Snapshot the current delivery information in the skb, to generate
33 * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered().37 * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered().
34 */38 */
@@ -58,6 +62,7 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb) · 23 unmodified lines
58 TCP_SKB_CB(skb)->tx.first_tx_mstamp = tp->first_tx_mstamp;62 TCP_SKB_CB(skb)->tx.first_tx_mstamp = tp->first_tx_mstamp;
59 TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp;63 TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp;
60 TCP_SKB_CB(skb)->tx.delivered = tp->delivered;64 TCP_SKB_CB(skb)->tx.delivered = tp->delivered;
65 TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0;
61}66}
62 67
63/* When an skb is sacked or acked, we fill in the rate sample with the (prior)68/* When an skb is sacked or acked, we fill in the rate sample with the (prior)
@@ -80,6 +85,7 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb, · 16 unmodified lines
80 after(scb->tx.delivered, rs->prior_delivered)) {85 after(scb->tx.delivered, rs->prior_delivered)) {
81 rs->prior_delivered = scb->tx.delivered;86 rs->prior_delivered = scb->tx.delivered;
82 rs->prior_mstamp = scb->tx.delivered_mstamp;87 rs->prior_mstamp = scb->tx.delivered_mstamp;
88 rs->is_app_limited = scb->tx.is_app_limited;
83 rs->is_retrans = scb->sacked & TCPCB_RETRANS;89 rs->is_retrans = scb->sacked & TCPCB_RETRANS;
84 90
85 /* Find the duration of the "send phase" of this window: */91 /* Find the duration of the "send phase" of this window: */
@@ -105,6 +111,10 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, · 19 unmodified lines
105 struct tcp_sock *tp = tcp_sk(sk);111 struct tcp_sock *tp = tcp_sk(sk);
106 u32 snd_us, ack_us;112 u32 snd_us, ack_us;
107 113
114 /* Clear app limited if bubble is acked and gone. */
115 if (tp->app_limited && after(tp->delivered, tp->app_limited))
116 tp->app_limited = 0;
117
108 /* TODO: there are multiple places throughout tcp_ack() to get118 /* TODO: there are multiple places throughout tcp_ack() to get
109 * current time. Refactor the code using a new "tcp_acktag_state"119 * current time. Refactor the code using a new "tcp_acktag_state"
110 * to carry current time, flags, stats like "tcp_sacktag_state".120 * to carry current time, flags, stats like "tcp_sacktag_state".
@@ -147,3 +157,20 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost, · 36 unmodified lines
147 tp->rx_opt.sack_ok, tcp_min_rtt(tp));157 tp->rx_opt.sack_ok, tcp_min_rtt(tp));
148 }158 }
149}159}
160
161/* If a gap is detected between sends, mark the socket application-limited. */
162void tcp_rate_check_app_limited(struct sock *sk)
163{
164 struct tcp_sock *tp = tcp_sk(sk);
165
166 if (/* We have less than one packet to send. */
167 tp->write_seq - tp->snd_nxt < tp->mss_cache &&
168 /* Nothing in sending host's qdisc queues or NIC tx queue. */
169 sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) &&
170 /* We are not limited by CWND. */
171 tcp_packets_in_flight(tp) < tp->snd_cwnd &&
172 /* All lost packets have been retransmitted. */
173 tp->lost_out <= tp->retrans_out)
174 tp->app_limited =
175 (tp->delivered + tcp_packets_in_flight(tp)) ? : 1;
176}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t