History

Merge branch 'revert-tun-tap-vhost-net-apply-qdisc-backpressure-on-full-ptr_ring-to-reduce-tx-drops'

Simon Schippers says: ==================== Revert "tun/tap & vhost-net: apply qdisc backpressure on full ptr_ring to reduce TX drops" Commit 1d6e569b7d0c ("tun/tap & vhost-net: avoid ptr_ring tail-drop when a qdisc is present") did not show a relevant performance regression in my testing, but on Brett Sheffield's librecast testbed it causes a significant throughput drop in an IPv6 multicast testcase. The regression can be pinpointed to multiple iperf3 TCP threads sending: for 8 threads the throughput dropped from 13.5 Gbit/s to 9.13 Gbit/s. Therefore this series reverts the qdisc backpressure work. Making the backpressure opt-in via a new IFF_BACKPRESSURE flag was proposed in [1], but a new IFF_* flag needs more review scrutiny than is available at the moment, so a revert was requested instead. The opt-in will be resubmitted for net-next later. [1] Link: https://lore.kernel.org/netdev/20260709095511.168235-1-simon.schippers@tu-dortmund.de/ Reported-by: Brett Sheffield <brett@librecast.net> Closes: https://lore.kernel.org/netdev/akVnoOYQOrt8k-Gu@karahi.librecast.net/ ====================

Jakub Kicinski committed 3 months ago

Commit 7e107ab2 parents 82f0e98 c3da92a4 files · +14 −139Browse files

Unified · Split

4 files
  1. drivers/net/tun.c +6 −103
  2. drivers/vhost/net.c +6 −15
  3. include/linux/if_tun.h +0 −3
  4. include/linux/ptr_ring.h +2 −18
drivers/net/tun.c+6−103
@@ -145,8 +145,6 @@ struct tun_file { · 144 unmodified lines
145 struct list_head next;145 struct list_head next;
146 struct tun_struct *detached;146 struct tun_struct *detached;
147 struct ptr_ring tx_ring;147 struct ptr_ring tx_ring;
148 /* Protected by tx_ring.consumer_lock */
149 int cons_cnt;
150 struct xdp_rxq_info xdp_rxq;148 struct xdp_rxq_info xdp_rxq;
151};149};
152 150
@@ -590,13 +588,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean) · 437 unmodified lines
590 rcu_assign_pointer(tun->tfiles[index],588 rcu_assign_pointer(tun->tfiles[index],
591 tun->tfiles[tun->numqueues - 1]);589 tun->tfiles[tun->numqueues - 1]);
592 ntfile = rtnl_dereference(tun->tfiles[index]);590 ntfile = rtnl_dereference(tun->tfiles[index]);
593 spin_lock(&ntfile->tx_ring.consumer_lock);
594 ntfile->queue_index = index;591 ntfile->queue_index = index;
595 ntfile->xdp_rxq.queue_index = index;592 ntfile->xdp_rxq.queue_index = index;
596 ntfile->cons_cnt = 0;
597 if (__ptr_ring_empty(&ntfile->tx_ring))
598 netif_wake_subqueue(tun->dev, index);
599 spin_unlock(&ntfile->tx_ring.consumer_lock);
600 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],593 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
601 NULL);594 NULL);
602 595
@@ -737,9 +730,6 @@ static int tun_attach(struct tun_struct *tun, struct file *file, · 134 unmodified lines
737 goto out;730 goto out;
738 }731 }
739 732
740 spin_lock(&tfile->tx_ring.consumer_lock);
741 tfile->cons_cnt = 0;
742 spin_unlock(&tfile->tx_ring.consumer_lock);
743 tfile->queue_index = tun->numqueues;733 tfile->queue_index = tun->numqueues;
744 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;734 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
745 735
@@ -1018,7 +1008,6 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 272 unmodified lines
1018 struct netdev_queue *queue;1008 struct netdev_queue *queue;
1019 struct tun_file *tfile;1009 struct tun_file *tfile;
1020 int len = skb->len;1010 int len = skb->len;
1021 int ret;
1022 1011
1023 rcu_read_lock();1012 rcu_read_lock();
1024 tfile = rcu_dereference(tun->tfiles[txq]);1013 tfile = rcu_dereference(tun->tfiles[txq]);
@@ -1073,33 +1062,13 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 48 unmodified lines
1073 1062
1074 nf_reset_ct(skb);1063 nf_reset_ct(skb);
1075 1064
1076 queue = netdev_get_tx_queue(dev, txq);1065 if (ptr_ring_produce(&tfile->tx_ring, skb)) {
1077
1078 spin_lock(&tfile->tx_ring.producer_lock);
1079 ret = __ptr_ring_produce(&tfile->tx_ring, skb);
1080 if (!qdisc_txq_has_no_queue(queue) &&
1081 __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) {
1082 netif_tx_stop_queue(queue);
1083 /* Paired with smp_mb() in __tun_wake_queue() */
1084 smp_mb__after_atomic();
1085 if (!__ptr_ring_check_produce(&tfile->tx_ring))
1086 netif_tx_wake_queue(queue);
1087 }
1088 spin_unlock(&tfile->tx_ring.producer_lock);
1089
1090 if (ret) {
1091 /* This should be a rare case if a qdisc is present, but
1092 * can happen due to lltx.
1093 * Since skb_tx_timestamp(), skb_orphan(),
1094 * run_ebpf_filter() and pskb_trim() could have tinkered
1095 * with the SKB, returning NETDEV_TX_BUSY is unsafe and
1096 * we must drop instead.
1097 */
1098 drop_reason = SKB_DROP_REASON_FULL_RING;1066 drop_reason = SKB_DROP_REASON_FULL_RING;
1099 goto drop;1067 goto drop;
1100 }1068 }
1101 1069
1102 /* dev->lltx requires to do our own update of trans_start */1070 /* dev->lltx requires to do our own update of trans_start */
1071 queue = netdev_get_tx_queue(dev, txq);
1103 txq_trans_cond_update(queue);1072 txq_trans_cond_update(queue);
1104 1073
1105 /* Notify and wake up reader process */1074 /* Notify and wake up reader process */
@@ -2147,46 +2116,13 @@ done: · 1041 unmodified lines
2147 return total;2116 return total;
2148}2117}
2149 2118
2150/* Callers must hold ring.consumer_lock */2119static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
2151static void __tun_wake_queue(struct tun_struct *tun,
2152 struct tun_file *tfile, int consumed)
2153{
2154 struct netdev_queue *txq = netdev_get_tx_queue(tun->dev,
2155 tfile->queue_index);
2156
2157 /* Paired with smp_mb__after_atomic() in tun_net_xmit() */
2158 smp_mb();
2159 if (netif_tx_queue_stopped(txq)) {
2160 tfile->cons_cnt += consumed;
2161 if (tfile->cons_cnt >= tfile->tx_ring.size / 2 ||
2162 __ptr_ring_empty(&tfile->tx_ring)) {
2163 netif_tx_wake_queue(txq);
2164 tfile->cons_cnt = 0;
2165 }
2166 }
2167}
2168
2169static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile)
2170{
2171 void *ptr;
2172
2173 spin_lock(&tfile->tx_ring.consumer_lock);
2174 ptr = __ptr_ring_consume(&tfile->tx_ring);
2175 if (ptr)
2176 __tun_wake_queue(tun, tfile, 1);
2177
2178 spin_unlock(&tfile->tx_ring.consumer_lock);
2179 return ptr;
2180}
2181
2182static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile,
2183 int noblock, int *err)
2184{2120{
2185 DECLARE_WAITQUEUE(wait, current);2121 DECLARE_WAITQUEUE(wait, current);
2186 void *ptr = NULL;2122 void *ptr = NULL;
2187 int error = 0;2123 int error = 0;
2188 2124
2189 ptr = tun_ring_consume(tun, tfile);2125 ptr = ptr_ring_consume(&tfile->tx_ring);
2190 if (ptr)2126 if (ptr)
2191 goto out;2127 goto out;
2192 if (noblock) {2128 if (noblock) {
@@ -2198,7 +2134,7 @@ static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, · 5 unmodified lines
2198 2134
2199 while (1) {2135 while (1) {
2200 set_current_state(TASK_INTERRUPTIBLE);2136 set_current_state(TASK_INTERRUPTIBLE);
2201 ptr = tun_ring_consume(tun, tfile);2137 ptr = ptr_ring_consume(&tfile->tx_ring);
2202 if (ptr)2138 if (ptr)
2203 break;2139 break;
2204 if (signal_pending(current)) {2140 if (signal_pending(current)) {
@@ -2235,7 +2171,7 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, · 30 unmodified lines
2235 2171
2236 if (!ptr) {2172 if (!ptr) {
2237 /* Read frames from ring */2173 /* Read frames from ring */
2238 ptr = tun_ring_recv(tun, tfile, noblock, &err);2174 ptr = tun_ring_recv(tfile, noblock, &err);
2239 if (!ptr)2175 if (!ptr)
2240 return err;2176 return err;
2241 }2177 }
@@ -3690,16 +3626,6 @@ static int tun_queue_resize(struct tun_struct *tun) · 1448 unmodified lines
3690 dev->tx_queue_len, GFP_KERNEL,3626 dev->tx_queue_len, GFP_KERNEL,
3691 tun_ptr_free);3627 tun_ptr_free);
3692 3628
3693 if (!ret) {
3694 for (i = 0; i < tun->numqueues; i++) {
3695 tfile = rtnl_dereference(tun->tfiles[i]);
3696 spin_lock(&tfile->tx_ring.consumer_lock);
3697 netif_wake_subqueue(tun->dev, tfile->queue_index);
3698 tfile->cons_cnt = 0;
3699 spin_unlock(&tfile->tx_ring.consumer_lock);
3700 }
3701 }
3702
3703 kfree(rings);3629 kfree(rings);
3704 return ret;3630 return ret;
3705}3631}
@@ -3808,29 +3734,6 @@ struct ptr_ring *tun_get_tx_ring(struct file *file) · 102 unmodified lines
3808}3734}
3809EXPORT_SYMBOL_GPL(tun_get_tx_ring);3735EXPORT_SYMBOL_GPL(tun_get_tx_ring);
3810 3736
3811/* Callers must hold ring.consumer_lock */
3812void tun_wake_queue(struct file *file, int consumed)
3813{
3814 struct tun_file *tfile;
3815 struct tun_struct *tun;
3816
3817 if (file->f_op != &tun_fops)
3818 return;
3819
3820 tfile = file->private_data;
3821 if (!tfile)
3822 return;
3823
3824 rcu_read_lock();
3825
3826 tun = rcu_dereference(tfile->tun);
3827 if (tun)
3828 __tun_wake_queue(tun, tfile, consumed);
3829
3830 rcu_read_unlock();
3831}
3832EXPORT_SYMBOL_GPL(tun_wake_queue);
3833
3834module_init(tun_init);3737module_init(tun_init);
3835module_exit(tun_cleanup);3738module_exit(tun_cleanup);
3836MODULE_DESCRIPTION(DRV_DESCRIPTION);3739MODULE_DESCRIPTION(DRV_DESCRIPTION);
drivers/vhost/net.c+6−15
@@ -176,21 +176,13 @@ static void *vhost_net_buf_consume(struct vhost_net_buf *rxq) · 175 unmodified lines
176 return ret;176 return ret;
177}177}
178 178
179static int vhost_net_buf_produce(struct sock *sk,179static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
180 struct vhost_net_virtqueue *nvq)
181{180{
182 struct file *file = sk->sk_socket->file;
183 struct vhost_net_buf *rxq = &nvq->rxq;181 struct vhost_net_buf *rxq = &nvq->rxq;
184 182
185 rxq->head = 0;183 rxq->head = 0;
186 spin_lock(&nvq->rx_ring->consumer_lock);184 rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
187 rxq->tail = __ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,185 VHOST_NET_BATCH);
188 VHOST_NET_BATCH);
189
190 if (rxq->tail)
191 tun_wake_queue(file, rxq->tail);
192
193 spin_unlock(&nvq->rx_ring->consumer_lock);
194 return rxq->tail;186 return rxq->tail;
195}187}
196 188
@@ -217,15 +209,14 @@ static int vhost_net_buf_peek_len(void *ptr) · 20 unmodified lines
217 return __skb_array_len_with_tag(ptr);209 return __skb_array_len_with_tag(ptr);
218}210}
219 211
220static int vhost_net_buf_peek(struct sock *sk,212static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
221 struct vhost_net_virtqueue *nvq)
222{213{
223 struct vhost_net_buf *rxq = &nvq->rxq;214 struct vhost_net_buf *rxq = &nvq->rxq;
224 215
225 if (!vhost_net_buf_is_empty(rxq))216 if (!vhost_net_buf_is_empty(rxq))
226 goto out;217 goto out;
227 218
228 if (!vhost_net_buf_produce(sk, nvq))219 if (!vhost_net_buf_produce(nvq))
229 return 0;220 return 0;
230 221
231out:222out:
@@ -1013,7 +1004,7 @@ static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk) · 781 unmodified lines
1013 unsigned long flags;1004 unsigned long flags;
1014 1005
1015 if (rvq->rx_ring)1006 if (rvq->rx_ring)
1016 return vhost_net_buf_peek(sk, rvq);1007 return vhost_net_buf_peek(rvq);
1017 1008
1018 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);1009 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
1019 head = skb_peek(&sk->sk_receive_queue);1010 head = skb_peek(&sk->sk_receive_queue);
include/linux/if_tun.h+0−3
@@ -22,7 +22,6 @@ struct tun_msg_ctl { · 21 unmodified lines
22#if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE)22#if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE)
23struct socket *tun_get_socket(struct file *);23struct socket *tun_get_socket(struct file *);
24struct ptr_ring *tun_get_tx_ring(struct file *file);24struct ptr_ring *tun_get_tx_ring(struct file *file);
25void tun_wake_queue(struct file *file, int consumed);
26 25
27static inline bool tun_is_xdp_frame(void *ptr)26static inline bool tun_is_xdp_frame(void *ptr)
28{27{
@@ -56,8 +55,6 @@ static inline struct ptr_ring *tun_get_tx_ring(struct file *f) · 27 unmodified lines
56 return ERR_PTR(-EINVAL);55 return ERR_PTR(-EINVAL);
57}56}
58 57
59static inline void tun_wake_queue(struct file *f, int consumed) {}
60
61static inline bool tun_is_xdp_frame(void *ptr)58static inline bool tun_is_xdp_frame(void *ptr)
62{59{
63 return false;60 return false;
include/linux/ptr_ring.h+2−18
@@ -96,20 +96,6 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r) · 95 unmodified lines
96 return ret;96 return ret;
97}97}
98 98
99/* Note: callers invoking this in a loop must use a compiler barrier,
100 * for example cpu_relax(). Callers must hold producer_lock.
101 */
102static inline int __ptr_ring_check_produce(struct ptr_ring *r)
103{
104 if (unlikely(!r->size))
105 return -EINVAL;
106
107 if (data_race(r->queue[r->producer]))
108 return -ENOSPC;
109
110 return 0;
111}
112
113/* Note: callers invoking this in a loop must use a compiler barrier,99/* Note: callers invoking this in a loop must use a compiler barrier,
114 * for example cpu_relax(). Callers must hold producer_lock.100 * for example cpu_relax(). Callers must hold producer_lock.
115 * Callers are responsible for making sure pointer that is being queued101 * Callers are responsible for making sure pointer that is being queued
@@ -117,10 +103,8 @@ static inline int __ptr_ring_check_produce(struct ptr_ring *r) · 1 unmodified line
117 */103 */
118static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr)104static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr)
119{105{
120 int p = __ptr_ring_check_produce(r);106 if (unlikely(!r->size) || data_race(r->queue[r->producer]))
121 107 return -ENOSPC;
122 if (p)
123 return p;
124 108
125 /* Make sure the pointer we are storing points to a valid data. */109 /* Make sure the pointer we are storing points to a valid data. */
126 /* Pairs with the dependency ordering in __ptr_ring_consume. */110 /* Pairs with the dependency ordering in __ptr_ring_consume. */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t