History

Merge branch 'revert-tun-tap-vhost-net-apply-qdisc-backpressure-on-full-ptr_ring-to-reduce-tx-drops'

Simon Schippers says: ==================== Revert "tun/tap & vhost-net: apply qdisc backpressure on full ptr_ring to reduce TX drops" Commit 1d6e569b7d0c ("tun/tap & vhost-net: avoid ptr_ring tail-drop when a qdisc is present") did not show a relevant performance regression in my testing, but on Brett Sheffield's librecast testbed it causes a significant throughput drop in an IPv6 multicast testcase. The regression can be pinpointed to multiple iperf3 TCP threads sending: for 8 threads the throughput dropped from 13.5 Gbit/s to 9.13 Gbit/s. Therefore this series reverts the qdisc backpressure work. Making the backpressure opt-in via a new IFF_BACKPRESSURE flag was proposed in [1], but a new IFF_* flag needs more review scrutiny than is available at the moment, so a revert was requested instead. The opt-in will be resubmitted for net-next later. [1] Link: https://lore.kernel.org/netdev/20260709095511.168235-1-simon.schippers@tu-dortmund.de/ Reported-by: Brett Sheffield <brett@librecast.net> Closes: https://lore.kernel.org/netdev/akVnoOYQOrt8k-Gu@karahi.librecast.net/ ====================

Jakub Kicinski committed 3 months ago

Commit 7e107ab2 parents 82f0e98 c3da92a4 files · +14 −139Browse files

Unified · Split

4 files
  1. drivers/net/tun.c +6 −103
  2. drivers/vhost/net.c +6 −15
  3. include/linux/if_tun.h +0 −3
  4. include/linux/ptr_ring.h +2 −18
drivers/net/tun.c+6−103
@@ -145,8 +145,6 @@ struct tun_file { · 144 unmodified lines
145145 struct list_head next;
146146 struct tun_struct *detached;
147147 struct ptr_ring tx_ring;
148− /* Protected by tx_ring.consumer_lock */
149− int cons_cnt;
150148 struct xdp_rxq_info xdp_rxq;
151149 };
152150
@@ -590,13 +588,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean) · 437 unmodified lines
590588 rcu_assign_pointer(tun->tfiles[index],
591589 tun->tfiles[tun->numqueues - 1]);
592590 ntfile = rtnl_dereference(tun->tfiles[index]);
593− spin_lock(&ntfile->tx_ring.consumer_lock);
594591 ntfile->queue_index = index;
595592 ntfile->xdp_rxq.queue_index = index;
596− ntfile->cons_cnt = 0;
597− if (__ptr_ring_empty(&ntfile->tx_ring))
598− netif_wake_subqueue(tun->dev, index);
599− spin_unlock(&ntfile->tx_ring.consumer_lock);
600593 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
601594 NULL);
602595
@@ -737,9 +730,6 @@ static int tun_attach(struct tun_struct *tun, struct file *file, · 134 unmodified lines
737730 goto out;
738731 }
739732
740− spin_lock(&tfile->tx_ring.consumer_lock);
741− tfile->cons_cnt = 0;
742− spin_unlock(&tfile->tx_ring.consumer_lock);
743733 tfile->queue_index = tun->numqueues;
744734 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
745735
@@ -1018,7 +1008,6 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 272 unmodified lines
10181008 struct netdev_queue *queue;
10191009 struct tun_file *tfile;
10201010 int len = skb->len;
1021− int ret;
10221011
10231012 rcu_read_lock();
10241013 tfile = rcu_dereference(tun->tfiles[txq]);
@@ -1073,33 +1062,13 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 48 unmodified lines
10731062
10741063 nf_reset_ct(skb);
10751064
1076− queue = netdev_get_tx_queue(dev, txq);
1077−
1078− spin_lock(&tfile->tx_ring.producer_lock);
1079− ret = __ptr_ring_produce(&tfile->tx_ring, skb);
1080− if (!qdisc_txq_has_no_queue(queue) &&
1081− __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) {
1082− netif_tx_stop_queue(queue);
1083− /* Paired with smp_mb() in __tun_wake_queue() */
1084− smp_mb__after_atomic();
1085− if (!__ptr_ring_check_produce(&tfile->tx_ring))
1086− netif_tx_wake_queue(queue);
1087− }
1088− spin_unlock(&tfile->tx_ring.producer_lock);
1089−
1090− if (ret) {
1091− /* This should be a rare case if a qdisc is present, but
1092− * can happen due to lltx.
1093− * Since skb_tx_timestamp(), skb_orphan(),
1094− * run_ebpf_filter() and pskb_trim() could have tinkered
1095− * with the SKB, returning NETDEV_TX_BUSY is unsafe and
1096− * we must drop instead.
1097− */
1065+ if (ptr_ring_produce(&tfile->tx_ring, skb)) {
10981066 drop_reason = SKB_DROP_REASON_FULL_RING;
10991067 goto drop;
11001068 }
11011069
11021070 /* dev->lltx requires to do our own update of trans_start */
1071+ queue = netdev_get_tx_queue(dev, txq);
11031072 txq_trans_cond_update(queue);
11041073
11051074 /* Notify and wake up reader process */
@@ -2147,46 +2116,13 @@ done: · 1041 unmodified lines
21472116 return total;
21482117 }
21492118
2150−/* Callers must hold ring.consumer_lock */
2151−static void __tun_wake_queue(struct tun_struct *tun,
2152− struct tun_file *tfile, int consumed)
2153−{
2154− struct netdev_queue *txq = netdev_get_tx_queue(tun->dev,
2155− tfile->queue_index);
2156−
2157− /* Paired with smp_mb__after_atomic() in tun_net_xmit() */
2158− smp_mb();
2159− if (netif_tx_queue_stopped(txq)) {
2160− tfile->cons_cnt += consumed;
2161− if (tfile->cons_cnt >= tfile->tx_ring.size / 2 ||
2162− __ptr_ring_empty(&tfile->tx_ring)) {
2163− netif_tx_wake_queue(txq);
2164− tfile->cons_cnt = 0;
2165− }
2166− }
2167−}
2168−
2169−static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile)
2170−{
2171− void *ptr;
2172−
2173− spin_lock(&tfile->tx_ring.consumer_lock);
2174− ptr = __ptr_ring_consume(&tfile->tx_ring);
2175− if (ptr)
2176− __tun_wake_queue(tun, tfile, 1);
2177−
2178− spin_unlock(&tfile->tx_ring.consumer_lock);
2179− return ptr;
2180−}
2181−
2182−static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile,
2183− int noblock, int *err)
2119+static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
21842120 {
21852121 DECLARE_WAITQUEUE(wait, current);
21862122 void *ptr = NULL;
21872123 int error = 0;
21882124
2189− ptr = tun_ring_consume(tun, tfile);
2125+ ptr = ptr_ring_consume(&tfile->tx_ring);
21902126 if (ptr)
21912127 goto out;
21922128 if (noblock) {
@@ -2198,7 +2134,7 @@ static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, · 5 unmodified lines
21982134
21992135 while (1) {
22002136 set_current_state(TASK_INTERRUPTIBLE);
2201− ptr = tun_ring_consume(tun, tfile);
2137+ ptr = ptr_ring_consume(&tfile->tx_ring);
22022138 if (ptr)
22032139 break;
22042140 if (signal_pending(current)) {
@@ -2235,7 +2171,7 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, · 30 unmodified lines
22352171
22362172 if (!ptr) {
22372173 /* Read frames from ring */
2238− ptr = tun_ring_recv(tun, tfile, noblock, &err);
2174+ ptr = tun_ring_recv(tfile, noblock, &err);
22392175 if (!ptr)
22402176 return err;
22412177 }
@@ -3690,16 +3626,6 @@ static int tun_queue_resize(struct tun_struct *tun) · 1448 unmodified lines
36903626 dev->tx_queue_len, GFP_KERNEL,
36913627 tun_ptr_free);
36923628
3693− if (!ret) {
3694− for (i = 0; i < tun->numqueues; i++) {
3695− tfile = rtnl_dereference(tun->tfiles[i]);
3696− spin_lock(&tfile->tx_ring.consumer_lock);
3697− netif_wake_subqueue(tun->dev, tfile->queue_index);
3698− tfile->cons_cnt = 0;
3699− spin_unlock(&tfile->tx_ring.consumer_lock);
3700− }
3701− }
3702−
37033629 kfree(rings);
37043630 return ret;
37053631 }
@@ -3808,29 +3734,6 @@ struct ptr_ring *tun_get_tx_ring(struct file *file) · 102 unmodified lines
38083734 }
38093735 EXPORT_SYMBOL_GPL(tun_get_tx_ring);
38103736
3811−/* Callers must hold ring.consumer_lock */
3812−void tun_wake_queue(struct file *file, int consumed)
3813−{
3814− struct tun_file *tfile;
3815− struct tun_struct *tun;
3816−
3817− if (file->f_op != &tun_fops)
3818− return;
3819−
3820− tfile = file->private_data;
3821− if (!tfile)
3822− return;
3823−
3824− rcu_read_lock();
3825−
3826− tun = rcu_dereference(tfile->tun);
3827− if (tun)
3828− __tun_wake_queue(tun, tfile, consumed);
3829−
3830− rcu_read_unlock();
3831−}
3832−EXPORT_SYMBOL_GPL(tun_wake_queue);
3833−
38343737 module_init(tun_init);
38353738 module_exit(tun_cleanup);
38363739 MODULE_DESCRIPTION(DRV_DESCRIPTION);
drivers/vhost/net.c+6−15
@@ -176,21 +176,13 @@ static void *vhost_net_buf_consume(struct vhost_net_buf *rxq) · 175 unmodified lines
176176 return ret;
177177 }
178178
179−static int vhost_net_buf_produce(struct sock *sk,
180− struct vhost_net_virtqueue *nvq)
179+static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
181180 {
182− struct file *file = sk->sk_socket->file;
183181 struct vhost_net_buf *rxq = &nvq->rxq;
184182
185183 rxq->head = 0;
186− spin_lock(&nvq->rx_ring->consumer_lock);
187− rxq->tail = __ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
188− VHOST_NET_BATCH);
189−
190− if (rxq->tail)
191− tun_wake_queue(file, rxq->tail);
192−
193− spin_unlock(&nvq->rx_ring->consumer_lock);
184+ rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
185+ VHOST_NET_BATCH);
194186 return rxq->tail;
195187 }
196188
@@ -217,15 +209,14 @@ static int vhost_net_buf_peek_len(void *ptr) · 20 unmodified lines
217209 return __skb_array_len_with_tag(ptr);
218210 }
219211
220−static int vhost_net_buf_peek(struct sock *sk,
221− struct vhost_net_virtqueue *nvq)
212+static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
222213 {
223214 struct vhost_net_buf *rxq = &nvq->rxq;
224215
225216 if (!vhost_net_buf_is_empty(rxq))
226217 goto out;
227218
228− if (!vhost_net_buf_produce(sk, nvq))
219+ if (!vhost_net_buf_produce(nvq))
229220 return 0;
230221
231222 out:
@@ -1013,7 +1004,7 @@ static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk) · 781 unmodified lines
10131004 unsigned long flags;
10141005
10151006 if (rvq->rx_ring)
1016− return vhost_net_buf_peek(sk, rvq);
1007+ return vhost_net_buf_peek(rvq);
10171008
10181009 spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
10191010 head = skb_peek(&sk->sk_receive_queue);
include/linux/if_tun.h+0−3
@@ -22,7 +22,6 @@ struct tun_msg_ctl { · 21 unmodified lines
2222 #if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE)
2323 struct socket *tun_get_socket(struct file *);
2424 struct ptr_ring *tun_get_tx_ring(struct file *file);
25−void tun_wake_queue(struct file *file, int consumed);
2625
2726 static inline bool tun_is_xdp_frame(void *ptr)
2827 {
@@ -56,8 +55,6 @@ static inline struct ptr_ring *tun_get_tx_ring(struct file *f) · 27 unmodified lines
5655 return ERR_PTR(-EINVAL);
5756 }
5857
59−static inline void tun_wake_queue(struct file *f, int consumed) {}
60−
6158 static inline bool tun_is_xdp_frame(void *ptr)
6259 {
6360 return false;
include/linux/ptr_ring.h+2−18
@@ -96,20 +96,6 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r) · 95 unmodified lines
9696 return ret;
9797 }
9898
99−/* Note: callers invoking this in a loop must use a compiler barrier,
100− * for example cpu_relax(). Callers must hold producer_lock.
101− */
102−static inline int __ptr_ring_check_produce(struct ptr_ring *r)
103−{
104− if (unlikely(!r->size))
105− return -EINVAL;
106−
107− if (data_race(r->queue[r->producer]))
108− return -ENOSPC;
109−
110− return 0;
111−}
112−
11399 /* Note: callers invoking this in a loop must use a compiler barrier,
114100 * for example cpu_relax(). Callers must hold producer_lock.
115101 * Callers are responsible for making sure pointer that is being queued
@@ -117,10 +103,8 @@ static inline int __ptr_ring_check_produce(struct ptr_ring *r) · 1 unmodified line
117103 */
118104 static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr)
119105 {
120− int p = __ptr_ring_check_produce(r);
121−
122− if (p)
123− return p;
106+ if (unlikely(!r->size) || data_race(r->queue[r->producer]))
107+ return -ENOSPC;
124108
125109 /* Make sure the pointer we are storing points to a valid data. */
126110 /* Pairs with the dependency ordering in __ptr_ring_consume. */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t