| @@ -145,8 +145,6 @@ struct tun_file { · 144 unmodified lines |
| 145 | 145 | struct list_head next; |
| 146 | 146 | struct tun_struct *detached; |
| 147 | 147 | struct ptr_ring tx_ring; |
| 148 | | − /* Protected by tx_ring.consumer_lock */ |
| 149 | | − int cons_cnt; |
| 150 | 148 | struct xdp_rxq_info xdp_rxq; |
| 151 | 149 | }; |
| 152 | 150 | |
| @@ -590,13 +588,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean) · 437 unmodified lines |
| 590 | 588 | rcu_assign_pointer(tun->tfiles[index], |
| 591 | 589 | tun->tfiles[tun->numqueues - 1]); |
| 592 | 590 | ntfile = rtnl_dereference(tun->tfiles[index]); |
| 593 | | − spin_lock(&ntfile->tx_ring.consumer_lock); |
| 594 | 591 | ntfile->queue_index = index; |
| 595 | 592 | ntfile->xdp_rxq.queue_index = index; |
| 596 | | − ntfile->cons_cnt = 0; |
| 597 | | − if (__ptr_ring_empty(&ntfile->tx_ring)) |
| 598 | | − netif_wake_subqueue(tun->dev, index); |
| 599 | | − spin_unlock(&ntfile->tx_ring.consumer_lock); |
| 600 | 593 | rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], |
| 601 | 594 | NULL); |
| 602 | 595 | |
| @@ -737,9 +730,6 @@ static int tun_attach(struct tun_struct *tun, struct file *file, · 134 unmodified lines |
| 737 | 730 | goto out; |
| 738 | 731 | } |
| 739 | 732 | |
| 740 | | − spin_lock(&tfile->tx_ring.consumer_lock); |
| 741 | | − tfile->cons_cnt = 0; |
| 742 | | − spin_unlock(&tfile->tx_ring.consumer_lock); |
| 743 | 733 | tfile->queue_index = tun->numqueues; |
| 744 | 734 | tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; |
| 745 | 735 | |
| @@ -1018,7 +1008,6 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 272 unmodified lines |
| 1018 | 1008 | struct netdev_queue *queue; |
| 1019 | 1009 | struct tun_file *tfile; |
| 1020 | 1010 | int len = skb->len; |
| 1021 | | − int ret; |
| 1022 | 1011 | |
| 1023 | 1012 | rcu_read_lock(); |
| 1024 | 1013 | tfile = rcu_dereference(tun->tfiles[txq]); |
| @@ -1073,33 +1062,13 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 48 unmodified lines |
| 1073 | 1062 | |
| 1074 | 1063 | nf_reset_ct(skb); |
| 1075 | 1064 | |
| 1076 | | − queue = netdev_get_tx_queue(dev, txq); |
| 1077 | | − |
| 1078 | | − spin_lock(&tfile->tx_ring.producer_lock); |
| 1079 | | − ret = __ptr_ring_produce(&tfile->tx_ring, skb); |
| 1080 | | − if (!qdisc_txq_has_no_queue(queue) && |
| 1081 | | − __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) { |
| 1082 | | − netif_tx_stop_queue(queue); |
| 1083 | | − /* Paired with smp_mb() in __tun_wake_queue() */ |
| 1084 | | − smp_mb__after_atomic(); |
| 1085 | | − if (!__ptr_ring_check_produce(&tfile->tx_ring)) |
| 1086 | | − netif_tx_wake_queue(queue); |
| 1087 | | − } |
| 1088 | | − spin_unlock(&tfile->tx_ring.producer_lock); |
| 1089 | | − |
| 1090 | | − if (ret) { |
| 1091 | | − /* This should be a rare case if a qdisc is present, but |
| 1092 | | − * can happen due to lltx. |
| 1093 | | − * Since skb_tx_timestamp(), skb_orphan(), |
| 1094 | | − * run_ebpf_filter() and pskb_trim() could have tinkered |
| 1095 | | − * with the SKB, returning NETDEV_TX_BUSY is unsafe and |
| 1096 | | − * we must drop instead. |
| 1097 | | − */ |
| 1065 | + if (ptr_ring_produce(&tfile->tx_ring, skb)) { |
| 1098 | 1066 | drop_reason = SKB_DROP_REASON_FULL_RING; |
| 1099 | 1067 | goto drop; |
| 1100 | 1068 | } |
| 1101 | 1069 | |
| 1102 | 1070 | /* dev->lltx requires to do our own update of trans_start */ |
| 1071 | + queue = netdev_get_tx_queue(dev, txq); |
| 1103 | 1072 | txq_trans_cond_update(queue); |
| 1104 | 1073 | |
| 1105 | 1074 | /* Notify and wake up reader process */ |
| @@ -2147,46 +2116,13 @@ done: · 1041 unmodified lines |
| 2147 | 2116 | return total; |
| 2148 | 2117 | } |
| 2149 | 2118 | |
| 2150 | | −/* Callers must hold ring.consumer_lock */ |
| 2151 | | −static void __tun_wake_queue(struct tun_struct *tun, |
| 2152 | | − struct tun_file *tfile, int consumed) |
| 2153 | | −{ |
| 2154 | | − struct netdev_queue *txq = netdev_get_tx_queue(tun->dev, |
| 2155 | | − tfile->queue_index); |
| 2156 | | − |
| 2157 | | − /* Paired with smp_mb__after_atomic() in tun_net_xmit() */ |
| 2158 | | − smp_mb(); |
| 2159 | | − if (netif_tx_queue_stopped(txq)) { |
| 2160 | | − tfile->cons_cnt += consumed; |
| 2161 | | − if (tfile->cons_cnt >= tfile->tx_ring.size / 2 || |
| 2162 | | − __ptr_ring_empty(&tfile->tx_ring)) { |
| 2163 | | − netif_tx_wake_queue(txq); |
| 2164 | | − tfile->cons_cnt = 0; |
| 2165 | | − } |
| 2166 | | − } |
| 2167 | | −} |
| 2168 | | − |
| 2169 | | −static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile) |
| 2170 | | −{ |
| 2171 | | − void *ptr; |
| 2172 | | − |
| 2173 | | − spin_lock(&tfile->tx_ring.consumer_lock); |
| 2174 | | − ptr = __ptr_ring_consume(&tfile->tx_ring); |
| 2175 | | − if (ptr) |
| 2176 | | − __tun_wake_queue(tun, tfile, 1); |
| 2177 | | − |
| 2178 | | − spin_unlock(&tfile->tx_ring.consumer_lock); |
| 2179 | | − return ptr; |
| 2180 | | −} |
| 2181 | | − |
| 2182 | | −static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, |
| 2183 | | − int noblock, int *err) |
| 2119 | +static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) |
| 2184 | 2120 | { |
| 2185 | 2121 | DECLARE_WAITQUEUE(wait, current); |
| 2186 | 2122 | void *ptr = NULL; |
| 2187 | 2123 | int error = 0; |
| 2188 | 2124 | |
| 2189 | | − ptr = tun_ring_consume(tun, tfile); |
| 2125 | + ptr = ptr_ring_consume(&tfile->tx_ring); |
| 2190 | 2126 | if (ptr) |
| 2191 | 2127 | goto out; |
| 2192 | 2128 | if (noblock) { |
| @@ -2198,7 +2134,7 @@ static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, · 5 unmodified lines |
| 2198 | 2134 | |
| 2199 | 2135 | while (1) { |
| 2200 | 2136 | set_current_state(TASK_INTERRUPTIBLE); |
| 2201 | | − ptr = tun_ring_consume(tun, tfile); |
| 2137 | + ptr = ptr_ring_consume(&tfile->tx_ring); |
| 2202 | 2138 | if (ptr) |
| 2203 | 2139 | break; |
| 2204 | 2140 | if (signal_pending(current)) { |
| @@ -2235,7 +2171,7 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, · 30 unmodified lines |
| 2235 | 2171 | |
| 2236 | 2172 | if (!ptr) { |
| 2237 | 2173 | /* Read frames from ring */ |
| 2238 | | − ptr = tun_ring_recv(tun, tfile, noblock, &err); |
| 2174 | + ptr = tun_ring_recv(tfile, noblock, &err); |
| 2239 | 2175 | if (!ptr) |
| 2240 | 2176 | return err; |
| 2241 | 2177 | } |
| @@ -3690,16 +3626,6 @@ static int tun_queue_resize(struct tun_struct *tun) · 1448 unmodified lines |
| 3690 | 3626 | dev->tx_queue_len, GFP_KERNEL, |
| 3691 | 3627 | tun_ptr_free); |
| 3692 | 3628 | |
| 3693 | | − if (!ret) { |
| 3694 | | − for (i = 0; i < tun->numqueues; i++) { |
| 3695 | | − tfile = rtnl_dereference(tun->tfiles[i]); |
| 3696 | | − spin_lock(&tfile->tx_ring.consumer_lock); |
| 3697 | | − netif_wake_subqueue(tun->dev, tfile->queue_index); |
| 3698 | | − tfile->cons_cnt = 0; |
| 3699 | | − spin_unlock(&tfile->tx_ring.consumer_lock); |
| 3700 | | − } |
| 3701 | | − } |
| 3702 | | − |
| 3703 | 3629 | kfree(rings); |
| 3704 | 3630 | return ret; |
| 3705 | 3631 | } |
| @@ -3808,29 +3734,6 @@ struct ptr_ring *tun_get_tx_ring(struct file *file) · 102 unmodified lines |
| 3808 | 3734 | } |
| 3809 | 3735 | EXPORT_SYMBOL_GPL(tun_get_tx_ring); |
| 3810 | 3736 | |
| 3811 | | −/* Callers must hold ring.consumer_lock */ |
| 3812 | | −void tun_wake_queue(struct file *file, int consumed) |
| 3813 | | −{ |
| 3814 | | − struct tun_file *tfile; |
| 3815 | | − struct tun_struct *tun; |
| 3816 | | − |
| 3817 | | − if (file->f_op != &tun_fops) |
| 3818 | | − return; |
| 3819 | | − |
| 3820 | | − tfile = file->private_data; |
| 3821 | | − if (!tfile) |
| 3822 | | − return; |
| 3823 | | − |
| 3824 | | − rcu_read_lock(); |
| 3825 | | − |
| 3826 | | − tun = rcu_dereference(tfile->tun); |
| 3827 | | − if (tun) |
| 3828 | | − __tun_wake_queue(tun, tfile, consumed); |
| 3829 | | − |
| 3830 | | − rcu_read_unlock(); |
| 3831 | | −} |
| 3832 | | −EXPORT_SYMBOL_GPL(tun_wake_queue); |
| 3833 | | − |
| 3834 | 3737 | module_init(tun_init); |
| 3835 | 3738 | module_exit(tun_cleanup); |
| 3836 | 3739 | MODULE_DESCRIPTION(DRV_DESCRIPTION); |