| @@ -145,8 +145,6 @@ struct tun_file { · 144 unmodified lines |
| 145 | struct list_head next; | 145 | struct list_head next; |
| 146 | struct tun_struct *detached; | 146 | struct tun_struct *detached; |
| 147 | struct ptr_ring tx_ring; | 147 | struct ptr_ring tx_ring; |
| 148 | /* Protected by tx_ring.consumer_lock */ | | |
| 149 | int cons_cnt; | | |
| 150 | struct xdp_rxq_info xdp_rxq; | 148 | struct xdp_rxq_info xdp_rxq; |
| 151 | }; | 149 | }; |
| 152 | | 150 | |
| @@ -590,13 +588,8 @@ static void __tun_detach(struct tun_file *tfile, bool clean) · 437 unmodified lines |
| 590 | rcu_assign_pointer(tun->tfiles[index], | 588 | rcu_assign_pointer(tun->tfiles[index], |
| 591 | tun->tfiles[tun->numqueues - 1]); | 589 | tun->tfiles[tun->numqueues - 1]); |
| 592 | ntfile = rtnl_dereference(tun->tfiles[index]); | 590 | ntfile = rtnl_dereference(tun->tfiles[index]); |
| 593 | spin_lock(&ntfile->tx_ring.consumer_lock); | | |
| 594 | ntfile->queue_index = index; | 591 | ntfile->queue_index = index; |
| 595 | ntfile->xdp_rxq.queue_index = index; | 592 | ntfile->xdp_rxq.queue_index = index; |
| 596 | ntfile->cons_cnt = 0; | | |
| 597 | if (__ptr_ring_empty(&ntfile->tx_ring)) | | |
| 598 | netif_wake_subqueue(tun->dev, index); | | |
| 599 | spin_unlock(&ntfile->tx_ring.consumer_lock); | | |
| 600 | rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], | 593 | rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], |
| 601 | NULL); | 594 | NULL); |
| 602 | | 595 | |
| @@ -737,9 +730,6 @@ static int tun_attach(struct tun_struct *tun, struct file *file, · 134 unmodified lines |
| 737 | goto out; | 730 | goto out; |
| 738 | } | 731 | } |
| 739 | | 732 | |
| 740 | spin_lock(&tfile->tx_ring.consumer_lock); | | |
| 741 | tfile->cons_cnt = 0; | | |
| 742 | spin_unlock(&tfile->tx_ring.consumer_lock); | | |
| 743 | tfile->queue_index = tun->numqueues; | 733 | tfile->queue_index = tun->numqueues; |
| 744 | tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; | 734 | tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; |
| 745 | | 735 | |
| @@ -1018,7 +1008,6 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 272 unmodified lines |
| 1018 | struct netdev_queue *queue; | 1008 | struct netdev_queue *queue; |
| 1019 | struct tun_file *tfile; | 1009 | struct tun_file *tfile; |
| 1020 | int len = skb->len; | 1010 | int len = skb->len; |
| 1021 | int ret; | | |
| 1022 | | 1011 | |
| 1023 | rcu_read_lock(); | 1012 | rcu_read_lock(); |
| 1024 | tfile = rcu_dereference(tun->tfiles[txq]); | 1013 | tfile = rcu_dereference(tun->tfiles[txq]); |
| @@ -1073,33 +1062,13 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) · 48 unmodified lines |
| 1073 | | 1062 | |
| 1074 | nf_reset_ct(skb); | 1063 | nf_reset_ct(skb); |
| 1075 | | 1064 | |
| 1076 | queue = netdev_get_tx_queue(dev, txq); | 1065 | if (ptr_ring_produce(&tfile->tx_ring, skb)) { |
| 1077 | | | |
| 1078 | spin_lock(&tfile->tx_ring.producer_lock); | | |
| 1079 | ret = __ptr_ring_produce(&tfile->tx_ring, skb); | | |
| 1080 | if (!qdisc_txq_has_no_queue(queue) && | | |
| 1081 | __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) { | | |
| 1082 | netif_tx_stop_queue(queue); | | |
| 1083 | /* Paired with smp_mb() in __tun_wake_queue() */ | | |
| 1084 | smp_mb__after_atomic(); | | |
| 1085 | if (!__ptr_ring_check_produce(&tfile->tx_ring)) | | |
| 1086 | netif_tx_wake_queue(queue); | | |
| 1087 | } | | |
| 1088 | spin_unlock(&tfile->tx_ring.producer_lock); | | |
| 1089 | | | |
| 1090 | if (ret) { | | |
| 1091 | /* This should be a rare case if a qdisc is present, but | | |
| 1092 | * can happen due to lltx. | | |
| 1093 | * Since skb_tx_timestamp(), skb_orphan(), | | |
| 1094 | * run_ebpf_filter() and pskb_trim() could have tinkered | | |
| 1095 | * with the SKB, returning NETDEV_TX_BUSY is unsafe and | | |
| 1096 | * we must drop instead. | | |
| 1097 | */ | | |
| 1098 | drop_reason = SKB_DROP_REASON_FULL_RING; | 1066 | drop_reason = SKB_DROP_REASON_FULL_RING; |
| 1099 | goto drop; | 1067 | goto drop; |
| 1100 | } | 1068 | } |
| 1101 | | 1069 | |
| 1102 | /* dev->lltx requires to do our own update of trans_start */ | 1070 | /* dev->lltx requires to do our own update of trans_start */ |
| | 1071 | queue = netdev_get_tx_queue(dev, txq); |
| 1103 | txq_trans_cond_update(queue); | 1072 | txq_trans_cond_update(queue); |
| 1104 | | 1073 | |
| 1105 | /* Notify and wake up reader process */ | 1074 | /* Notify and wake up reader process */ |
| @@ -2147,46 +2116,13 @@ done: · 1041 unmodified lines |
| 2147 | return total; | 2116 | return total; |
| 2148 | } | 2117 | } |
| 2149 | | 2118 | |
| 2150 | /* Callers must hold ring.consumer_lock */ | 2119 | static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) |
| 2151 | static void __tun_wake_queue(struct tun_struct *tun, | | |
| 2152 | struct tun_file *tfile, int consumed) | | |
| 2153 | { | | |
| 2154 | struct netdev_queue *txq = netdev_get_tx_queue(tun->dev, | | |
| 2155 | tfile->queue_index); | | |
| 2156 | | | |
| 2157 | /* Paired with smp_mb__after_atomic() in tun_net_xmit() */ | | |
| 2158 | smp_mb(); | | |
| 2159 | if (netif_tx_queue_stopped(txq)) { | | |
| 2160 | tfile->cons_cnt += consumed; | | |
| 2161 | if (tfile->cons_cnt >= tfile->tx_ring.size / 2 || | | |
| 2162 | __ptr_ring_empty(&tfile->tx_ring)) { | | |
| 2163 | netif_tx_wake_queue(txq); | | |
| 2164 | tfile->cons_cnt = 0; | | |
| 2165 | } | | |
| 2166 | } | | |
| 2167 | } | | |
| 2168 | | | |
| 2169 | static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile) | | |
| 2170 | { | | |
| 2171 | void *ptr; | | |
| 2172 | | | |
| 2173 | spin_lock(&tfile->tx_ring.consumer_lock); | | |
| 2174 | ptr = __ptr_ring_consume(&tfile->tx_ring); | | |
| 2175 | if (ptr) | | |
| 2176 | __tun_wake_queue(tun, tfile, 1); | | |
| 2177 | | | |
| 2178 | spin_unlock(&tfile->tx_ring.consumer_lock); | | |
| 2179 | return ptr; | | |
| 2180 | } | | |
| 2181 | | | |
| 2182 | static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, | | |
| 2183 | int noblock, int *err) | | |
| 2184 | { | 2120 | { |
| 2185 | DECLARE_WAITQUEUE(wait, current); | 2121 | DECLARE_WAITQUEUE(wait, current); |
| 2186 | void *ptr = NULL; | 2122 | void *ptr = NULL; |
| 2187 | int error = 0; | 2123 | int error = 0; |
| 2188 | | 2124 | |
| 2189 | ptr = tun_ring_consume(tun, tfile); | 2125 | ptr = ptr_ring_consume(&tfile->tx_ring); |
| 2190 | if (ptr) | 2126 | if (ptr) |
| 2191 | goto out; | 2127 | goto out; |
| 2192 | if (noblock) { | 2128 | if (noblock) { |
| @@ -2198,7 +2134,7 @@ static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, · 5 unmodified lines |
| 2198 | | 2134 | |
| 2199 | while (1) { | 2135 | while (1) { |
| 2200 | set_current_state(TASK_INTERRUPTIBLE); | 2136 | set_current_state(TASK_INTERRUPTIBLE); |
| 2201 | ptr = tun_ring_consume(tun, tfile); | 2137 | ptr = ptr_ring_consume(&tfile->tx_ring); |
| 2202 | if (ptr) | 2138 | if (ptr) |
| 2203 | break; | 2139 | break; |
| 2204 | if (signal_pending(current)) { | 2140 | if (signal_pending(current)) { |
| @@ -2235,7 +2171,7 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, · 30 unmodified lines |
| 2235 | | 2171 | |
| 2236 | if (!ptr) { | 2172 | if (!ptr) { |
| 2237 | /* Read frames from ring */ | 2173 | /* Read frames from ring */ |
| 2238 | ptr = tun_ring_recv(tun, tfile, noblock, &err); | 2174 | ptr = tun_ring_recv(tfile, noblock, &err); |
| 2239 | if (!ptr) | 2175 | if (!ptr) |
| 2240 | return err; | 2176 | return err; |
| 2241 | } | 2177 | } |
| @@ -3690,16 +3626,6 @@ static int tun_queue_resize(struct tun_struct *tun) · 1448 unmodified lines |
| 3690 | dev->tx_queue_len, GFP_KERNEL, | 3626 | dev->tx_queue_len, GFP_KERNEL, |
| 3691 | tun_ptr_free); | 3627 | tun_ptr_free); |
| 3692 | | 3628 | |
| 3693 | if (!ret) { | | |
| 3694 | for (i = 0; i < tun->numqueues; i++) { | | |
| 3695 | tfile = rtnl_dereference(tun->tfiles[i]); | | |
| 3696 | spin_lock(&tfile->tx_ring.consumer_lock); | | |
| 3697 | netif_wake_subqueue(tun->dev, tfile->queue_index); | | |
| 3698 | tfile->cons_cnt = 0; | | |
| 3699 | spin_unlock(&tfile->tx_ring.consumer_lock); | | |
| 3700 | } | | |
| 3701 | } | | |
| 3702 | | | |
| 3703 | kfree(rings); | 3629 | kfree(rings); |
| 3704 | return ret; | 3630 | return ret; |
| 3705 | } | 3631 | } |
| @@ -3808,29 +3734,6 @@ struct ptr_ring *tun_get_tx_ring(struct file *file) · 102 unmodified lines |
| 3808 | } | 3734 | } |
| 3809 | EXPORT_SYMBOL_GPL(tun_get_tx_ring); | 3735 | EXPORT_SYMBOL_GPL(tun_get_tx_ring); |
| 3810 | | 3736 | |
| 3811 | /* Callers must hold ring.consumer_lock */ | | |
| 3812 | void tun_wake_queue(struct file *file, int consumed) | | |
| 3813 | { | | |
| 3814 | struct tun_file *tfile; | | |
| 3815 | struct tun_struct *tun; | | |
| 3816 | | | |
| 3817 | if (file->f_op != &tun_fops) | | |
| 3818 | return; | | |
| 3819 | | | |
| 3820 | tfile = file->private_data; | | |
| 3821 | if (!tfile) | | |
| 3822 | return; | | |
| 3823 | | | |
| 3824 | rcu_read_lock(); | | |
| 3825 | | | |
| 3826 | tun = rcu_dereference(tfile->tun); | | |
| 3827 | if (tun) | | |
| 3828 | __tun_wake_queue(tun, tfile, consumed); | | |
| 3829 | | | |
| 3830 | rcu_read_unlock(); | | |
| 3831 | } | | |
| 3832 | EXPORT_SYMBOL_GPL(tun_wake_queue); | | |
| 3833 | | | |
| 3834 | module_init(tun_init); | 3737 | module_init(tun_init); |
| 3835 | module_exit(tun_cleanup); | 3738 | module_exit(tun_cleanup); |
| 3836 | MODULE_DESCRIPTION(DRV_DESCRIPTION); | 3739 | MODULE_DESCRIPTION(DRV_DESCRIPTION); |