History

net: stmmac: Rework coalesce timer and fix multi-queue races

This follows David Miller advice and tries to fix coalesce timer in multi-queue scenarios. We are now using per-queue coalesce values and per-queue TX timer. Coalesce timer default values was changed to 1ms and the coalesce frames to 25. Tested in B2B setup between XGMAC2 and GMAC5.

Jose Abreu committed 8 years ago

Commit 8fce3331 parent 5211da94 files · +146 −106Browse files

Unified · Split

4 files
  1. drivers/net/ethernet/stmicro/stmmac/common.h +2 −2
  2. drivers/net/ethernet/stmicro/stmmac/stmmac.h +12 −2
  3. drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +131 −102
  4. include/linux/stmmac.h +1 −0
drivers/net/ethernet/stmicro/stmmac/common.h+2−2
@@ -258,10 +258,10 @@ struct stmmac_safety_stats { · 257 unmodified lines
258258 #define MAX_DMA_RIWT 0xff
259259 #define MIN_DMA_RIWT 0x20
260260 /* Tx coalesce parameters */
261−#define STMMAC_COAL_TX_TIMER 40000
261+#define STMMAC_COAL_TX_TIMER 1000
262262 #define STMMAC_MAX_COAL_TX_TICK 100000
263263 #define STMMAC_TX_MAX_FRAMES 256
264−#define STMMAC_TX_FRAMES 64
264+#define STMMAC_TX_FRAMES 25
265265
266266 /* Packets types */
267267 enum packets_types {
drivers/net/ethernet/stmicro/stmmac/stmmac.h+12−2
@@ -48,6 +48,8 @@ struct stmmac_tx_info { · 47 unmodified lines
4848
4949 /* Frequently used values are kept adjacent for cache effect */
5050 struct stmmac_tx_queue {
51+ u32 tx_count_frames;
52+ struct timer_list txtimer;
5153 u32 queue_index;
5254 struct stmmac_priv *priv_data;
5355 struct dma_extended_desc *dma_etx ____cacheline_aligned_in_smp;
@@ -73,7 +75,14 @@ struct stmmac_rx_queue { · 19 unmodified lines
7375 u32 rx_zeroc_thresh;
7476 dma_addr_t dma_rx_phy;
7577 u32 rx_tail_addr;
78+};
79+
80+struct stmmac_channel {
7681 struct napi_struct napi ____cacheline_aligned_in_smp;
82+ struct stmmac_priv *priv_data;
83+ u32 index;
84+ int has_rx;
85+ int has_tx;
7786 };
7887
7988 struct stmmac_tc_entry {
@@ -109,14 +118,12 @@ struct stmmac_pps_cfg { · 29 unmodified lines
109118
110119 struct stmmac_priv {
111120 /* Frequently used values are kept adjacent for cache effect */
112− u32 tx_count_frames;
113121 u32 tx_coal_frames;
114122 u32 tx_coal_timer;
115123
116124 int tx_coalesce;
117125 int hwts_tx_en;
118126 bool tx_path_in_lpi_mode;
119− struct timer_list txtimer;
120127 bool tso;
121128
122129 unsigned int dma_buf_sz;
@@ -137,6 +144,9 @@ struct stmmac_priv { · 14 unmodified lines
137144 /* TX Queue */
138145 struct stmmac_tx_queue tx_queue[MTL_MAX_TX_QUEUES];
139146
147+ /* Generic channel for NAPI */
148+ struct stmmac_channel channel[STMMAC_CH_MAX];
149+
140150 bool oldlink;
141151 int speed;
142152 int oldduplex;
drivers/net/ethernet/stmicro/stmmac/stmmac_main.c+131−102
@@ -148,12 +148,14 @@ static void stmmac_verify_args(void) · 147 unmodified lines
148148 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
149149 {
150150 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
151+ u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
152+ u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
151153 u32 queue;
152154
153− for (queue = 0; queue < rx_queues_cnt; queue++) {
154− struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
155+ for (queue = 0; queue < maxq; queue++) {
156+ struct stmmac_channel *ch = &priv->channel[queue];
155157
156− napi_disable(&rx_q->napi);
158+ napi_disable(&ch->napi);
157159 }
158160 }
159161
@@ -164,12 +166,14 @@ static void stmmac_disable_all_queues(struct stmmac_priv *priv) · 4 unmodified lines
164166 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
165167 {
166168 u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
169+ u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
170+ u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
167171 u32 queue;
168172
169− for (queue = 0; queue < rx_queues_cnt; queue++) {
170− struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
173+ for (queue = 0; queue < maxq; queue++) {
174+ struct stmmac_channel *ch = &priv->channel[queue];
171175
172− napi_enable(&rx_q->napi);
176+ napi_enable(&ch->napi);
173177 }
174178 }
175179
@@ -1843,18 +1847,18 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) · 1667 unmodified lines
18431847 * @queue: TX queue index
18441848 * Description: it reclaims the transmit resources after transmission completes.
18451849 */
1846−static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue)
1850+static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
18471851 {
18481852 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
18491853 unsigned int bytes_compl = 0, pkts_compl = 0;
1850− unsigned int entry;
1854+ unsigned int entry, count = 0;
18511855
1852− netif_tx_lock(priv->dev);
1856+ __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
18531857
18541858 priv->xstats.tx_clean++;
18551859
18561860 entry = tx_q->dirty_tx;
1857− while (entry != tx_q->cur_tx) {
1861+ while ((entry != tx_q->cur_tx) && (count < budget)) {
18581862 struct sk_buff *skb = tx_q->tx_skbuff[entry];
18591863 struct dma_desc *p;
18601864 int status;
@@ -1870,6 +1874,8 @@ static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue) · 9 unmodified lines
18701874 if (unlikely(status & tx_dma_own))
18711875 break;
18721876
1877+ count++;
1878+
18731879 /* Make sure descriptor fields are read after reading
18741880 * the own bit.
18751881 */
@@ -1937,7 +1943,10 @@ static void stmmac_tx_clean(struct stmmac_priv *priv, u32 queue) · 61 unmodified lines
19371943 stmmac_enable_eee_mode(priv);
19381944 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
19391945 }
1940− netif_tx_unlock(priv->dev);
1946+
1947+ __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
1948+
1949+ return count;
19411950 }
19421951
19431952 /**
@@ -2020,6 +2029,33 @@ static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv) · 76 unmodified lines
20202029 return false;
20212030 }
20222031
2032+static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
2033+{
2034+ int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2035+ &priv->xstats, chan);
2036+ struct stmmac_channel *ch = &priv->channel[chan];
2037+ bool needs_work = false;
2038+
2039+ if ((status & handle_rx) && ch->has_rx) {
2040+ needs_work = true;
2041+ } else {
2042+ status &= ~handle_rx;
2043+ }
2044+
2045+ if ((status & handle_tx) && ch->has_tx) {
2046+ needs_work = true;
2047+ } else {
2048+ status &= ~handle_tx;
2049+ }
2050+
2051+ if (needs_work && napi_schedule_prep(&ch->napi)) {
2052+ stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
2053+ __napi_schedule(&ch->napi);
2054+ }
2055+
2056+ return status;
2057+}
2058+
20232059 /**
20242060 * stmmac_dma_interrupt - DMA ISR
20252061 * @priv: driver private structure
@@ -2034,57 +2070,14 @@ static void stmmac_dma_interrupt(struct stmmac_priv *priv) · 8 unmodified lines
20342070 u32 channels_to_check = tx_channel_count > rx_channel_count ?
20352071 tx_channel_count : rx_channel_count;
20362072 u32 chan;
2037− bool poll_scheduled = false;
20382073 int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
20392074
20402075 /* Make sure we never check beyond our status buffer. */
20412076 if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
20422077 channels_to_check = ARRAY_SIZE(status);
20432078
2044− /* Each DMA channel can be used for rx and tx simultaneously, yet
2045− * napi_struct is embedded in struct stmmac_rx_queue rather than in a
2046− * stmmac_channel struct.
2047− * Because of this, stmmac_poll currently checks (and possibly wakes)
2048− * all tx queues rather than just a single tx queue.
2049− */
20502079 for (chan = 0; chan < channels_to_check; chan++)
2051− status[chan] = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2052− &priv->xstats, chan);
2053−
2054− for (chan = 0; chan < rx_channel_count; chan++) {
2055− if (likely(status[chan] & handle_rx)) {
2056− struct stmmac_rx_queue *rx_q = &priv->rx_queue[chan];
2057−
2058− if (likely(napi_schedule_prep(&rx_q->napi))) {
2059− stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
2060− __napi_schedule(&rx_q->napi);
2061− poll_scheduled = true;
2062− }
2063− }
2064− }
2065−
2066− /* If we scheduled poll, we already know that tx queues will be checked.
2067− * If we didn't schedule poll, see if any DMA channel (used by tx) has a
2068− * completed transmission, if so, call stmmac_poll (once).
2069− */
2070− if (!poll_scheduled) {
2071− for (chan = 0; chan < tx_channel_count; chan++) {
2072− if (status[chan] & handle_tx) {
2073− /* It doesn't matter what rx queue we choose
2074− * here. We use 0 since it always exists.
2075− */
2076− struct stmmac_rx_queue *rx_q =
2077− &priv->rx_queue[0];
2078−
2079− if (likely(napi_schedule_prep(&rx_q->napi))) {
2080− stmmac_disable_dma_irq(priv,
2081− priv->ioaddr, chan);
2082− __napi_schedule(&rx_q->napi);
2083− }
2084− break;
2085− }
2086− }
2087− }
2080+ status[chan] = stmmac_napi_check(priv, chan);
20882081
20892082 for (chan = 0; chan < tx_channel_count; chan++) {
20902083 if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
@@ -2233,6 +2226,13 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv) · 142 unmodified lines
22332226 return ret;
22342227 }
22352228
2229+static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2230+{
2231+ struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2232+
2233+ mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2234+}
2235+
22362236 /**
22372237 * stmmac_tx_timer - mitigation sw timer for tx.
22382238 * @data: data pointer
@@ -2241,13 +2241,14 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv) · 2 unmodified lines
22412241 */
22422242 static void stmmac_tx_timer(struct timer_list *t)
22432243 {
2244− struct stmmac_priv *priv = from_timer(priv, t, txtimer);
2245− u32 tx_queues_count = priv->plat->tx_queues_to_use;
2246− u32 queue;
2244+ struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
2245+ struct stmmac_priv *priv = tx_q->priv_data;
2246+ struct stmmac_channel *ch;
2247+
2248+ ch = &priv->channel[tx_q->queue_index];
22472249
2248− /* let's scan all the tx queues */
2249− for (queue = 0; queue < tx_queues_count; queue++)
2250− stmmac_tx_clean(priv, queue);
2250+ if (likely(napi_schedule_prep(&ch->napi)))
2251+ __napi_schedule(&ch->napi);
22512252 }
22522253
22532254 /**
@@ -2260,11 +2261,17 @@ static void stmmac_tx_timer(struct timer_list *t) · 6 unmodified lines
22602261 */
22612262 static void stmmac_init_tx_coalesce(struct stmmac_priv *priv)
22622263 {
2264+ u32 tx_channel_count = priv->plat->tx_queues_to_use;
2265+ u32 chan;
2266+
22632267 priv->tx_coal_frames = STMMAC_TX_FRAMES;
22642268 priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2265− timer_setup(&priv->txtimer, stmmac_tx_timer, 0);
2266− priv->txtimer.expires = STMMAC_COAL_TIMER(priv->tx_coal_timer);
2267− add_timer(&priv->txtimer);
2269+
2270+ for (chan = 0; chan < tx_channel_count; chan++) {
2271+ struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2272+
2273+ timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
2274+ }
22682275 }
22692276
22702277 static void stmmac_set_rings_length(struct stmmac_priv *priv)
@@ -2592,6 +2599,7 @@ static void stmmac_hw_teardown(struct net_device *dev) · 321 unmodified lines
25922599 static int stmmac_open(struct net_device *dev)
25932600 {
25942601 struct stmmac_priv *priv = netdev_priv(dev);
2602+ u32 chan;
25952603 int ret;
25962604
25972605 stmmac_check_ether_addr(priv);
@@ -2688,7 +2696,9 @@ irq_error: · 90 unmodified lines
26882696 if (dev->phydev)
26892697 phy_stop(dev->phydev);
26902698
2691− del_timer_sync(&priv->txtimer);
2699+ for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2700+ del_timer_sync(&priv->tx_queue[chan].txtimer);
2701+
26922702 stmmac_hw_teardown(dev);
26932703 init_error:
26942704 free_dma_desc_resources(priv);
@@ -2708,6 +2718,7 @@ dma_desc_error: · 13 unmodified lines
27082718 static int stmmac_release(struct net_device *dev)
27092719 {
27102720 struct stmmac_priv *priv = netdev_priv(dev);
2721+ u32 chan;
27112722
27122723 if (priv->eee_enabled)
27132724 del_timer_sync(&priv->eee_ctrl_timer);
@@ -2722,7 +2733,8 @@ static int stmmac_release(struct net_device *dev) · 8 unmodified lines
27222733
27232734 stmmac_disable_all_queues(priv);
27242735
2725− del_timer_sync(&priv->txtimer);
2736+ for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2737+ del_timer_sync(&priv->tx_queue[chan].txtimer);
27262738
27272739 /* Free the IRQ lines */
27282740 free_irq(dev->irq, dev);
@@ -2936,14 +2948,13 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) · 207 unmodified lines
29362948 priv->xstats.tx_tso_nfrags += nfrags;
29372949
29382950 /* Manage tx mitigation */
2939− priv->tx_count_frames += nfrags + 1;
2940− if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
2941− mod_timer(&priv->txtimer,
2942− STMMAC_COAL_TIMER(priv->tx_coal_timer));
2943− } else {
2944− priv->tx_count_frames = 0;
2951+ tx_q->tx_count_frames += nfrags + 1;
2952+ if (priv->tx_coal_frames <= tx_q->tx_count_frames) {
29452953 stmmac_set_tx_ic(priv, desc);
29462954 priv->xstats.tx_set_ic_bit++;
2955+ tx_q->tx_count_frames = 0;
2956+ } else {
2957+ stmmac_tx_timer_arm(priv, queue);
29472958 }
29482959
29492960 skb_tx_timestamp(skb);
@@ -3146,14 +3157,13 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) · 196 unmodified lines
31463157 * This approach takes care about the fragments: desc is the first
31473158 * element in case of no SG.
31483159 */
3149− priv->tx_count_frames += nfrags + 1;
3150− if (likely(priv->tx_coal_frames > priv->tx_count_frames)) {
3151− mod_timer(&priv->txtimer,
3152− STMMAC_COAL_TIMER(priv->tx_coal_timer));
3153− } else {
3154− priv->tx_count_frames = 0;
3160+ tx_q->tx_count_frames += nfrags + 1;
3161+ if (priv->tx_coal_frames <= tx_q->tx_count_frames) {
31553162 stmmac_set_tx_ic(priv, desc);
31563163 priv->xstats.tx_set_ic_bit++;
3164+ tx_q->tx_count_frames = 0;
3165+ } else {
3166+ stmmac_tx_timer_arm(priv, queue);
31573167 }
31583168
31593169 skb_tx_timestamp(skb);
@@ -3199,6 +3209,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) · 39 unmodified lines
31993209 netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
32003210
32013211 stmmac_enable_dma_transmission(priv, priv->ioaddr);
3212+
32023213 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
32033214
32043215 return NETDEV_TX_OK;
@@ -3319,6 +3330,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) · 114 unmodified lines
33193330 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
33203331 {
33213332 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3333+ struct stmmac_channel *ch = &priv->channel[queue];
33223334 unsigned int entry = rx_q->cur_rx;
33233335 int coe = priv->hw->rx_csum;
33243336 unsigned int next_entry;
@@ -3491,7 +3503,7 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) · 166 unmodified lines
34913503 else
34923504 skb->ip_summed = CHECKSUM_UNNECESSARY;
34933505
3494− napi_gro_receive(&rx_q->napi, skb);
3506+ napi_gro_receive(&ch->napi, skb);
34953507
34963508 priv->dev->stats.rx_packets++;
34973509 priv->dev->stats.rx_bytes += frame_len;
@@ -3514,27 +3526,33 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) · 16 unmodified lines
35143526 * Description :
35153527 * To look at the incoming frames and clear the tx resources.
35163528 */
3517−static int stmmac_poll(struct napi_struct *napi, int budget)
3529+static int stmmac_napi_poll(struct napi_struct *napi, int budget)
35183530 {
3519− struct stmmac_rx_queue *rx_q =
3520− container_of(napi, struct stmmac_rx_queue, napi);
3521− struct stmmac_priv *priv = rx_q->priv_data;
3522− u32 tx_count = priv->plat->tx_queues_to_use;
3523− u32 chan = rx_q->queue_index;
3524− int work_done = 0;
3525− u32 queue;
3531+ struct stmmac_channel *ch =
3532+ container_of(napi, struct stmmac_channel, napi);
3533+ struct stmmac_priv *priv = ch->priv_data;
3534+ int work_done = 0, work_rem = budget;
3535+ u32 chan = ch->index;
35263536
35273537 priv->xstats.napi_poll++;
35283538
3529− /* check all the queues */
3530− for (queue = 0; queue < tx_count; queue++)
3531− stmmac_tx_clean(priv, queue);
3539+ if (ch->has_tx) {
3540+ int done = stmmac_tx_clean(priv, work_rem, chan);
35323541
3533− work_done = stmmac_rx(priv, budget, rx_q->queue_index);
3534− if (work_done < budget) {
3535− napi_complete_done(napi, work_done);
3536− stmmac_enable_dma_irq(priv, priv->ioaddr, chan);
3542+ work_done += done;
3543+ work_rem -= done;
3544+ }
3545+
3546+ if (ch->has_rx) {
3547+ int done = stmmac_rx(priv, work_rem, chan);
3548+
3549+ work_done += done;
3550+ work_rem -= done;
35373551 }
3552+
3553+ if (work_done < budget && napi_complete_done(napi, work_done))
3554+ stmmac_enable_dma_irq(priv, priv->ioaddr, chan);
3555+
35383556 return work_done;
35393557 }
35403558
@@ -4198,8 +4216,8 @@ int stmmac_dvr_probe(struct device *device, · 657 unmodified lines
41984216 {
41994217 struct net_device *ndev = NULL;
42004218 struct stmmac_priv *priv;
4219+ u32 queue, maxq;
42014220 int ret = 0;
4202− u32 queue;
42034221
42044222 ndev = alloc_etherdev_mqs(sizeof(struct stmmac_priv),
42054223 MTL_MAX_TX_QUEUES,
@@ -4322,11 +4340,22 @@ int stmmac_dvr_probe(struct device *device, · 116 unmodified lines
43224340 "Enable RX Mitigation via HW Watchdog Timer\n");
43234341 }
43244342
4325− for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4326− struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4343+ /* Setup channels NAPI */
4344+ maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
43274345
4328− netif_napi_add(ndev, &rx_q->napi, stmmac_poll,
4329− (8 * priv->plat->rx_queues_to_use));
4346+ for (queue = 0; queue < maxq; queue++) {
4347+ struct stmmac_channel *ch = &priv->channel[queue];
4348+
4349+ ch->priv_data = priv;
4350+ ch->index = queue;
4351+
4352+ if (queue < priv->plat->rx_queues_to_use)
4353+ ch->has_rx = true;
4354+ if (queue < priv->plat->tx_queues_to_use)
4355+ ch->has_tx = true;
4356+
4357+ netif_napi_add(ndev, &ch->napi, stmmac_napi_poll,
4358+ NAPI_POLL_WEIGHT);
43304359 }
43314360
43324361 mutex_init(&priv->lock);
@@ -4372,10 +4401,10 @@ error_netdev_register: · 39 unmodified lines
43724401 priv->hw->pcs != STMMAC_PCS_RTBI)
43734402 stmmac_mdio_unregister(ndev);
43744403 error_mdio_register:
4375− for (queue = 0; queue < priv->plat->rx_queues_to_use; queue++) {
4376− struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4404+ for (queue = 0; queue < maxq; queue++) {
4405+ struct stmmac_channel *ch = &priv->channel[queue];
43774406
4378− netif_napi_del(&rx_q->napi);
4407+ netif_napi_del(&ch->napi);
43794408 }
43804409 error_hw_init:
43814410 destroy_workqueue(priv->wq);
include/linux/stmmac.h+1−0
@@ -30,6 +30,7 @@ · 29 unmodified lines
3030
3131 #define MTL_MAX_RX_QUEUES 8
3232 #define MTL_MAX_TX_QUEUES 8
33+#define STMMAC_CH_MAX 8
3334
3435 #define STMMAC_RX_COE_NONE 0
3536 #define STMMAC_RX_COE_TYPE1 1