When running iperf in a two host configuration the following trace can occur: [ 319.728730] NETDEV WATCHDOG: ib0 (hfi1): transmit queue 0 timed out The issue happens because the current implementation relies on the netif txq being stopped to control the flushing of the tx list. There are two resources that the transmit logic can wait on and stop the txq: - SDMA descriptors - Ring space to hold completions The ring space is tested on the sending side and relieved when the ring is consumed in the napi tx reaping. Unfortunately, that reaping can run conncurrently with the workqueue flushing of the txlist. If the txq is started just before the workitem executes, the txlist will never be flushed, leading to the txq being stuck. Fix by: - Adding sleep/wakeup wrappers * Use an atomic to control the call to the netif routines inside the wrappers - Use another atomic to record ring space exhaustion * Only wakeup when the a ring space exhaustion has happened and it relieved Add additional wrappers to clarify the ring space resource handling.
Unified · Split
| @@ -67,6 +67,9 @@ struct hfi1_ipoib_circ_buf { · 66 unmodified lines | ||
| 67 | 67 | * @sde: sdma engine |
| 68 | 68 | * @tx_list: tx request list |
| 69 | 69 | * @sent_txreqs: count of txreqs posted to sdma |
| 70 | + * @stops: count of stops of queue | |
| 71 | + * @ring_full: ring has been filled | |
| 72 | + * @no_desc: descriptor shortage seen | |
| 70 | 73 | * @flow: tracks when list needs to be flushed for a flow change |
| 71 | 74 | * @q_idx: ipoib Tx queue index |
| 72 | 75 | * @pkts_sent: indicator packets have been sent from this queue |
| @@ -80,6 +83,9 @@ struct hfi1_ipoib_txq { · 7 unmodified lines | ||
| 80 | 83 | struct sdma_engine *sde; |
| 81 | 84 | struct list_head tx_list; |
| 82 | 85 | u64 sent_txreqs; |
| 86 | + atomic_t stops; | |
| 87 | + atomic_t ring_full; | |
| 88 | + atomic_t no_desc; | |
| 83 | 89 | union hfi1_ipoib_flow flow; |
| 84 | 90 | u8 q_idx; |
| 85 | 91 | bool pkts_sent; |
| @@ -55,23 +55,48 @@ static u64 hfi1_ipoib_txreqs(const u64 sent, const u64 completed) · 54 unmodified lines | ||
| 55 | 55 | return sent - completed; |
| 56 | 56 | } |
| 57 | 57 | |
| 58 | −static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq) | |
| 58 | +static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq) | |
| 59 | +{ | |
| 60 | + return hfi1_ipoib_txreqs(txq->sent_txreqs, | |
| 61 | + atomic64_read(&txq->complete_txreqs)); | |
| 62 | +} | |
| 63 | + | |
| 64 | +static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq) | |
| 59 | 65 | { |
| 60 | − if (unlikely(hfi1_ipoib_txreqs(++txq->sent_txreqs, | |
| 61 | − atomic64_read(&txq->complete_txreqs)) >= | |
| 62 | − min_t(unsigned int, txq->priv->netdev->tx_queue_len, | |
| 63 | − txq->tx_ring.max_items - 1))) | |
| 66 | + if (atomic_inc_return(&txq->stops) == 1) | |
| 64 | 67 | netif_stop_subqueue(txq->priv->netdev, txq->q_idx); |
| 65 | 68 | } |
| 66 | 69 | |
| 70 | +static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq) | |
| 71 | +{ | |
| 72 | + if (atomic_dec_and_test(&txq->stops)) | |
| 73 | + netif_wake_subqueue(txq->priv->netdev, txq->q_idx); | |
| 74 | +} | |
| 75 | + | |
| 76 | +static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq) | |
| 77 | +{ | |
| 78 | + return min_t(uint, txq->priv->netdev->tx_queue_len, | |
| 79 | + txq->tx_ring.max_items - 1); | |
| 80 | +} | |
| 81 | + | |
| 82 | +static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq) | |
| 83 | +{ | |
| 84 | + return min_t(uint, txq->priv->netdev->tx_queue_len, | |
| 85 | + txq->tx_ring.max_items) >> 1; | |
| 86 | +} | |
| 87 | + | |
| 88 | +static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq) | |
| 89 | +{ | |
| 90 | + ++txq->sent_txreqs; | |
| 91 | + if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) && | |
| 92 | + !atomic_xchg(&txq->ring_full, 1)) | |
| 93 | + hfi1_ipoib_stop_txq(txq); | |
| 94 | +} | |
| 95 | + | |
| 67 | 96 | static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq) |
| 68 | 97 | { |
| 69 | 98 | struct net_device *dev = txq->priv->netdev; |
| 70 | 99 | |
| 71 | − /* If the queue is already running just return */ | |
| 72 | − if (likely(!__netif_subqueue_stopped(dev, txq->q_idx))) | |
| 73 | − return; | |
| 74 | − | |
| 75 | 100 | /* If shutting down just return as queue state is irrelevant */ |
| 76 | 101 | if (unlikely(dev->reg_state != NETREG_REGISTERED)) |
| 77 | 102 | return; |
| @@ -86,11 +111,9 @@ static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq) · 8 unmodified lines | ||
| 86 | 111 | * Use the minimum of the current tx_queue_len or the rings max txreqs |
| 87 | 112 | * to protect against ring overflow. |
| 88 | 113 | */ |
| 89 | − if (hfi1_ipoib_txreqs(txq->sent_txreqs, | |
| 90 | − atomic64_read(&txq->complete_txreqs)) | |
| 91 | − < min_t(unsigned int, dev->tx_queue_len, | |
| 92 | − txq->tx_ring.max_items) >> 1) | |
| 93 | − netif_wake_subqueue(dev, txq->q_idx); | |
| 114 | + if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) && | |
| 115 | + atomic_xchg(&txq->ring_full, 0)) | |
| 116 | + hfi1_ipoib_wake_txq(txq); | |
| 94 | 117 | } |
| 95 | 118 | |
| 96 | 119 | static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget) |
| @@ -608,13 +631,14 @@ static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde, · 511 unmodified lines | ||
| 608 | 631 | return -EAGAIN; |
| 609 | 632 | } |
| 610 | 633 | |
| 611 | − netif_stop_subqueue(txq->priv->netdev, txq->q_idx); | |
| 612 | − | |
| 613 | 634 | if (list_empty(&txreq->list)) |
| 614 | 635 | /* came from non-list submit */ |
| 615 | 636 | list_add_tail(&txreq->list, &txq->tx_list); |
| 616 | − if (list_empty(&txq->wait.list)) | |
| 637 | + if (list_empty(&txq->wait.list)) { | |
| 638 | + if (!atomic_xchg(&txq->no_desc, 1)) | |
| 639 | + hfi1_ipoib_stop_txq(txq); | |
| 617 | 640 | iowait_queue(pkts_sent, wait->iow, &sde->dmawait); |
| 641 | + } | |
| 618 | 642 | |
| 619 | 643 | write_sequnlock(&sde->waitlock); |
| 620 | 644 | return -EBUSY; |
| @@ -649,9 +673,9 @@ static void hfi1_ipoib_flush_txq(struct work_struct *work) · 28 unmodified lines | ||
| 649 | 673 | struct net_device *dev = txq->priv->netdev; |
| 650 | 674 | |
| 651 | 675 | if (likely(dev->reg_state == NETREG_REGISTERED) && |
| 652 | − likely(__netif_subqueue_stopped(dev, txq->q_idx)) && | |
| 653 | 676 | likely(!hfi1_ipoib_flush_tx_list(dev, txq))) |
| 654 | − netif_wake_subqueue(dev, txq->q_idx); | |
| 677 | + if (atomic_xchg(&txq->no_desc, 0)) | |
| 678 | + hfi1_ipoib_wake_txq(txq); | |
| 655 | 679 | } |
| 656 | 680 | |
| 657 | 681 | int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) |
| @@ -705,6 +729,9 @@ int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv) · 47 unmodified lines | ||
| 705 | 729 | txq->sde = NULL; |
| 706 | 730 | INIT_LIST_HEAD(&txq->tx_list); |
| 707 | 731 | atomic64_set(&txq->complete_txreqs, 0); |
| 732 | + atomic_set(&txq->stops, 0); | |
| 733 | + atomic_set(&txq->ring_full, 0); | |
| 734 | + atomic_set(&txq->no_desc, 0); | |
| 708 | 735 | txq->q_idx = i; |
| 709 | 736 | txq->flow.tx_queue = 0xff; |
| 710 | 737 | txq->flow.sc5 = 0xff; |
| @@ -770,7 +797,7 @@ static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq) · 59 unmodified lines | ||
| 770 | 797 | atomic64_inc(complete_txreqs); |
| 771 | 798 | } |
| 772 | 799 | |
| 773 | − if (hfi1_ipoib_txreqs(txq->sent_txreqs, atomic64_read(complete_txreqs))) | |
| 800 | + if (hfi1_ipoib_used(txq)) | |
| 774 | 801 | dd_dev_warn(txq->priv->dd, |
| 775 | 802 | "txq %d not empty found %llu requests\n", |
| 776 | 803 | txq->q_idx, |