History

qede: sync udp_tunnel ports outside qede_lock in the recovery path

A TX timeout on a qede NIC that has VXLAN/GENEVE tunnel ports configured wedges the rtnetlink control plane of the whole machine: NETDEV WATCHDOG: ens6f1 (qede): transmit queue 2 timed out 10226 ms [qede_tx_timeout:586(ens6f1)]TX timeout on queue 2! [qede_recovery_handler:2665(ens6f0)]Starting a recovery process The recovery path deadlocks on the driver's own mutex: qede_sp_task rtnl_lock() mutex_lock(&edev->qede_lock) <- taken qede_recovery_handler qede_load udp_tunnel_nic_reset_ntf __udp_tunnel_nic_device_sync info->sync_table == qede_udp_tunnel_sync mutex_lock(&edev->qede_lock) <- same task: deadlock The mutex is not recursive, so the kworker blocks on itself with rtnl_lock held, and neither lock is ever released. Every task that calls rtnl_lock() afterwards (ip, ovs-vswitchd, lldpad, IPv6 addrconf, sshd) blocks forever while the node still answers ping. In a vmcore from an affected production node rtnl_mutex.owner decodes to the very kworker blocked at the innermost mutex_lock() above. Re-sync the tunnel ports from qede_sp_task() after the internal lock is dropped, still under rtnl_lock as the udp_tunnel API requires. This mirrors qede_open(), which calls udp_tunnel_nic_reset_ntf() under rtnl without the internal lock. qede_recovery_handler() now returns whether it has successfully reloaded an open device, and the caller re-syncs the ports only in that case. This keeps the old gating exactly: a device that was down or a failed recovery returns false, as those paths never reached the udp_tunnel_nic_reset_ntf() call before either. This was the only user of the qede_lock()/qede_unlock() helpers, so remove them.

Denis V. Lunev committed 3 months ago

Commit 451c9071 parent c92922c1 file · +22 −22Browse files

Unified · Split

drivers/net/ethernet/qlogic/qede/qede_main.c+22−22
@@ -107,7 +107,7 @@ static void qede_remove(struct pci_dev *pdev); · 106 unmodified lines
107static void qede_shutdown(struct pci_dev *pdev);107static void qede_shutdown(struct pci_dev *pdev);
108static void qede_link_update(void *dev, struct qed_link_output *link);108static void qede_link_update(void *dev, struct qed_link_output *link);
109static void qede_schedule_recovery_handler(void *dev);109static void qede_schedule_recovery_handler(void *dev);
110static void qede_recovery_handler(struct qede_dev *edev);110static bool qede_recovery_handler(struct qede_dev *edev);
111static void qede_schedule_hw_err_handler(void *dev,111static void qede_schedule_hw_err_handler(void *dev,
112 enum qed_hw_err_type err_type);112 enum qed_hw_err_type err_type);
113static void qede_get_eth_tlv_data(void *edev, void *data);113static void qede_get_eth_tlv_data(void *edev, void *data);
@@ -1043,21 +1043,6 @@ void __qede_unlock(struct qede_dev *edev) · 929 unmodified lines
1043 mutex_unlock(&edev->qede_lock);1043 mutex_unlock(&edev->qede_lock);
1044}1044}
1045 1045
1046/* This version of the lock should be used when acquiring the RTNL lock is also
1047 * needed in addition to the internal qede lock.
1048 */
1049static void qede_lock(struct qede_dev *edev)
1050{
1051 rtnl_lock();
1052 __qede_lock(edev);
1053}
1054
1055static void qede_unlock(struct qede_dev *edev)
1056{
1057 __qede_unlock(edev);
1058 rtnl_unlock();
1059}
1060
1061static void qede_periodic_task(struct work_struct *work)1046static void qede_periodic_task(struct work_struct *work)
1062{1047{
1063 struct qede_dev *edev = container_of(work, struct qede_dev,1048 struct qede_dev *edev = container_of(work, struct qede_dev,
@@ -1094,6 +1079,8 @@ static void qede_sp_task(struct work_struct *work) · 30 unmodified lines
1094 */1079 */
1095 1080
1096 if (test_and_clear_bit(QEDE_SP_RECOVERY, &edev->sp_flags)) {1081 if (test_and_clear_bit(QEDE_SP_RECOVERY, &edev->sp_flags)) {
1082 bool reloaded;
1083
1097 cancel_delayed_work_sync(&edev->periodic_task);1084 cancel_delayed_work_sync(&edev->periodic_task);
1098#ifdef CONFIG_QED_SRIOV1085#ifdef CONFIG_QED_SRIOV
1099 /* SRIOV must be disabled outside the lock to avoid a deadlock.1086 /* SRIOV must be disabled outside the lock to avoid a deadlock.
@@ -1102,9 +1089,17 @@ static void qede_sp_task(struct work_struct *work) · 2 unmodified lines
1102 if (pci_num_vf(edev->pdev))1089 if (pci_num_vf(edev->pdev))
1103 qede_sriov_configure(edev->pdev, 0);1090 qede_sriov_configure(edev->pdev, 0);
1104#endif1091#endif
1105 qede_lock(edev);1092 rtnl_lock();
1106 qede_recovery_handler(edev);1093 __qede_lock(edev);
1107 qede_unlock(edev);1094 reloaded = qede_recovery_handler(edev);
1095 __qede_unlock(edev);
1096
1097 /* The udp_tunnel core synchronously calls back into
1098 * qede_udp_tunnel_sync(), which takes the qede lock.
1099 */
1100 if (reloaded)
1101 udp_tunnel_nic_reset_ntf(edev->ndev);
1102 rtnl_unlock();
1108 }1103 }
1109 1104
1110 __qede_lock(edev);1105 __qede_lock(edev);
@@ -2645,9 +2640,13 @@ static void qede_recovery_failed(struct qede_dev *edev) · 1534 unmodified lines
2645 edev->ops->common->set_power_state(edev->cdev, PCI_D3hot);2640 edev->ops->common->set_power_state(edev->cdev, PCI_D3hot);
2646}2641}
2647 2642
2648static void qede_recovery_handler(struct qede_dev *edev)2643/* Returns true if an open device was successfully reloaded and its
2644 * udp_tunnel ports need to be re-synced by the caller.
2645 */
2646static bool qede_recovery_handler(struct qede_dev *edev)
2649{2647{
2650 u32 curr_state = edev->state;2648 u32 curr_state = edev->state;
2649 bool reloaded = false;
2651 int rc;2650 int rc;
2652 2651
2653 DP_NOTICE(edev, "Starting a recovery process\n");2652 DP_NOTICE(edev, "Starting a recovery process\n");
@@ -2677,17 +2676,18 @@ static void qede_recovery_handler(struct qede_dev *edev) · 23 unmodified lines
2677 goto err;2676 goto err;
2678 2677
2679 qede_config_rx_mode(edev->ndev);2678 qede_config_rx_mode(edev->ndev);
2680 udp_tunnel_nic_reset_ntf(edev->ndev);2679 reloaded = true;
2681 }2680 }
2682 2681
2683 edev->state = curr_state;2682 edev->state = curr_state;
2684 2683
2685 DP_NOTICE(edev, "Recovery handling is done\n");2684 DP_NOTICE(edev, "Recovery handling is done\n");
2686 2685
2687 return;2686 return reloaded;
2688 2687
2689err:2688err:
2690 qede_recovery_failed(edev);2689 qede_recovery_failed(edev);
2690 return false;
2691}2691}
2692 2692
2693static void qede_atomic_hw_err_handler(struct qede_dev *edev)2693static void qede_atomic_hw_err_handler(struct qede_dev *edev)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t