A TX timeout on a qede NIC that has VXLAN/GENEVE tunnel ports configured wedges the rtnetlink control plane of the whole machine: NETDEV WATCHDOG: ens6f1 (qede): transmit queue 2 timed out 10226 ms [qede_tx_timeout:586(ens6f1)]TX timeout on queue 2! [qede_recovery_handler:2665(ens6f0)]Starting a recovery process The recovery path deadlocks on the driver's own mutex: qede_sp_task rtnl_lock() mutex_lock(&edev->qede_lock) <- taken qede_recovery_handler qede_load udp_tunnel_nic_reset_ntf __udp_tunnel_nic_device_sync info->sync_table == qede_udp_tunnel_sync mutex_lock(&edev->qede_lock) <- same task: deadlock The mutex is not recursive, so the kworker blocks on itself with rtnl_lock held, and neither lock is ever released. Every task that calls rtnl_lock() afterwards (ip, ovs-vswitchd, lldpad, IPv6 addrconf, sshd) blocks forever while the node still answers ping. In a vmcore from an affected production node rtnl_mutex.owner decodes to the very kworker blocked at the innermost mutex_lock() above. Re-sync the tunnel ports from qede_sp_task() after the internal lock is dropped, still under rtnl_lock as the udp_tunnel API requires. This mirrors qede_open(), which calls udp_tunnel_nic_reset_ntf() under rtnl without the internal lock. qede_recovery_handler() now returns whether it has successfully reloaded an open device, and the caller re-syncs the ports only in that case. This keeps the old gating exactly: a device that was down or a failed recovery returns false, as those paths never reached the udp_tunnel_nic_reset_ntf() call before either. This was the only user of the qede_lock()/qede_unlock() helpers, so remove them.
Unified · Split
| @@ -107,7 +107,7 @@ static void qede_remove(struct pci_dev *pdev); · 106 unmodified lines | |||
| 107 | static void qede_shutdown(struct pci_dev *pdev); | 107 | static void qede_shutdown(struct pci_dev *pdev); |
| 108 | static void qede_link_update(void *dev, struct qed_link_output *link); | 108 | static void qede_link_update(void *dev, struct qed_link_output *link); |
| 109 | static void qede_schedule_recovery_handler(void *dev); | 109 | static void qede_schedule_recovery_handler(void *dev); |
| 110 | static void qede_recovery_handler(struct qede_dev *edev); | 110 | static bool qede_recovery_handler(struct qede_dev *edev); |
| 111 | static void qede_schedule_hw_err_handler(void *dev, | 111 | static void qede_schedule_hw_err_handler(void *dev, |
| 112 | enum qed_hw_err_type err_type); | 112 | enum qed_hw_err_type err_type); |
| 113 | static void qede_get_eth_tlv_data(void *edev, void *data); | 113 | static void qede_get_eth_tlv_data(void *edev, void *data); |
| @@ -1043,21 +1043,6 @@ void __qede_unlock(struct qede_dev *edev) · 929 unmodified lines | |||
| 1043 | mutex_unlock(&edev->qede_lock); | 1043 | mutex_unlock(&edev->qede_lock); |
| 1044 | } | 1044 | } |
| 1045 | 1045 | ||
| 1046 | /* This version of the lock should be used when acquiring the RTNL lock is also | ||
| 1047 | * needed in addition to the internal qede lock. | ||
| 1048 | */ | ||
| 1049 | static void qede_lock(struct qede_dev *edev) | ||
| 1050 | { | ||
| 1051 | rtnl_lock(); | ||
| 1052 | __qede_lock(edev); | ||
| 1053 | } | ||
| 1054 | |||
| 1055 | static void qede_unlock(struct qede_dev *edev) | ||
| 1056 | { | ||
| 1057 | __qede_unlock(edev); | ||
| 1058 | rtnl_unlock(); | ||
| 1059 | } | ||
| 1060 | |||
| 1061 | static void qede_periodic_task(struct work_struct *work) | 1046 | static void qede_periodic_task(struct work_struct *work) |
| 1062 | { | 1047 | { |
| 1063 | struct qede_dev *edev = container_of(work, struct qede_dev, | 1048 | struct qede_dev *edev = container_of(work, struct qede_dev, |
| @@ -1094,6 +1079,8 @@ static void qede_sp_task(struct work_struct *work) · 30 unmodified lines | |||
| 1094 | */ | 1079 | */ |
| 1095 | 1080 | ||
| 1096 | if (test_and_clear_bit(QEDE_SP_RECOVERY, &edev->sp_flags)) { | 1081 | if (test_and_clear_bit(QEDE_SP_RECOVERY, &edev->sp_flags)) { |
| 1082 | bool reloaded; | ||
| 1083 | |||
| 1097 | cancel_delayed_work_sync(&edev->periodic_task); | 1084 | cancel_delayed_work_sync(&edev->periodic_task); |
| 1098 | #ifdef CONFIG_QED_SRIOV | 1085 | #ifdef CONFIG_QED_SRIOV |
| 1099 | /* SRIOV must be disabled outside the lock to avoid a deadlock. | 1086 | /* SRIOV must be disabled outside the lock to avoid a deadlock. |
| @@ -1102,9 +1089,17 @@ static void qede_sp_task(struct work_struct *work) · 2 unmodified lines | |||
| 1102 | if (pci_num_vf(edev->pdev)) | 1089 | if (pci_num_vf(edev->pdev)) |
| 1103 | qede_sriov_configure(edev->pdev, 0); | 1090 | qede_sriov_configure(edev->pdev, 0); |
| 1104 | #endif | 1091 | #endif |
| 1105 | qede_lock(edev); | 1092 | rtnl_lock(); |
| 1106 | qede_recovery_handler(edev); | 1093 | __qede_lock(edev); |
| 1107 | qede_unlock(edev); | 1094 | reloaded = qede_recovery_handler(edev); |
| 1095 | __qede_unlock(edev); | ||
| 1096 | |||
| 1097 | /* The udp_tunnel core synchronously calls back into | ||
| 1098 | * qede_udp_tunnel_sync(), which takes the qede lock. | ||
| 1099 | */ | ||
| 1100 | if (reloaded) | ||
| 1101 | udp_tunnel_nic_reset_ntf(edev->ndev); | ||
| 1102 | rtnl_unlock(); | ||
| 1108 | } | 1103 | } |
| 1109 | 1104 | ||
| 1110 | __qede_lock(edev); | 1105 | __qede_lock(edev); |
| @@ -2645,9 +2640,13 @@ static void qede_recovery_failed(struct qede_dev *edev) · 1534 unmodified lines | |||
| 2645 | edev->ops->common->set_power_state(edev->cdev, PCI_D3hot); | 2640 | edev->ops->common->set_power_state(edev->cdev, PCI_D3hot); |
| 2646 | } | 2641 | } |
| 2647 | 2642 | ||
| 2648 | static void qede_recovery_handler(struct qede_dev *edev) | 2643 | /* Returns true if an open device was successfully reloaded and its |
| 2644 | * udp_tunnel ports need to be re-synced by the caller. | ||
| 2645 | */ | ||
| 2646 | static bool qede_recovery_handler(struct qede_dev *edev) | ||
| 2649 | { | 2647 | { |
| 2650 | u32 curr_state = edev->state; | 2648 | u32 curr_state = edev->state; |
| 2649 | bool reloaded = false; | ||
| 2651 | int rc; | 2650 | int rc; |
| 2652 | 2651 | ||
| 2653 | DP_NOTICE(edev, "Starting a recovery process\n"); | 2652 | DP_NOTICE(edev, "Starting a recovery process\n"); |
| @@ -2677,17 +2676,18 @@ static void qede_recovery_handler(struct qede_dev *edev) · 23 unmodified lines | |||
| 2677 | goto err; | 2676 | goto err; |
| 2678 | 2677 | ||
| 2679 | qede_config_rx_mode(edev->ndev); | 2678 | qede_config_rx_mode(edev->ndev); |
| 2680 | udp_tunnel_nic_reset_ntf(edev->ndev); | 2679 | reloaded = true; |
| 2681 | } | 2680 | } |
| 2682 | 2681 | ||
| 2683 | edev->state = curr_state; | 2682 | edev->state = curr_state; |
| 2684 | 2683 | ||
| 2685 | DP_NOTICE(edev, "Recovery handling is done\n"); | 2684 | DP_NOTICE(edev, "Recovery handling is done\n"); |
| 2686 | 2685 | ||
| 2687 | return; | 2686 | return reloaded; |
| 2688 | 2687 | ||
| 2689 | err: | 2688 | err: |
| 2690 | qede_recovery_failed(edev); | 2689 | qede_recovery_failed(edev); |
| 2690 | return false; | ||
| 2691 | } | 2691 | } |
| 2692 | 2692 | ||
| 2693 | static void qede_atomic_hw_err_handler(struct qede_dev *edev) | 2693 | static void qede_atomic_hw_err_handler(struct qede_dev *edev) |