History

net/mlx5: Report devlink health on FW fatal issues

Report devlink health on FW fatal issues via fw_fatal_reporter. The driver recover flow for FW fatal error is now being handled by the devlink health. Having the recovery controlled by devlink health, the user has the ability to cancel the auto-recovery for debug session and run it manually. Call mlx5_enter_error_state() before calling devlink_health_report() to ensure entering device error state even if auto-recovery is off.

Moshe Shemesh committed 7 years ago

Commit b3bd0761 parent 9b1f2983 files · +31 −23Browse files

Unified · Split

3 files
  1. drivers/net/ethernet/mellanox/mlx5/core/health.c +26 −16
  2. drivers/net/ethernet/mellanox/mlx5/core/main.c +4 −6
  3. include/linux/mlx5/driver.h +1 −1
drivers/net/ethernet/mellanox/mlx5/core/health.c+26−16
@@ -327,19 +327,6 @@ static int mlx5_health_try_recover(struct mlx5_core_dev *dev) · 326 unmodified lines
327 return 0;327 return 0;
328}328}
329 329
330static void health_recover_work(struct work_struct *work)
331{
332 struct mlx5_core_health *health;
333 struct mlx5_core_dev *dev;
334 struct mlx5_priv *priv;
335
336 health = container_of(work, struct mlx5_core_health, work);
337 priv = container_of(health, struct mlx5_priv, health);
338 dev = container_of(priv, struct mlx5_core_dev, priv);
339
340 mlx5_health_try_recover(dev);
341}
342
343static const char *hsynd_str(u8 synd)330static const char *hsynd_str(u8 synd)
344{331{
345 switch (synd) {332 switch (synd) {
@@ -614,6 +601,29 @@ free_data: · 268 unmodified lines
614 return err;601 return err;
615}602}
616 603
604static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work)
605{
606 struct mlx5_fw_reporter_ctx fw_reporter_ctx;
607 struct mlx5_core_health *health;
608 struct mlx5_core_dev *dev;
609 struct mlx5_priv *priv;
610
611 health = container_of(work, struct mlx5_core_health, fatal_report_work);
612 priv = container_of(health, struct mlx5_priv, health);
613 dev = container_of(priv, struct mlx5_core_dev, priv);
614
615 mlx5_enter_error_state(dev, false);
616 if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) {
617 if (mlx5_health_try_recover(dev))
618 mlx5_core_err(dev, "health recovery failed\n");
619 return;
620 }
621 fw_reporter_ctx.err_synd = health->synd;
622 fw_reporter_ctx.miss_counter = health->miss_counter;
623 devlink_health_report(health->fw_fatal_reporter,
624 "FW fatal error reported", &fw_reporter_ctx);
625}
626
617static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = {627static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = {
618 .name = "fw_fatal",628 .name = "fw_fatal",
619 .recover = mlx5_fw_fatal_reporter_recover,629 .recover = mlx5_fw_fatal_reporter_recover,
@@ -672,7 +682,7 @@ void mlx5_trigger_health_work(struct mlx5_core_dev *dev) · 52 unmodified lines
672 682
673 spin_lock_irqsave(&health->wq_lock, flags);683 spin_lock_irqsave(&health->wq_lock, flags);
674 if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))684 if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags))
675 queue_work(health->wq, &health->work);685 queue_work(health->wq, &health->fatal_report_work);
676 else686 else
677 mlx5_core_err(dev, "new health works are not permitted at this stage\n");687 mlx5_core_err(dev, "new health works are not permitted at this stage\n");
678 spin_unlock_irqrestore(&health->wq_lock, flags);688 spin_unlock_irqrestore(&health->wq_lock, flags);
@@ -758,7 +768,7 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev) · 79 unmodified lines
758 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);768 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
759 spin_unlock_irqrestore(&health->wq_lock, flags);769 spin_unlock_irqrestore(&health->wq_lock, flags);
760 cancel_work_sync(&health->report_work);770 cancel_work_sync(&health->report_work);
761 cancel_work_sync(&health->work);771 cancel_work_sync(&health->fatal_report_work);
762}772}
763 773
764void mlx5_health_flush(struct mlx5_core_dev *dev)774void mlx5_health_flush(struct mlx5_core_dev *dev)
@@ -795,7 +805,7 @@ int mlx5_health_init(struct mlx5_core_dev *dev) · 30 unmodified lines
795 if (!health->wq)805 if (!health->wq)
796 goto out_err;806 goto out_err;
797 spin_lock_init(&health->wq_lock);807 spin_lock_init(&health->wq_lock);
798 INIT_WORK(&health->work, health_recover_work);808 INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work);
799 INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work);809 INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work);
800 810
801 return 0;811 return 0;
drivers/net/ethernet/mellanox/mlx5/core/main.c+4−6
@@ -1363,11 +1363,8 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, · 1362 unmodified lines
1363 mlx5_enter_error_state(dev, false);1363 mlx5_enter_error_state(dev, false);
1364 mlx5_error_sw_reset(dev);1364 mlx5_error_sw_reset(dev);
1365 mlx5_unload_one(dev, false);1365 mlx5_unload_one(dev, false);
1366 /* In case of kernel call drain the health wq */1366 mlx5_drain_health_wq(dev);
1367 if (state) {1367 mlx5_pci_disable_device(dev);
1368 mlx5_drain_health_wq(dev);
1369 mlx5_pci_disable_device(dev);
1370 }
1371 1368
1372 return state == pci_channel_io_perm_failure ?1369 return state == pci_channel_io_perm_failure ?
1373 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;1370 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
@@ -1535,7 +1532,8 @@ MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); · 161 unmodified lines
1535 1532
1536void mlx5_disable_device(struct mlx5_core_dev *dev)1533void mlx5_disable_device(struct mlx5_core_dev *dev)
1537{1534{
1538 mlx5_pci_err_detected(dev->pdev, 0);1535 mlx5_error_sw_reset(dev);
1536 mlx5_unload_one(dev, false);
1539}1537}
1540 1538
1541void mlx5_recover_device(struct mlx5_core_dev *dev)1539void mlx5_recover_device(struct mlx5_core_dev *dev)
include/linux/mlx5/driver.h+1−1
@@ -442,7 +442,7 @@ struct mlx5_core_health { · 441 unmodified lines
442 spinlock_t wq_lock;442 spinlock_t wq_lock;
443 struct workqueue_struct *wq;443 struct workqueue_struct *wq;
444 unsigned long flags;444 unsigned long flags;
445 struct work_struct work;445 struct work_struct fatal_report_work;
446 struct work_struct report_work;446 struct work_struct report_work;
447 struct delayed_work recover_work;447 struct delayed_work recover_work;
448 struct devlink_health_reporter *fw_reporter;448 struct devlink_health_reporter *fw_reporter;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t