Saeed Mahameed says: ==================== mlx5-updates-2019-06-13 Mlx5 devlink health fw reporters and sw reset support This series provides mlx5 firmware reset support and firmware devlink health reporters. 1) Add initial mlx5 kernel documentation and include devlink health reporters 2) Add CR-Space access and FW Crdump snapshot support via devlink region_snapshot 3) Issue software reset upon FW asserts 4) Add fw and fw_fatal devlink heath reporters to follow fw errors indication by dump and recover procedures and enable trigger these functionality by user. 4.1) fw reporter: The fw reporter implements diagnose and dump callbacks. It follows symptoms of fw error such as fw syndrome by triggering fw core dump and storing it and any other fw trace into the dump buffer. The fw reporter diagnose command can be triggered any time by the user to check current fw status. 4.2) fw_fatal repoter: The fw_fatal reporter implements dump and recover callbacks. It follows fatal errors indications by CR-space dump and recover flow. The CR-space dump uses vsc interface which is valid even if the FW command interface is not functional, which is the case in most FW fatal errors. The CR-space dump is stored as a memory region snapshot to ease read by address. The recover function runs recover flow which reloads the driver and triggers fw reset if needed. ====================
Showing the changes to include/linux (2 of 19 files). Show all
Unified · Split
| @@ -510,6 +510,10 @@ struct mlx5_cmd_layout { · 509 unmodified lines | ||
| 510 | 510 | u8 status_own; |
| 511 | 511 | }; |
| 512 | 512 | |
| 513 | +enum mlx5_fatal_assert_bit_offsets { | |
| 514 | + MLX5_RFR_OFFSET = 31, | |
| 515 | +}; | |
| 516 | + | |
| 513 | 517 | struct health_buffer { |
| 514 | 518 | __be32 assert_var[5]; |
| 515 | 519 | __be32 rsvd0[3]; |
| @@ -518,12 +522,16 @@ struct health_buffer { · 2 unmodified lines | ||
| 518 | 522 | __be32 rsvd1[2]; |
| 519 | 523 | __be32 fw_ver; |
| 520 | 524 | __be32 hw_id; |
| 521 | − __be32 rsvd2; | |
| 525 | + __be32 rfr; | |
| 522 | 526 | u8 irisc_index; |
| 523 | 527 | u8 synd; |
| 524 | 528 | __be16 ext_synd; |
| 525 | 529 | }; |
| 526 | 530 | |
| 531 | +enum mlx5_initializing_bit_offsets { | |
| 532 | + MLX5_FW_RESET_SUPPORTED_OFFSET = 30, | |
| 533 | +}; | |
| 534 | + | |
| 527 | 535 | enum mlx5_cmd_addr_l_sz_offset { |
| 528 | 536 | MLX5_NIC_IFC_OFFSET = 8, |
| 529 | 537 | }; |
| @@ -53,6 +53,7 @@ · 52 unmodified lines | ||
| 53 | 53 | #include <linux/mlx5/eq.h> |
| 54 | 54 | #include <linux/timecounter.h> |
| 55 | 55 | #include <linux/ptp_clock_kernel.h> |
| 56 | +#include <net/devlink.h> | |
| 56 | 57 | |
| 57 | 58 | enum { |
| 58 | 59 | MLX5_BOARD_ID_LEN = 64, |
| @@ -434,13 +435,18 @@ struct mlx5_core_health { · 375 unmodified lines | ||
| 434 | 435 | struct timer_list timer; |
| 435 | 436 | u32 prev; |
| 436 | 437 | int miss_counter; |
| 437 | − bool sick; | |
| 438 | + u8 synd; | |
| 439 | + u32 fatal_error; | |
| 440 | + u32 crdump_size; | |
| 438 | 441 | /* wq spinlock to synchronize draining */ |
| 439 | 442 | spinlock_t wq_lock; |
| 440 | 443 | struct workqueue_struct *wq; |
| 441 | 444 | unsigned long flags; |
| 442 | − struct work_struct work; | |
| 445 | + struct work_struct fatal_report_work; | |
| 446 | + struct work_struct report_work; | |
| 443 | 447 | struct delayed_work recover_work; |
| 448 | + struct devlink_health_reporter *fw_reporter; | |
| 449 | + struct devlink_health_reporter *fw_fatal_reporter; | |
| 444 | 450 | }; |
| 445 | 451 | |
| 446 | 452 | struct mlx5_qp_table { |
| @@ -581,6 +587,7 @@ struct mlx5_priv { · 134 unmodified lines | ||
| 581 | 587 | }; |
| 582 | 588 | |
| 583 | 589 | enum mlx5_device_state { |
| 590 | + MLX5_DEVICE_STATE_UNINITIALIZED, | |
| 584 | 591 | MLX5_DEVICE_STATE_UP, |
| 585 | 592 | MLX5_DEVICE_STATE_INTERNAL_ERROR, |
| 586 | 593 | }; |
| @@ -693,6 +700,7 @@ struct mlx5_core_dev { · 106 unmodified lines | ||
| 693 | 700 | struct mlx5_clock clock; |
| 694 | 701 | struct mlx5_ib_clock_info *clock_info; |
| 695 | 702 | struct mlx5_fw_tracer *tracer; |
| 703 | + u32 vsc_addr; | |
| 696 | 704 | }; |
| 697 | 705 | |
| 698 | 706 | struct mlx5_db { |
| @@ -904,7 +912,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev); · 205 unmodified lines | ||
| 904 | 912 | void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health); |
| 905 | 913 | void mlx5_drain_health_wq(struct mlx5_core_dev *dev); |
| 906 | 914 | void mlx5_trigger_health_work(struct mlx5_core_dev *dev); |
| 907 | −void mlx5_drain_health_recovery(struct mlx5_core_dev *dev); | |
| 908 | 915 | int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size, |
| 909 | 916 | struct mlx5_frag_buf *buf, int node); |
| 910 | 917 | int mlx5_buf_alloc(struct mlx5_core_dev *dev, |