History

net/mlx5: Handle SW reset of FW in error flow

New mlx5 adapters allow the driver to reset the FW in the event of an error, this action called "SW Reset". When an SW reset is issued on any PF all PFs enter reset state which is a recoverable condition. The existing recovery flow was designed to allow the recovery of a VF after a PF driver reload. This patch adds the sw reset to the NIC states as a preparation for sw reset handling. When a software reset is issued the following occurs: 1. The NIC interface mode is set to 7 while the reset is in progress. 2. Once the reset completes the NIC interface mode is set to 1.

Feras Daoud committed 8 years ago

Commit 63cbc551 parent 8b9d8ba5 files · +48 −65Browse files

Unified · Split

5 files
  1. drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c +1 −1
  2. drivers/net/ethernet/mellanox/mlx5/core/health.c +44 −61
  3. drivers/net/ethernet/mellanox/mlx5/core/main.c +1 −1
  4. drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +1 −1
  5. include/linux/mlx5/driver.h +1 −1
drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c+1−1
@@ -64,7 +64,7 @@ static int mlx5e_test_health_info(struct mlx5e_priv *priv) · 63 unmodified lines
64{64{
65 struct mlx5_core_health *health = &priv->mdev->priv.health;65 struct mlx5_core_health *health = &priv->mdev->priv.health;
66 66
67 return health->sick ? 1 : 0;67 return health->fatal_error ? 1 : 0;
68}68}
69 69
70static int mlx5e_test_link_state(struct mlx5e_priv *priv)70static int mlx5e_test_link_state(struct mlx5e_priv *priv)
drivers/net/ethernet/mellanox/mlx5/core/health.c+44−61
@@ -62,12 +62,18 @@ enum { · 61 unmodified lines
62 62
63enum {63enum {
64 MLX5_DROP_NEW_HEALTH_WORK,64 MLX5_DROP_NEW_HEALTH_WORK,
65 MLX5_DROP_NEW_RECOVERY_WORK,65};
66
67enum {
68 MLX5_SENSOR_NO_ERR = 0,
69 MLX5_SENSOR_PCI_COMM_ERR = 1,
70 MLX5_SENSOR_NIC_DISABLED = 2,
71 MLX5_SENSOR_NIC_SW_RESET = 3,
66};72};
67 73
68u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)74u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
69{75{
70 return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;76 return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7;
71}77}
72 78
73void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)79void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
@@ -80,18 +86,25 @@ void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) · 6 unmodified lines
80 &dev->iseg->cmdq_addr_l_sz);86 &dev->iseg->cmdq_addr_l_sz);
81}87}
82 88
83static int in_fatal(struct mlx5_core_dev *dev)89static bool sensor_pci_not_working(struct mlx5_core_dev *dev)
84{90{
85 struct mlx5_core_health *health = &dev->priv.health;91 struct mlx5_core_health *health = &dev->priv.health;
86 struct health_buffer __iomem *h = health->health;92 struct health_buffer __iomem *h = health->health;
87 93
88 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)94 /* Offline PCI reads return 0xffffffff */
89 return 1;95 return (ioread32be(&h->fw_ver) == 0xffffffff);
96}
90 97
91 if (ioread32be(&h->fw_ver) == 0xffffffff)98static u32 check_fatal_sensors(struct mlx5_core_dev *dev)
92 return 1;99{
100 if (sensor_pci_not_working(dev))
101 return MLX5_SENSOR_PCI_COMM_ERR;
102 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
103 return MLX5_SENSOR_NIC_DISABLED;
104 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET)
105 return MLX5_SENSOR_NIC_SW_RESET;
93 106
94 return 0;107 return MLX5_SENSOR_NO_ERR;
95}108}
96 109
97void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)110void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
@@ -101,7 +114,8 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) · 3 unmodified lines
101 goto unlock;114 goto unlock;
102 115
103 mlx5_core_err(dev, "start\n");116 mlx5_core_err(dev, "start\n");
104 if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) {117 if (pci_channel_offline(dev->pdev) ||
118 dev->priv.health.fatal_error != MLX5_SENSOR_NO_ERR || force) {
105 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;119 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
106 mlx5_cmd_flush(dev);120 mlx5_cmd_flush(dev);
107 }121 }
@@ -137,38 +151,14 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) · 29 unmodified lines
137 mlx5_disable_device(dev);151 mlx5_disable_device(dev);
138}152}
139 153
140static void health_recover(struct work_struct *work)
141{
142 struct mlx5_core_health *health;
143 struct delayed_work *dwork;
144 struct mlx5_core_dev *dev;
145 struct mlx5_priv *priv;
146 u8 nic_state;
147
148 dwork = container_of(work, struct delayed_work, work);
149 health = container_of(dwork, struct mlx5_core_health, recover_work);
150 priv = container_of(health, struct mlx5_priv, health);
151 dev = container_of(priv, struct mlx5_core_dev, priv);
152
153 nic_state = mlx5_get_nic_state(dev);
154 if (nic_state == MLX5_NIC_IFC_INVALID) {
155 mlx5_core_err(dev, "health recovery flow aborted since the nic state is invalid\n");
156 return;
157 }
158
159 mlx5_core_err(dev, "starting health recovery flow\n");
160 mlx5_recover_device(dev);
161}
162
163/* How much time to wait until health resetting the driver (in msecs) */154/* How much time to wait until health resetting the driver (in msecs) */
164#define MLX5_RECOVERY_DELAY_MSECS 60000155#define MLX5_RECOVERY_WAIT_MSECS 60000
165static void health_care(struct work_struct *work)156static void health_care(struct work_struct *work)
166{157{
167 unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
168 struct mlx5_core_health *health;158 struct mlx5_core_health *health;
169 struct mlx5_core_dev *dev;159 struct mlx5_core_dev *dev;
170 struct mlx5_priv *priv;160 struct mlx5_priv *priv;
171 unsigned long flags;161 unsigned long end;
172 162
173 health = container_of(work, struct mlx5_core_health, work);163 health = container_of(work, struct mlx5_core_health, work);
174 priv = container_of(health, struct mlx5_priv, health);164 priv = container_of(health, struct mlx5_priv, health);
@@ -176,13 +166,18 @@ static void health_care(struct work_struct *work) · 1 unmodified line
176 mlx5_core_warn(dev, "handling bad device here\n");166 mlx5_core_warn(dev, "handling bad device here\n");
177 mlx5_handle_bad_state(dev);167 mlx5_handle_bad_state(dev);
178 168
179 spin_lock_irqsave(&health->wq_lock, flags);169 end = jiffies + msecs_to_jiffies(MLX5_RECOVERY_WAIT_MSECS);
180 if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags))170 while (sensor_pci_not_working(dev)) {
181 schedule_delayed_work(&health->recover_work, recover_delay);171 if (time_after(jiffies, end)) {
182 else172 mlx5_core_err(dev,
183 mlx5_core_err(dev,173 "health recovery flow aborted, PCI reads still not working\n");
184 "new health works are not permitted at this stage\n");174 return;
185 spin_unlock_irqrestore(&health->wq_lock, flags);175 }
176 msleep(100);
177 }
178
179 mlx5_core_err(dev, "starting health recovery flow\n");
180 mlx5_recover_device(dev);
186}181}
187 182
188static const char *hsynd_str(u8 synd)183static const char *hsynd_str(u8 synd)
@@ -274,6 +269,7 @@ static void poll_health(struct timer_list *t) · 85 unmodified lines
274{269{
275 struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);270 struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
276 struct mlx5_core_health *health = &dev->priv.health;271 struct mlx5_core_health *health = &dev->priv.health;
272 u32 fatal_error;
277 u32 count;273 u32 count;
278 274
279 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)275 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
@@ -291,8 +287,11 @@ static void poll_health(struct timer_list *t) · 11 unmodified lines
291 print_health_info(dev);287 print_health_info(dev);
292 }288 }
293 289
294 if (in_fatal(dev) && !health->sick) {290 fatal_error = check_fatal_sensors(dev);
295 health->sick = true;291
292 if (fatal_error && !health->fatal_error) {
293 mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error);
294 dev->priv.health.fatal_error = fatal_error;
296 print_health_info(dev);295 print_health_info(dev);
297 mlx5_trigger_health_work(dev);296 mlx5_trigger_health_work(dev);
298 }297 }
@@ -306,9 +305,8 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) · 7 unmodified lines
306 struct mlx5_core_health *health = &dev->priv.health;305 struct mlx5_core_health *health = &dev->priv.health;
307 306
308 timer_setup(&health->timer, poll_health, 0);307 timer_setup(&health->timer, poll_health, 0);
309 health->sick = 0;308 health->fatal_error = MLX5_SENSOR_NO_ERR;
310 clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);309 clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
311 clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
312 health->health = &dev->iseg->health;310 health->health = &dev->iseg->health;
313 health->health_counter = &dev->iseg->health_counter;311 health->health_counter = &dev->iseg->health_counter;
314 312
@@ -324,7 +322,6 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) · 9 unmodified lines
324 if (disable_health) {322 if (disable_health) {
325 spin_lock_irqsave(&health->wq_lock, flags);323 spin_lock_irqsave(&health->wq_lock, flags);
326 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);324 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
327 set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
328 spin_unlock_irqrestore(&health->wq_lock, flags);325 spin_unlock_irqrestore(&health->wq_lock, flags);
329 }326 }
330 327
@@ -338,23 +335,10 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev) · 7 unmodified lines
338 335
339 spin_lock_irqsave(&health->wq_lock, flags);336 spin_lock_irqsave(&health->wq_lock, flags);
340 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);337 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
341 set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
342 spin_unlock_irqrestore(&health->wq_lock, flags);338 spin_unlock_irqrestore(&health->wq_lock, flags);
343 cancel_delayed_work_sync(&health->recover_work);
344 cancel_work_sync(&health->work);339 cancel_work_sync(&health->work);
345}340}
346 341
347void mlx5_drain_health_recovery(struct mlx5_core_dev *dev)
348{
349 struct mlx5_core_health *health = &dev->priv.health;
350 unsigned long flags;
351
352 spin_lock_irqsave(&health->wq_lock, flags);
353 set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
354 spin_unlock_irqrestore(&health->wq_lock, flags);
355 cancel_delayed_work_sync(&dev->priv.health.recover_work);
356}
357
358void mlx5_health_flush(struct mlx5_core_dev *dev)342void mlx5_health_flush(struct mlx5_core_dev *dev)
359{343{
360 struct mlx5_core_health *health = &dev->priv.health;344 struct mlx5_core_health *health = &dev->priv.health;
@@ -387,7 +371,6 @@ int mlx5_health_init(struct mlx5_core_dev *dev) · 26 unmodified lines
387 return -ENOMEM;371 return -ENOMEM;
388 spin_lock_init(&health->wq_lock);372 spin_lock_init(&health->wq_lock);
389 INIT_WORK(&health->work, health_care);373 INIT_WORK(&health->work, health_care);
390 INIT_DELAYED_WORK(&health->recover_work, health_recover);
391 374
392 return 0;375 return 0;
393}376}
drivers/net/ethernet/mellanox/mlx5/core/main.c+1−1
@@ -1191,7 +1191,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup) · 1190 unmodified lines
1191 int err = 0;1191 int err = 0;
1192 1192
1193 if (cleanup)1193 if (cleanup)
1194 mlx5_drain_health_recovery(dev);1194 mlx5_drain_health_wq(dev);
1195 1195
1196 mutex_lock(&dev->intf_state_mutex);1196 mutex_lock(&dev->intf_state_mutex);
1197 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {1197 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h+1−1
@@ -214,7 +214,7 @@ enum { · 213 unmodified lines
214 MLX5_NIC_IFC_FULL = 0,214 MLX5_NIC_IFC_FULL = 0,
215 MLX5_NIC_IFC_DISABLED = 1,215 MLX5_NIC_IFC_DISABLED = 1,
216 MLX5_NIC_IFC_NO_DRAM_NIC = 2,216 MLX5_NIC_IFC_NO_DRAM_NIC = 2,
217 MLX5_NIC_IFC_INVALID = 3217 MLX5_NIC_IFC_SW_RESET = 7
218};218};
219 219
220u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);220u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);
include/linux/mlx5/driver.h+1−1
@@ -435,6 +435,7 @@ struct mlx5_core_health { · 434 unmodified lines
435 u32 prev;435 u32 prev;
436 int miss_counter;436 int miss_counter;
437 bool sick;437 bool sick;
438 u32 fatal_error;
438 u32 crdump_size;439 u32 crdump_size;
439 /* wq spinlock to synchronize draining */440 /* wq spinlock to synchronize draining */
440 spinlock_t wq_lock;441 spinlock_t wq_lock;
@@ -906,7 +907,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev); · 465 unmodified lines
906void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health);907void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health);
907void mlx5_drain_health_wq(struct mlx5_core_dev *dev);908void mlx5_drain_health_wq(struct mlx5_core_dev *dev);
908void mlx5_trigger_health_work(struct mlx5_core_dev *dev);909void mlx5_trigger_health_work(struct mlx5_core_dev *dev);
909void mlx5_drain_health_recovery(struct mlx5_core_dev *dev);
910int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,910int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
911 struct mlx5_frag_buf *buf, int node);911 struct mlx5_frag_buf *buf, int node);
912int mlx5_buf_alloc(struct mlx5_core_dev *dev,912int mlx5_buf_alloc(struct mlx5_core_dev *dev,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t