net/mlx5: Handle SW reset of FW in error flow

New mlx5 adapters allow the driver to reset the FW in the event of an error, this action called "SW Reset". When an SW reset is issued on any PF all PFs enter reset state which is a recoverable condition. The existing recovery flow was designed to allow the recovery of a VF after a PF driver reload. This patch adds the sw reset to the NIC states as a preparation for sw reset handling. When a software reset is issued the following occurs: 1. The NIC interface mode is set to 7 while the reset is in progress. 2. Once the reset completes the NIC interface mode is set to 1.

Feras Daoud committed 8 years ago

Commit 63cbc551 parent 8b9d8ba5 files · +48 −65Browse files

Unified · Split

5 files

drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c +1 −1
drivers/net/ethernet/mellanox/mlx5/core/health.c +44 −61
drivers/net/ethernet/mellanox/mlx5/core/main.c +1 −1
drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +1 −1
include/linux/mlx5/driver.h +1 −1

drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c+1−1

@@ -64,7 +64,7 @@ static int mlx5e_test_health_info(struct mlx5e_priv *priv) · 63 unmodified lines
64	64	{
65	65	struct mlx5_core_health *health = &priv->mdev->priv.health;
66	66
67		− return health->sick ? 1 : 0;
	67	+ return health->fatal_error ? 1 : 0;
68	68	}
69	69
70	70	static int mlx5e_test_link_state(struct mlx5e_priv *priv)

drivers/net/ethernet/mellanox/mlx5/core/health.c+44−61

@@ -62,12 +62,18 @@ enum { · 61 unmodified lines
62	62
63	63	enum {
64	64	MLX5_DROP_NEW_HEALTH_WORK,
65		− MLX5_DROP_NEW_RECOVERY_WORK,
	65	+};
	66	+
	67	+enum {
	68	+ MLX5_SENSOR_NO_ERR = 0,
	69	+ MLX5_SENSOR_PCI_COMM_ERR = 1,
	70	+ MLX5_SENSOR_NIC_DISABLED = 2,
	71	+ MLX5_SENSOR_NIC_SW_RESET = 3,
66	72	};
67	73
68	74	u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
69	75	{
70		− return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3;
	76	+ return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7;
71	77	}
72	78
73	79	void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
@@ -80,18 +86,25 @@ void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) · 6 unmodified lines
80	86	&dev->iseg->cmdq_addr_l_sz);
81	87	}
82	88
83		−static int in_fatal(struct mlx5_core_dev *dev)
	89	+static bool sensor_pci_not_working(struct mlx5_core_dev *dev)
84	90	{
85	91	struct mlx5_core_health *health = &dev->priv.health;
86	92	struct health_buffer __iomem *h = health->health;
87	93
88		− if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
89		− return 1;
	94	+ /* Offline PCI reads return 0xffffffff */
	95	+ return (ioread32be(&h->fw_ver) == 0xffffffff);
	96	+}
90	97
91		− if (ioread32be(&h->fw_ver) == 0xffffffff)
92		− return 1;
	98	+static u32 check_fatal_sensors(struct mlx5_core_dev *dev)
	99	+{
	100	+ if (sensor_pci_not_working(dev))
	101	+ return MLX5_SENSOR_PCI_COMM_ERR;
	102	+ if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
	103	+ return MLX5_SENSOR_NIC_DISABLED;
	104	+ if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET)
	105	+ return MLX5_SENSOR_NIC_SW_RESET;
93	106
94		− return 0;
	107	+ return MLX5_SENSOR_NO_ERR;
95	108	}
96	109
97	110	void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
@@ -101,7 +114,8 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) · 3 unmodified lines
101	114	goto unlock;
102	115
103	116	mlx5_core_err(dev, "start\n");
104		− if (pci_channel_offline(dev->pdev) \|\| in_fatal(dev) \|\| force) {
	117	+ if (pci_channel_offline(dev->pdev) \|\|
	118	+ dev->priv.health.fatal_error != MLX5_SENSOR_NO_ERR \|\| force) {
105	119	dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
106	120	mlx5_cmd_flush(dev);
107	121	}
@@ -137,38 +151,14 @@ static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) · 29 unmodified lines
137	151	mlx5_disable_device(dev);
138	152	}
139	153
140		−static void health_recover(struct work_struct *work)
141		−{
142		− struct mlx5_core_health *health;
143		− struct delayed_work *dwork;
144		− struct mlx5_core_dev *dev;
145		− struct mlx5_priv *priv;
146		− u8 nic_state;
147		−
148		− dwork = container_of(work, struct delayed_work, work);
149		− health = container_of(dwork, struct mlx5_core_health, recover_work);
150		− priv = container_of(health, struct mlx5_priv, health);
151		− dev = container_of(priv, struct mlx5_core_dev, priv);
152		−
153		− nic_state = mlx5_get_nic_state(dev);
154		− if (nic_state == MLX5_NIC_IFC_INVALID) {
155		− mlx5_core_err(dev, "health recovery flow aborted since the nic state is invalid\n");
156		− return;
157		− }
158		−
159		− mlx5_core_err(dev, "starting health recovery flow\n");
160		− mlx5_recover_device(dev);
161		−}
162		−
163	154	/* How much time to wait until health resetting the driver (in msecs) */
164		−#define MLX5_RECOVERY_DELAY_MSECS 60000
	155	+#define MLX5_RECOVERY_WAIT_MSECS 60000
165	156	static void health_care(struct work_struct *work)
166	157	{
167		− unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS);
168	158	struct mlx5_core_health *health;
169	159	struct mlx5_core_dev *dev;
170	160	struct mlx5_priv *priv;
171		− unsigned long flags;
	161	+ unsigned long end;
172	162
173	163	health = container_of(work, struct mlx5_core_health, work);
174	164	priv = container_of(health, struct mlx5_priv, health);
@@ -176,13 +166,18 @@ static void health_care(struct work_struct *work) · 1 unmodified line
176	166	mlx5_core_warn(dev, "handling bad device here\n");
177	167	mlx5_handle_bad_state(dev);
178	168
179		− spin_lock_irqsave(&health->wq_lock, flags);
180		− if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags))
181		− schedule_delayed_work(&health->recover_work, recover_delay);
182		− else
183		− mlx5_core_err(dev,
184		− "new health works are not permitted at this stage\n");
185		− spin_unlock_irqrestore(&health->wq_lock, flags);
	169	+ end = jiffies + msecs_to_jiffies(MLX5_RECOVERY_WAIT_MSECS);
	170	+ while (sensor_pci_not_working(dev)) {
	171	+ if (time_after(jiffies, end)) {
	172	+ mlx5_core_err(dev,
	173	+ "health recovery flow aborted, PCI reads still not working\n");
	174	+ return;
	175	+ }
	176	+ msleep(100);
	177	+ }
	178	+
	179	+ mlx5_core_err(dev, "starting health recovery flow\n");
	180	+ mlx5_recover_device(dev);
186	181	}
187	182
188	183	static const char *hsynd_str(u8 synd)
@@ -274,6 +269,7 @@ static void poll_health(struct timer_list *t) · 85 unmodified lines
274	269	{
275	270	struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
276	271	struct mlx5_core_health *health = &dev->priv.health;
	272	+ u32 fatal_error;
277	273	u32 count;
278	274
279	275	if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
@@ -291,8 +287,11 @@ static void poll_health(struct timer_list *t) · 11 unmodified lines
291	287	print_health_info(dev);
292	288	}
293	289
294		− if (in_fatal(dev) && !health->sick) {
295		− health->sick = true;
	290	+ fatal_error = check_fatal_sensors(dev);
	291	+
	292	+ if (fatal_error && !health->fatal_error) {
	293	+ mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error);
	294	+ dev->priv.health.fatal_error = fatal_error;
296	295	print_health_info(dev);
297	296	mlx5_trigger_health_work(dev);
298	297	}
@@ -306,9 +305,8 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev) · 7 unmodified lines
306	305	struct mlx5_core_health *health = &dev->priv.health;
307	306
308	307	timer_setup(&health->timer, poll_health, 0);
309		− health->sick = 0;
	308	+ health->fatal_error = MLX5_SENSOR_NO_ERR;
310	309	clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
311		− clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
312	310	health->health = &dev->iseg->health;
313	311	health->health_counter = &dev->iseg->health_counter;
314	312
@@ -324,7 +322,6 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) · 9 unmodified lines
324	322	if (disable_health) {
325	323	spin_lock_irqsave(&health->wq_lock, flags);
326	324	set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
327		− set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
328	325	spin_unlock_irqrestore(&health->wq_lock, flags);
329	326	}
330	327
@@ -338,23 +335,10 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev) · 7 unmodified lines
338	335
339	336	spin_lock_irqsave(&health->wq_lock, flags);
340	337	set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags);
341		− set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
342	338	spin_unlock_irqrestore(&health->wq_lock, flags);
343		− cancel_delayed_work_sync(&health->recover_work);
344	339	cancel_work_sync(&health->work);
345	340	}
346	341
347		−void mlx5_drain_health_recovery(struct mlx5_core_dev *dev)
348		−{
349		− struct mlx5_core_health *health = &dev->priv.health;
350		− unsigned long flags;
351		−
352		− spin_lock_irqsave(&health->wq_lock, flags);
353		− set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags);
354		− spin_unlock_irqrestore(&health->wq_lock, flags);
355		− cancel_delayed_work_sync(&dev->priv.health.recover_work);
356		−}
357		−
358	342	void mlx5_health_flush(struct mlx5_core_dev *dev)
359	343	{
360	344	struct mlx5_core_health *health = &dev->priv.health;
@@ -387,7 +371,6 @@ int mlx5_health_init(struct mlx5_core_dev *dev) · 26 unmodified lines
387	371	return -ENOMEM;
388	372	spin_lock_init(&health->wq_lock);
389	373	INIT_WORK(&health->work, health_care);
390		− INIT_DELAYED_WORK(&health->recover_work, health_recover);
391	374
392	375	return 0;
393	376	}

drivers/net/ethernet/mellanox/mlx5/core/main.c+1−1

@@ -1191,7 +1191,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, bool cleanup) · 1190 unmodified lines
1191	1191	int err = 0;
1192	1192
1193	1193	if (cleanup)
1194		− mlx5_drain_health_recovery(dev);
	1194	+ mlx5_drain_health_wq(dev);
1195	1195
1196	1196	mutex_lock(&dev->intf_state_mutex);
1197	1197	if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {

drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h+1−1

@@ -214,7 +214,7 @@ enum { · 213 unmodified lines
214	214	MLX5_NIC_IFC_FULL = 0,
215	215	MLX5_NIC_IFC_DISABLED = 1,
216	216	MLX5_NIC_IFC_NO_DRAM_NIC = 2,
217		− MLX5_NIC_IFC_INVALID = 3
	217	+ MLX5_NIC_IFC_SW_RESET = 7
218	218	};
219	219
220	220	u8 mlx5_get_nic_state(struct mlx5_core_dev *dev);

include/linux/mlx5/driver.h+1−1

@@ -435,6 +435,7 @@ struct mlx5_core_health { · 434 unmodified lines
435	435	u32 prev;
436	436	int miss_counter;
437	437	bool sick;
	438	+ u32 fatal_error;
438	439	u32 crdump_size;
439	440	/* wq spinlock to synchronize draining */
440	441	spinlock_t wq_lock;
@@ -906,7 +907,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev); · 465 unmodified lines
906	907	void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health);
907	908	void mlx5_drain_health_wq(struct mlx5_core_dev *dev);
908	909	void mlx5_trigger_health_work(struct mlx5_core_dev *dev);
909		−void mlx5_drain_health_recovery(struct mlx5_core_dev *dev);
910	910	int mlx5_buf_alloc_node(struct mlx5_core_dev *dev, int size,
911	911	struct mlx5_frag_buf *buf, int node);
912	912	int mlx5_buf_alloc(struct mlx5_core_dev *dev,

net/mlx5: Handle SW reset of FW in error flow

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code