History

net/mlx5: Change interrupt handler to call chain notifier

Multiple EQs may share the same IRQ in subsequent patches. Instead of calling the IRQ handler directly, the EQ will register to an atomic chain notfier. The Linux built-in shared IRQ is not used because it forces the caller to disable the IRQ and clear affinity before free_irq() can be called. This patch is the first step in the separation of IRQ and EQ logic.

Yuval Avnery committed 7 years ago

Commit ca390791 parent 081cc2d5 files · +105 −56Browse files

Unified · Split

5 files
  1. drivers/infiniband/hw/mlx5/mlx5_ib.h +1 −0
  2. drivers/infiniband/hw/mlx5/odp.c +6 −4
  3. drivers/net/ethernet/mellanox/mlx5/core/eq.c +89 −49
  4. drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h +8 −1
  5. include/linux/mlx5/eq.h +1 −2
drivers/infiniband/hw/mlx5/mlx5_ib.h+1−0
@@ -920,6 +920,7 @@ struct mlx5_ib_lb_state { · 919 unmodified lines
920920 };
921921
922922 struct mlx5_ib_pf_eq {
923+ struct notifier_block irq_nb;
923924 struct mlx5_ib_dev *dev;
924925 struct mlx5_eq *core;
925926 struct work_struct work;
drivers/infiniband/hw/mlx5/odp.c+6−4
@@ -1488,9 +1488,11 @@ static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) · 1487 unmodified lines
14881488 mlx5_eq_update_ci(eq->core, cc, 1);
14891489 }
14901490
1491−static irqreturn_t mlx5_ib_eq_pf_int(int irq, void *eq_ptr)
1491+static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type,
1492+ void *data)
14921493 {
1493− struct mlx5_ib_pf_eq *eq = eq_ptr;
1494+ struct mlx5_ib_pf_eq *eq =
1495+ container_of(nb, struct mlx5_ib_pf_eq, irq_nb);
14941496 unsigned long flags;
14951497
14961498 if (spin_trylock_irqsave(&eq->lock, flags)) {
@@ -1553,12 +1555,12 @@ mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) · 56 unmodified lines
15531555 goto err_mempool;
15541556 }
15551557
1558+ eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int;
15561559 param = (struct mlx5_eq_param) {
15571560 .index = MLX5_EQ_PFAULT_IDX,
15581561 .mask = 1 << MLX5_EVENT_TYPE_PAGE_FAULT,
15591562 .nent = MLX5_IB_NUM_PF_EQE,
1560− .context = eq,
1561− .handler = mlx5_ib_eq_pf_int
1563+ .nb = &eq->irq_nb,
15621564 };
15631565 eq->core = mlx5_eq_create_generic(dev->mdev, "mlx5_ib_page_fault_eq", &param);
15641566 if (IS_ERR(eq->core)) {
drivers/net/ethernet/mellanox/mlx5/core/eq.c+89−49
@@ -72,16 +72,16 @@ enum { · 71 unmodified lines
7272 static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE);
7373
7474 struct mlx5_irq_info {
75+ struct atomic_notifier_head nh;
7576 cpumask_var_t mask;
7677 char name[MLX5_MAX_IRQ_NAME];
77− void *context; /* dev_id provided to request_irq */
7878 };
7979
8080 struct mlx5_eq_table {
8181 struct list_head comp_eqs_list;
82− struct mlx5_eq pages_eq;
83− struct mlx5_eq cmd_eq;
84− struct mlx5_eq async_eq;
82+ struct mlx5_eq_async pages_eq;
83+ struct mlx5_eq_async cmd_eq;
84+ struct mlx5_eq_async async_eq;
8585
8686 struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX];
8787
@@ -109,6 +109,31 @@ struct mlx5_eq_table { · 21 unmodified lines
109109 (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE) | \
110110 (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
111111
112+static struct mlx5_irq_info *mlx5_irq_get(struct mlx5_core_dev *dev, int vecidx)
113+{
114+ struct mlx5_eq_table *eq_table = dev->priv.eq_table;
115+
116+ return &eq_table->irq_info[vecidx];
117+}
118+
119+static int mlx5_irq_attach_nb(struct mlx5_irq_info *irq,
120+ struct notifier_block *nb)
121+{
122+ return atomic_notifier_chain_register(&irq->nh, nb);
123+}
124+
125+static int mlx5_irq_detach_nb(struct mlx5_irq_info *irq,
126+ struct notifier_block *nb)
127+{
128+ return atomic_notifier_chain_unregister(&irq->nh, nb);
129+}
130+
131+static irqreturn_t mlx5_irq_int_handler(int irq, void *nh)
132+{
133+ atomic_notifier_call_chain(nh, 0, NULL);
134+ return IRQ_HANDLED;
135+}
136+
112137 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
113138 {
114139 u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {0};
@@ -134,10 +159,13 @@ static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn) · 19 unmodified lines
134159 return cq;
135160 }
136161
137−static irqreturn_t mlx5_eq_comp_int(int irq, void *eq_ptr)
162+static int mlx5_eq_comp_int(struct notifier_block *nb,
163+ __always_unused unsigned long action,
164+ __always_unused void *data)
138165 {
139− struct mlx5_eq_comp *eq_comp = eq_ptr;
140− struct mlx5_eq *eq = eq_ptr;
166+ struct mlx5_eq_comp *eq_comp =
167+ container_of(nb, struct mlx5_eq_comp, irq_nb);
168+ struct mlx5_eq *eq = &eq_comp->core;
141169 struct mlx5_eqe *eqe;
142170 int num_eqes = 0;
143171 u32 cqn = -1;
@@ -175,7 +203,7 @@ out: · 31 unmodified lines
175203 if (cqn != -1)
176204 tasklet_schedule(&eq_comp->tasklet_ctx.task);
177205
178− return IRQ_HANDLED;
206+ return 0;
179207 }
180208
181209 /* Some architectures don't latch interrupts when they are disabled, so using
@@ -189,16 +217,19 @@ u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq) · 7 unmodified lines
189217
190218 disable_irq(eq->core.irqn);
191219 count_eqe = eq->core.cons_index;
192− mlx5_eq_comp_int(eq->core.irqn, eq);
220+ mlx5_eq_comp_int(&eq->irq_nb, 0, NULL);
193221 count_eqe = eq->core.cons_index - count_eqe;
194222 enable_irq(eq->core.irqn);
195223
196224 return count_eqe;
197225 }
198226
199−static irqreturn_t mlx5_eq_async_int(int irq, void *eq_ptr)
227+static int mlx5_eq_async_int(struct notifier_block *nb,
228+ unsigned long action, void *data)
200229 {
201− struct mlx5_eq *eq = eq_ptr;
230+ struct mlx5_eq_async *eq_async =
231+ container_of(nb, struct mlx5_eq_async, irq_nb);
232+ struct mlx5_eq *eq = &eq_async->core;
202233 struct mlx5_eq_table *eqt;
203234 struct mlx5_core_dev *dev;
204235 struct mlx5_eqe *eqe;
@@ -232,7 +263,7 @@ static irqreturn_t mlx5_eq_async_int(int irq, void *eq_ptr) · 27 unmodified lines
232263 out:
233264 eq_update_ci(eq, 1);
234265
235− return IRQ_HANDLED;
266+ return 0;
236267 }
237268
238269 static void init_eq_buf(struct mlx5_eq *eq)
@@ -254,6 +285,7 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, const char *name, · 15 unmodified lines
254285 struct mlx5_cq_table *cq_table = &eq->cq_table;
255286 u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
256287 struct mlx5_priv *priv = &dev->priv;
288+ struct mlx5_irq_info *irq_info;
257289 u8 vecidx = param->index;
258290 __be64 *pas;
259291 void *eqc;
@@ -261,9 +293,6 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, const char *name, · 1 unmodified line
261293 u32 *in;
262294 int err;
263295
264− if (eq_table->irq_info[vecidx].context)
265− return -EEXIST;
266−
267296 /* Init CQ table */
268297 memset(cq_table, 0, sizeof(*cq_table));
269298 spin_lock_init(&cq_table->lock);
@@ -306,24 +335,31 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, const char *name, · 36 unmodified lines
306335 if (err)
307336 goto err_in;
308337
309− snprintf(eq_table->irq_info[vecidx].name, MLX5_MAX_IRQ_NAME, "%s@pci:%s",
310− name, pci_name(dev->pdev));
311− eq_table->irq_info[vecidx].context = param->context;
338+ irq_info = mlx5_irq_get(dev, vecidx);
339+ ATOMIC_INIT_NOTIFIER_HEAD(&irq_info->nh);
340+ snprintf(irq_info->name, MLX5_MAX_IRQ_NAME,
341+ "%s@pci:%s", name, pci_name(dev->pdev));
312342
313343 eq->vecidx = vecidx;
314344 eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
315345 eq->irqn = pci_irq_vector(dev->pdev, vecidx);
316346 eq->dev = dev;
317347 eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
318− err = request_irq(eq->irqn, param->handler, 0,
319− eq_table->irq_info[vecidx].name, param->context);
348+ eq->irq_nb = param->nb;
349+
350+ err = request_irq(eq->irqn, mlx5_irq_int_handler, 0, irq_info->name,
351+ &irq_info->nh);
320352 if (err)
321353 goto err_eq;
322354
323− err = mlx5_debug_eq_add(dev, eq);
355+ err = mlx5_irq_attach_nb(irq_info, param->nb);
324356 if (err)
325357 goto err_irq;
326358
359+ err = mlx5_debug_eq_add(dev, eq);
360+ if (err)
361+ goto err_detach;
362+
327363 /* EQs are created in ARMED state
328364 */
329365 eq_update_ci(eq, 1);
@@ -331,8 +367,11 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, const char *name, · 1 unmodified line
331367 kvfree(in);
332368 return 0;
333369
370+err_detach:
371+ mlx5_irq_detach_nb(irq_info, param->nb);
372+
334373 err_irq:
335− free_irq(eq->irqn, eq);
374+ free_irq(eq->irqn, &eq_table->irq_info[vecidx].nh);
336375
337376 err_eq:
338377 mlx5_cmd_destroy_eq(dev, eq->eqn);
@@ -355,9 +394,11 @@ static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) · 16 unmodified lines
355394
356395 mlx5_debug_eq_remove(dev, eq);
357396
358− free_irq(eq->irqn, irq_info->context);
359− irq_info->context = NULL;
360−
397+ err = mlx5_irq_detach_nb(irq_info, eq->irq_nb);
398+ if (err)
399+ mlx5_core_warn(eq->dev, "eq failed to detach from irq. err %d",
400+ err);
401+ free_irq(eq->irqn, &eq_table->irq_info[eq->vecidx].nh);
361402 err = mlx5_cmd_destroy_eq(dev, eq->eqn);
362403 if (err)
363404 mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
@@ -479,7 +520,7 @@ static int cq_err_event_notifier(struct notifier_block *nb, · 115 unmodified lines
479520 /* type == MLX5_EVENT_TYPE_CQ_ERROR */
480521
481522 eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb);
482− eq = &eqt->async_eq;
523+ eq = &eqt->async_eq.core;
483524 eqe = data;
484525
485526 cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
@@ -548,14 +589,14 @@ static int create_async_eqs(struct mlx5_core_dev *dev) · 62 unmodified lines
548589 MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR);
549590 mlx5_eq_notifier_register(dev, &table->cq_err_nb);
550591
592+ table->cmd_eq.irq_nb.notifier_call = mlx5_eq_async_int;
551593 param = (struct mlx5_eq_param) {
552594 .index = MLX5_EQ_CMD_IDX,
553595 .mask = 1ull << MLX5_EVENT_TYPE_CMD,
554596 .nent = MLX5_NUM_CMD_EQE,
555− .context = &table->cmd_eq,
556− .handler = mlx5_eq_async_int,
597+ .nb = &table->cmd_eq.irq_nb,
557598 };
558− err = create_async_eq(dev, "mlx5_cmd_eq", &table->cmd_eq, &param);
599+ err = create_async_eq(dev, "mlx5_cmd_eq", &table->cmd_eq.core, &param);
559600 if (err) {
560601 mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
561602 goto err0;
@@ -563,27 +604,29 @@ static int create_async_eqs(struct mlx5_core_dev *dev) · 1 unmodified line
563604
564605 mlx5_cmd_use_events(dev);
565606
607+ table->async_eq.irq_nb.notifier_call = mlx5_eq_async_int;
566608 param = (struct mlx5_eq_param) {
567609 .index = MLX5_EQ_ASYNC_IDX,
568610 .mask = gather_async_events_mask(dev),
569611 .nent = MLX5_NUM_ASYNC_EQE,
570− .context = &table->async_eq,
571− .handler = mlx5_eq_async_int,
612+ .nb = &table->async_eq.irq_nb,
572613 };
573− err = create_async_eq(dev, "mlx5_async_eq", &table->async_eq, &param);
614+ err = create_async_eq(dev, "mlx5_async_eq",
615+ &table->async_eq.core, &param);
574616 if (err) {
575617 mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
576618 goto err1;
577619 }
578620
621+ table->pages_eq.irq_nb.notifier_call = mlx5_eq_async_int;
579622 param = (struct mlx5_eq_param) {
580623 .index = MLX5_EQ_PAGEREQ_IDX,
581624 .mask = 1 << MLX5_EVENT_TYPE_PAGE_REQUEST,
582625 .nent = /* TODO: sriov max_vf + */ 1,
583− .context = &table->pages_eq,
584− .handler = mlx5_eq_async_int,
626+ .nb = &table->pages_eq.irq_nb,
585627 };
586− err = create_async_eq(dev, "mlx5_pages_eq", &table->pages_eq, &param);
628+ err = create_async_eq(dev, "mlx5_pages_eq",
629+ &table->pages_eq.core, &param);
587630 if (err) {
588631 mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
589632 goto err2;
@@ -592,11 +635,11 @@ static int create_async_eqs(struct mlx5_core_dev *dev) · 2 unmodified lines
592635 return err;
593636
594637 err2:
595− destroy_async_eq(dev, &table->async_eq);
638+ destroy_async_eq(dev, &table->async_eq.core);
596639
597640 err1:
598641 mlx5_cmd_use_polling(dev);
599− destroy_async_eq(dev, &table->cmd_eq);
642+ destroy_async_eq(dev, &table->cmd_eq.core);
600643 err0:
601644 mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
602645 return err;
@@ -607,19 +650,19 @@ static void destroy_async_eqs(struct mlx5_core_dev *dev) · 4 unmodified lines
607650 struct mlx5_eq_table *table = dev->priv.eq_table;
608651 int err;
609652
610− err = destroy_async_eq(dev, &table->pages_eq);
653+ err = destroy_async_eq(dev, &table->pages_eq.core);
611654 if (err)
612655 mlx5_core_err(dev, "failed to destroy pages eq, err(%d)\n",
613656 err);
614657
615− err = destroy_async_eq(dev, &table->async_eq);
658+ err = destroy_async_eq(dev, &table->async_eq.core);
616659 if (err)
617660 mlx5_core_err(dev, "failed to destroy async eq, err(%d)\n",
618661 err);
619662
620663 mlx5_cmd_use_polling(dev);
621664
622− err = destroy_async_eq(dev, &table->cmd_eq);
665+ err = destroy_async_eq(dev, &table->cmd_eq.core);
623666 if (err)
624667 mlx5_core_err(dev, "failed to destroy command eq, err(%d)\n",
625668 err);
@@ -629,17 +672,17 @@ static void destroy_async_eqs(struct mlx5_core_dev *dev) · 3 unmodified lines
629672
630673 struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev)
631674 {
632− return &dev->priv.eq_table->async_eq;
675+ return &dev->priv.eq_table->async_eq.core;
633676 }
634677
635678 void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev)
636679 {
637− synchronize_irq(dev->priv.eq_table->async_eq.irqn);
680+ synchronize_irq(dev->priv.eq_table->async_eq.core.irqn);
638681 }
639682
640683 void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev)
641684 {
642− synchronize_irq(dev->priv.eq_table->cmd_eq.irqn);
685+ synchronize_irq(dev->priv.eq_table->cmd_eq.core.irqn);
643686 }
644687
645688 /* Generic EQ API for mlx5_core consumers
@@ -837,12 +880,12 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) · 191 unmodified lines
837880 irq_cpu_rmap_add(table->rmap, pci_irq_vector(dev->pdev, vecidx));
838881 #endif
839882 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", i);
883+ eq->irq_nb.notifier_call = mlx5_eq_comp_int;
840884 param = (struct mlx5_eq_param) {
841885 .index = vecidx,
842886 .mask = 0,
843887 .nent = nent,
844− .context = &eq->core,
845− .handler = mlx5_eq_comp_int
888+ .nb = &eq->irq_nb,
846889 };
847890 err = create_map_eq(dev, &eq->core, name, &param);
848891 if (err) {
@@ -940,10 +983,7 @@ void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev) · 91 unmodified lines
940983 mutex_lock(&table->lock); /* sync with create/destroy_async_eq */
941984 max_eqs = table->num_comp_vectors + MLX5_EQ_VEC_COMP_BASE;
942985 for (i = max_eqs - 1; i >= 0; i--) {
943− if (!table->irq_info[i].context)
944− continue;
945− free_irq(pci_irq_vector(dev->pdev, i), table->irq_info[i].context);
946− table->irq_info[i].context = NULL;
986+ free_irq(pci_irq_vector(dev->pdev, i), &table->irq_info[i].nh);
947987 }
948988 mutex_unlock(&table->lock);
949989 pci_free_irq_vectors(dev->pdev);
drivers/net/ethernet/mellanox/mlx5/core/lib/eq.h+8−1
@@ -34,10 +34,17 @@ struct mlx5_eq { · 33 unmodified lines
3434 u8 eqn;
3535 int nent;
3636 struct mlx5_rsc_debug *dbg;
37+ struct notifier_block *irq_nb; /* For destroy only */
38+};
39+
40+struct mlx5_eq_async {
41+ struct mlx5_eq core;
42+ struct notifier_block irq_nb;
3743 };
3844
3945 struct mlx5_eq_comp {
40− struct mlx5_eq core; /* Must be first */
46+ struct mlx5_eq core;
47+ struct notifier_block irq_nb;
4148 struct mlx5_eq_tasklet tasklet_ctx;
4249 struct list_head list;
4350 };
include/linux/mlx5/eq.h+1−2
@@ -26,8 +26,7 @@ struct mlx5_eq_param { · 25 unmodified lines
2626 u8 index;
2727 int nent;
2828 u64 mask;
29− void *context;
30− irq_handler_t handler;
29+ struct notifier_block *nb;
3130 };
3231
3332 struct mlx5_eq *

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t