History

net/mlx5: Introduce API for bulk request and release of IRQs

Currently IRQs are requested one by one. To balance spreading IRQs among cpus using such scheme requires remembering cpu mask for the cpus used for a given device. This complicates the IRQ allocation scheme in subsequent patch. Hence, prepare the code for bulk IRQs allocation. This enables spreading IRQs among cpus in subsequent patch.

Shay Drory committed 4 years ago

Commit 79b60ca1 parent 424544d5 files · +135 −47Browse files

Unified · Split

5 files
  1. drivers/infiniband/hw/mlx5/odp.c +0 −6
  2. drivers/net/ethernet/mellanox/mlx5/core/eq.c +65 −33
  3. drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h +4 −1
  4. drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +64 −5
  5. include/linux/mlx5/eq.h +2 −2
drivers/infiniband/hw/mlx5/odp.c+0−6
@@ -1541,16 +1541,10 @@ int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) · 1540 unmodified lines
15411541
15421542 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int;
15431543 param = (struct mlx5_eq_param) {
1544− .irq_index = MLX5_IRQ_EQ_CTRL,
15451544 .nent = MLX5_IB_NUM_PF_EQE,
15461545 };
15471546 param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT;
1548− if (!zalloc_cpumask_var(&param.affinity, GFP_KERNEL)) {
1549− err = -ENOMEM;
1550− goto err_wq;
1551− }
15521547 eq->core = mlx5_eq_create_generic(dev->mdev, &param);
1553− free_cpumask_var(param.affinity);
15541548 if (IS_ERR(eq->core)) {
15551549 err = PTR_ERR(eq->core);
15561550 goto err_wq;
drivers/net/ethernet/mellanox/mlx5/core/eq.c+65−33
@@ -59,6 +59,8 @@ struct mlx5_eq_table { · 58 unmodified lines
5959 struct mutex lock; /* sync async eqs creations */
6060 int num_comp_eqs;
6161 struct mlx5_irq_table *irq_table;
62+ struct mlx5_irq **comp_irqs;
63+ struct mlx5_irq *ctrl_irq;
6264 #ifdef CONFIG_RFS_ACCEL
6365 struct cpu_rmap *rmap;
6466 #endif
@@ -266,8 +268,8 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, · 201 unmodified lines
266268 u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
267269 u8 log_eq_stride = ilog2(MLX5_EQE_SIZE);
268270 struct mlx5_priv *priv = &dev->priv;
269− u16 vecidx = param->irq_index;
270271 __be64 *pas;
272+ u16 vecidx;
271273 void *eqc;
272274 int inlen;
273275 u32 *in;
@@ -289,23 +291,16 @@ create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, · 15 unmodified lines
289291 mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc);
290292 init_eq_buf(eq);
291293
292− if (vecidx == MLX5_IRQ_EQ_CTRL)
293− eq->irq = mlx5_ctrl_irq_request(dev);
294− else
295− eq->irq = mlx5_irq_request(dev, vecidx, param->affinity);
296− if (IS_ERR(eq->irq)) {
297− err = PTR_ERR(eq->irq);
298− goto err_buf;
299− }
300−
294+ eq->irq = param->irq;
301295 vecidx = mlx5_irq_get_index(eq->irq);
296+
302297 inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
303298 MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages;
304299
305300 in = kvzalloc(inlen, GFP_KERNEL);
306301 if (!in) {
307302 err = -ENOMEM;
308− goto err_irq;
303+ goto err_buf;
309304 }
310305
311306 pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
@@ -349,8 +344,6 @@ err_eq: · 37 unmodified lines
349344 err_in:
350345 kvfree(in);
351346
352−err_irq:
353− mlx5_irq_release(eq->irq);
354347 err_buf:
355348 mlx5_frag_buf_free(dev, &eq->frag_buf);
356349 return err;
@@ -404,7 +397,6 @@ static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq) · 47 unmodified lines
404397 if (err)
405398 mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
406399 eq->eqn);
407− mlx5_irq_release(eq->irq);
408400
409401 mlx5_frag_buf_free(dev, &eq->frag_buf);
410402 return err;
@@ -597,11 +589,8 @@ setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq, · 186 unmodified lines
597589
598590 eq->irq_nb.notifier_call = mlx5_eq_async_int;
599591 spin_lock_init(&eq->lock);
600− if (!zalloc_cpumask_var(&param->affinity, GFP_KERNEL))
601− return -ENOMEM;
602592
603593 err = create_async_eq(dev, &eq->core, param);
604− free_cpumask_var(param->affinity);
605594 if (err) {
606595 mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err);
607596 return err;
@@ -646,11 +635,18 @@ static int create_async_eqs(struct mlx5_core_dev *dev) · 38 unmodified lines
646635 struct mlx5_eq_param param = {};
647636 int err;
648637
638+ /* All the async_eqs are using single IRQ, request one IRQ and share its
639+ * index among all the async_eqs of this device.
640+ */
641+ table->ctrl_irq = mlx5_ctrl_irq_request(dev);
642+ if (IS_ERR(table->ctrl_irq))
643+ return PTR_ERR(table->ctrl_irq);
644+
649645 MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR);
650646 mlx5_eq_notifier_register(dev, &table->cq_err_nb);
651647
652648 param = (struct mlx5_eq_param) {
653− .irq_index = MLX5_IRQ_EQ_CTRL,
649+ .irq = table->ctrl_irq,
654650 .nent = MLX5_NUM_CMD_EQE,
655651 .mask[0] = 1ull << MLX5_EVENT_TYPE_CMD,
656652 };
@@ -663,7 +659,7 @@ static int create_async_eqs(struct mlx5_core_dev *dev) · 6 unmodified lines
663659 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
664660
665661 param = (struct mlx5_eq_param) {
666− .irq_index = MLX5_IRQ_EQ_CTRL,
662+ .irq = table->ctrl_irq,
667663 .nent = async_eq_depth_devlink_param_get(dev),
668664 };
669665
@@ -673,7 +669,7 @@ static int create_async_eqs(struct mlx5_core_dev *dev) · 3 unmodified lines
673669 goto err2;
674670
675671 param = (struct mlx5_eq_param) {
676− .irq_index = MLX5_IRQ_EQ_CTRL,
672+ .irq = table->ctrl_irq,
677673 .nent = /* TODO: sriov max_vf + */ 1,
678674 .mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST,
679675 };
@@ -692,6 +688,7 @@ err2: · 12 unmodified lines
692688 err1:
693689 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
694690 mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
691+ mlx5_ctrl_irq_release(table->ctrl_irq);
695692 return err;
696693 }
697694
@@ -706,6 +703,7 @@ static void destroy_async_eqs(struct mlx5_core_dev *dev) · 8 unmodified lines
706703 cleanup_async_eq(dev, &table->cmd_eq, "cmd");
707704 mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
708705 mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
706+ mlx5_ctrl_irq_release(table->ctrl_irq);
709707 }
710708
711709 struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev)
@@ -733,12 +731,10 @@ mlx5_eq_create_generic(struct mlx5_core_dev *dev, · 21 unmodified lines
733731 struct mlx5_eq *eq = kvzalloc(sizeof(*eq), GFP_KERNEL);
734732 int err;
735733
736− if (!cpumask_available(param->affinity))
737− return ERR_PTR(-EINVAL);
738−
739734 if (!eq)
740735 return ERR_PTR(-ENOMEM);
741736
737+ param->irq = dev->priv.eq_table->ctrl_irq;
742738 err = create_async_eq(dev, eq, param);
743739 if (err) {
744740 kvfree(eq);
@@ -798,6 +794,45 @@ void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm) · 53 unmodified lines
798794 }
799795 EXPORT_SYMBOL(mlx5_eq_update_ci);
800796
797+static void comp_irqs_release(struct mlx5_core_dev *dev)
798+{
799+ struct mlx5_eq_table *table = dev->priv.eq_table;
800+
801+ mlx5_irqs_release_vectors(table->comp_irqs, table->num_comp_eqs);
802+ kfree(table->comp_irqs);
803+}
804+
805+static int comp_irqs_request(struct mlx5_core_dev *dev)
806+{
807+ struct mlx5_eq_table *table = dev->priv.eq_table;
808+ int ncomp_eqs = table->num_comp_eqs;
809+ u16 *cpus;
810+ int ret;
811+ int i;
812+
813+ ncomp_eqs = table->num_comp_eqs;
814+ table->comp_irqs = kcalloc(ncomp_eqs, sizeof(*table->comp_irqs), GFP_KERNEL);
815+ if (!table->comp_irqs)
816+ return -ENOMEM;
817+
818+ cpus = kcalloc(ncomp_eqs, sizeof(*cpus), GFP_KERNEL);
819+ if (!cpus) {
820+ ret = -ENOMEM;
821+ goto free_irqs;
822+ }
823+ for (i = 0; i < ncomp_eqs; i++)
824+ cpus[i] = cpumask_local_spread(i, dev->priv.numa_node);
825+ ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs);
826+ kfree(cpus);
827+ if (ret < 0)
828+ goto free_irqs;
829+ return ret;
830+
831+free_irqs:
832+ kfree(table->comp_irqs);
833+ return ret;
834+}
835+
801836 static void destroy_comp_eqs(struct mlx5_core_dev *dev)
802837 {
803838 struct mlx5_eq_table *table = dev->priv.eq_table;
@@ -812,6 +847,7 @@ static void destroy_comp_eqs(struct mlx5_core_dev *dev) · 8 unmodified lines
812847 tasklet_disable(&eq->tasklet_ctx.task);
813848 kfree(eq);
814849 }
850+ comp_irqs_release(dev);
815851 }
816852
817853 static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
@@ -838,12 +874,13 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) · 20 unmodified lines
838874 int err;
839875 int i;
840876
877+ ncomp_eqs = comp_irqs_request(dev);
878+ if (ncomp_eqs < 0)
879+ return ncomp_eqs;
841880 INIT_LIST_HEAD(&table->comp_eqs_list);
842− ncomp_eqs = table->num_comp_eqs;
843881 nent = comp_eq_depth_devlink_param_get(dev);
844882 for (i = 0; i < ncomp_eqs; i++) {
845883 struct mlx5_eq_param param = {};
846− int vecidx = i;
847884
848885 eq = kzalloc(sizeof(*eq), GFP_KERNEL);
849886 if (!eq) {
@@ -858,18 +895,11 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) · 8 unmodified lines
858895
859896 eq->irq_nb.notifier_call = mlx5_eq_comp_int;
860897 param = (struct mlx5_eq_param) {
861− .irq_index = vecidx,
898+ .irq = table->comp_irqs[i],
862899 .nent = nent,
863900 };
864901
865− if (!zalloc_cpumask_var(&param.affinity, GFP_KERNEL)) {
866− err = -ENOMEM;
867− goto clean_eq;
868− }
869− cpumask_set_cpu(cpumask_local_spread(i, dev->priv.numa_node),
870− param.affinity);
871902 err = create_map_eq(dev, &eq->core, &param);
872− free_cpumask_var(param.affinity);
873903 if (err)
874904 goto clean_eq;
875905 err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
@@ -883,7 +913,9 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) · 7 unmodified lines
883913 list_add_tail(&eq->list, &table->comp_eqs_list);
884914 }
885915
916+ table->num_comp_eqs = ncomp_eqs;
886917 return 0;
918+
887919 clean_eq:
888920 kfree(eq);
889921 clean:
drivers/net/ethernet/mellanox/mlx5/core/mlx5_irq.h+4−1
@@ -23,9 +23,12 @@ int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int devfn, · 22 unmodified lines
2323 int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs);
2424
2525 struct mlx5_irq *mlx5_ctrl_irq_request(struct mlx5_core_dev *dev);
26+void mlx5_ctrl_irq_release(struct mlx5_irq *ctrl_irq);
2627 struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx,
2728 struct cpumask *affinity);
28−void mlx5_irq_release(struct mlx5_irq *irq);
29+int mlx5_irqs_request_vectors(struct mlx5_core_dev *dev, u16 *cpus, int nirqs,
30+ struct mlx5_irq **irqs);
31+void mlx5_irqs_release_vectors(struct mlx5_irq **irqs, int nirqs);
2932 int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb);
3033 int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb);
3134 struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq);
drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c+64−5
@@ -342,13 +342,27 @@ static struct mlx5_irq_pool *ctrl_irq_pool_get(struct mlx5_core_dev *dev) · 341 unmodified lines
342342 }
343343
344344 /**
345− * mlx5_irq_release - release an IRQ back to the system.
346− * @irq: irq to be released.
345+ * mlx5_irqs_release - release one or more IRQs back to the system.
346+ * @irqs: IRQs to be released.
347+ * @nirqs: number of IRQs to be released.
347348 */
348−void mlx5_irq_release(struct mlx5_irq *irq)
349+static void mlx5_irqs_release(struct mlx5_irq **irqs, int nirqs)
349350 {
350− synchronize_irq(irq->irqn);
351− irq_put(irq);
351+ int i;
352+
353+ for (i = 0; i < nirqs; i++) {
354+ synchronize_irq(irqs[i]->irqn);
355+ irq_put(irqs[i]);
356+ }
357+}
358+
359+/**
360+ * mlx5_ctrl_irq_release - release a ctrl IRQ back to the system.
361+ * @ctrl_irq: ctrl IRQ to be released.
362+ */
363+void mlx5_ctrl_irq_release(struct mlx5_irq *ctrl_irq)
364+{
365+ mlx5_irqs_release(&ctrl_irq, 1);
352366 }
353367
354368 /**
@@ -423,6 +437,51 @@ out: · 68 unmodified lines
423437 return irq;
424438 }
425439
440+/**
441+ * mlx5_irqs_release_vectors - release one or more IRQs back to the system.
442+ * @irqs: IRQs to be released.
443+ * @nirqs: number of IRQs to be released.
444+ */
445+void mlx5_irqs_release_vectors(struct mlx5_irq **irqs, int nirqs)
446+{
447+ mlx5_irqs_release(irqs, nirqs);
448+}
449+
450+/**
451+ * mlx5_irqs_request_vectors - request one or more IRQs for mlx5 device.
452+ * @dev: mlx5 device that is requesting the IRQs.
453+ * @cpus: CPUs array for binding the IRQs
454+ * @nirqs: number of IRQs to request.
455+ * @irqs: an output array of IRQs pointers.
456+ *
457+ * Each IRQ is bound to at most 1 CPU.
458+ * This function is requests nirqs IRQs, starting from @vecidx.
459+ *
460+ * This function returns the number of IRQs requested, (which might be smaller than
461+ * @nirqs), if successful, or a negative error code in case of an error.
462+ */
463+int mlx5_irqs_request_vectors(struct mlx5_core_dev *dev, u16 *cpus, int nirqs,
464+ struct mlx5_irq **irqs)
465+{
466+ cpumask_var_t req_mask;
467+ struct mlx5_irq *irq;
468+ int i;
469+
470+ if (!zalloc_cpumask_var(&req_mask, GFP_KERNEL))
471+ return -ENOMEM;
472+ for (i = 0; i < nirqs; i++) {
473+ cpumask_set_cpu(cpus[i], req_mask);
474+ irq = mlx5_irq_request(dev, i, req_mask);
475+ if (IS_ERR(irq))
476+ break;
477+ cpumask_clear(req_mask);
478+ irqs[i] = irq;
479+ }
480+
481+ free_cpumask_var(req_mask);
482+ return i ? i : PTR_ERR(irq);
483+}
484+
426485 static struct mlx5_irq_pool *
427486 irq_pool_alloc(struct mlx5_core_dev *dev, int start, int size, char *name,
428487 u32 min_threshold, u32 max_threshold)
include/linux/mlx5/eq.h+2−2
@@ -9,13 +9,13 @@ · 8 unmodified lines
99 #define MLX5_NUM_SPARE_EQE (0x80)
1010
1111 struct mlx5_eq;
12+struct mlx5_irq;
1213 struct mlx5_core_dev;
1314
1415 struct mlx5_eq_param {
15− u8 irq_index;
1616 int nent;
1717 u64 mask[4];
18− cpumask_var_t affinity;
18+ struct mlx5_irq *irq;
1919 };
2020
2121 struct mlx5_eq *

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t