The lock protecting the data structure does not need to be an rwlock. The only read access to the lock is in an error path, and if that's limiting your scalability, you have bigger performance problems. Eliminate mlx5_mkey_table in favour of using the xarray directly. reg_mr_callback must use GFP_ATOMIC for allocating XArray nodes as it may be called in interrupt context. This also fixes a minor bug where SRCU locking was being used on the radix tree read side, when RCU was needed too.
Unified · Split
| @@ -522,9 +522,9 @@ repoll: · 521 unmodified lines | |||
| 522 | case MLX5_CQE_SIG_ERR: | 522 | case MLX5_CQE_SIG_ERR: |
| 523 | sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64; | 523 | sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64; |
| 524 | 524 | ||
| 525 | read_lock(&dev->mdev->priv.mkey_table.lock); | 525 | xa_lock(&dev->mdev->priv.mkey_table); |
| 526 | mmkey = __mlx5_mr_lookup(dev->mdev, | 526 | mmkey = xa_load(&dev->mdev->priv.mkey_table, |
| 527 | mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey))); | 527 | mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey))); |
| 528 | mr = to_mibmr(mmkey); | 528 | mr = to_mibmr(mmkey); |
| 529 | get_sig_err_item(sig_err_cqe, &mr->sig->err_item); | 529 | get_sig_err_item(sig_err_cqe, &mr->sig->err_item); |
| 530 | mr->sig->sig_err_exists = true; | 530 | mr->sig->sig_err_exists = true; |
| @@ -537,7 +537,7 @@ repoll: · 6 unmodified lines | |||
| 537 | mr->sig->err_item.expected, | 537 | mr->sig->err_item.expected, |
| 538 | mr->sig->err_item.actual); | 538 | mr->sig->err_item.actual); |
| 539 | 539 | ||
| 540 | read_unlock(&dev->mdev->priv.mkey_table.lock); | 540 | xa_unlock(&dev->mdev->priv.mkey_table); |
| 541 | goto repoll; | 541 | goto repoll; |
| 542 | } | 542 | } |
| 543 | 543 | ||
| @@ -1043,13 +1043,10 @@ static int devx_handle_mkey_indirect(struct devx_obj *obj, · 1042 unmodified lines | |||
| 1043 | struct mlx5_ib_dev *dev, | 1043 | struct mlx5_ib_dev *dev, |
| 1044 | void *in, void *out) | 1044 | void *in, void *out) |
| 1045 | { | 1045 | { |
| 1046 | struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table; | ||
| 1047 | struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr; | 1046 | struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr; |
| 1048 | unsigned long flags; | ||
| 1049 | struct mlx5_core_mkey *mkey; | 1047 | struct mlx5_core_mkey *mkey; |
| 1050 | void *mkc; | 1048 | void *mkc; |
| 1051 | u8 key; | 1049 | u8 key; |
| 1052 | int err; | ||
| 1053 | 1050 | ||
| 1054 | mkey = &devx_mr->mmkey; | 1051 | mkey = &devx_mr->mmkey; |
| 1055 | mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); | 1052 | mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); |
| @@ -1062,11 +1059,8 @@ static int devx_handle_mkey_indirect(struct devx_obj *obj, · 6 unmodified lines | |||
| 1062 | mkey->pd = MLX5_GET(mkc, mkc, pd); | 1059 | mkey->pd = MLX5_GET(mkc, mkc, pd); |
| 1063 | devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size); | 1060 | devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size); |
| 1064 | 1061 | ||
| 1065 | write_lock_irqsave(&table->lock, flags); | 1062 | return xa_err(xa_store(&dev->mdev->priv.mkey_table, |
| 1066 | err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key), | 1063 | mlx5_base_mkey(mkey->key), mkey, GFP_KERNEL)); |
| 1067 | mkey); | ||
| 1068 | write_unlock_irqrestore(&table->lock, flags); | ||
| 1069 | return err; | ||
| 1070 | } | 1064 | } |
| 1071 | 1065 | ||
| 1072 | static int devx_handle_mkey_create(struct mlx5_ib_dev *dev, | 1066 | static int devx_handle_mkey_create(struct mlx5_ib_dev *dev, |
| @@ -1117,12 +1111,8 @@ static void devx_free_indirect_mkey(struct rcu_head *rcu) · 44 unmodified lines | |||
| 1117 | */ | 1111 | */ |
| 1118 | static void devx_cleanup_mkey(struct devx_obj *obj) | 1112 | static void devx_cleanup_mkey(struct devx_obj *obj) |
| 1119 | { | 1113 | { |
| 1120 | struct mlx5_mkey_table *table = &obj->mdev->priv.mkey_table; | 1114 | xa_erase(&obj->mdev->priv.mkey_table, |
| 1121 | unsigned long flags; | 1115 | mlx5_base_mkey(obj->devx_mr.mmkey.key)); |
| 1122 | |||
| 1123 | write_lock_irqsave(&table->lock, flags); | ||
| 1124 | radix_tree_delete(&table->tree, mlx5_base_mkey(obj->devx_mr.mmkey.key)); | ||
| 1125 | write_unlock_irqrestore(&table->lock, flags); | ||
| 1126 | } | 1116 | } |
| 1127 | 1117 | ||
| 1128 | static int devx_obj_cleanup(struct ib_uobject *uobject, | 1118 | static int devx_obj_cleanup(struct ib_uobject *uobject, |
| @@ -130,7 +130,7 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context) · 129 unmodified lines | |||
| 130 | struct mlx5_cache_ent *ent = &cache->ent[c]; | 130 | struct mlx5_cache_ent *ent = &cache->ent[c]; |
| 131 | u8 key; | 131 | u8 key; |
| 132 | unsigned long flags; | 132 | unsigned long flags; |
| 133 | struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table; | 133 | struct xarray *mkeys = &dev->mdev->priv.mkey_table; |
| 134 | int err; | 134 | int err; |
| 135 | 135 | ||
| 136 | spin_lock_irqsave(&ent->lock, flags); | 136 | spin_lock_irqsave(&ent->lock, flags); |
| @@ -158,12 +158,12 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context) · 21 unmodified lines | |||
| 158 | ent->size++; | 158 | ent->size++; |
| 159 | spin_unlock_irqrestore(&ent->lock, flags); | 159 | spin_unlock_irqrestore(&ent->lock, flags); |
| 160 | 160 | ||
| 161 | write_lock_irqsave(&table->lock, flags); | 161 | xa_lock_irqsave(mkeys, flags); |
| 162 | err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key), | 162 | err = xa_err(__xa_store(mkeys, mlx5_base_mkey(mr->mmkey.key), |
| 163 | &mr->mmkey); | 163 | &mr->mmkey, GFP_ATOMIC)); |
| 164 | xa_unlock_irqrestore(mkeys, flags); | ||
| 164 | if (err) | 165 | if (err) |
| 165 | pr_err("Error inserting to mkey tree. 0x%x\n", -err); | 166 | pr_err("Error inserting to mkey tree. 0x%x\n", -err); |
| 166 | write_unlock_irqrestore(&table->lock, flags); | ||
| 167 | 167 | ||
| 168 | if (!completion_done(&ent->compl)) | 168 | if (!completion_done(&ent->compl)) |
| 169 | complete(&ent->compl); | 169 | complete(&ent->compl); |
| @@ -768,7 +768,7 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, · 767 unmodified lines | |||
| 768 | bcnt -= *bytes_committed; | 768 | bcnt -= *bytes_committed; |
| 769 | 769 | ||
| 770 | next_mr: | 770 | next_mr: |
| 771 | mmkey = __mlx5_mr_lookup(dev->mdev, mlx5_base_mkey(key)); | 771 | mmkey = xa_load(&dev->mdev->priv.mkey_table, mlx5_base_mkey(key)); |
| 772 | if (!mkey_is_eq(mmkey, key)) { | 772 | if (!mkey_is_eq(mmkey, key)) { |
| 773 | mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); | 773 | mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); |
| 774 | ret = -EFAULT; | 774 | ret = -EFAULT; |
| @@ -1686,8 +1686,8 @@ static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev, · 911 unmodified lines | |||
| 1686 | struct mlx5_core_mkey *mmkey; | 1686 | struct mlx5_core_mkey *mmkey; |
| 1687 | struct mlx5_ib_mr *mr; | 1687 | struct mlx5_ib_mr *mr; |
| 1688 | 1688 | ||
| 1689 | mmkey = __mlx5_mr_lookup(dev->mdev, | 1689 | mmkey = xa_load(&dev->mdev->priv.mkey_table, |
| 1690 | mlx5_base_mkey(sg_list[i].lkey)); | 1690 | mlx5_base_mkey(sg_list[i].lkey)); |
| 1691 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); | 1691 | mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); |
| 1692 | atomic_dec(&mr->num_pending_prefetch); | 1692 | atomic_dec(&mr->num_pending_prefetch); |
| 1693 | } | 1693 | } |
| @@ -1706,8 +1706,8 @@ static bool num_pending_prefetch_inc(struct ib_pd *pd, · 12 unmodified lines | |||
| 1706 | struct mlx5_core_mkey *mmkey; | 1706 | struct mlx5_core_mkey *mmkey; |
| 1707 | struct mlx5_ib_mr *mr; | 1707 | struct mlx5_ib_mr *mr; |
| 1708 | 1708 | ||
| 1709 | mmkey = __mlx5_mr_lookup(dev->mdev, | 1709 | mmkey = xa_load(&dev->mdev->priv.mkey_table, |
| 1710 | mlx5_base_mkey(sg_list[i].lkey)); | 1710 | mlx5_base_mkey(sg_list[i].lkey)); |
| 1711 | if (!mmkey || mmkey->key != sg_list[i].lkey) { | 1711 | if (!mmkey || mmkey->key != sg_list[i].lkey) { |
| 1712 | ret = false; | 1712 | ret = false; |
| 1713 | break; | 1713 | break; |
| @@ -38,15 +38,12 @@ · 37 unmodified lines | |||
| 38 | 38 | ||
| 39 | void mlx5_init_mkey_table(struct mlx5_core_dev *dev) | 39 | void mlx5_init_mkey_table(struct mlx5_core_dev *dev) |
| 40 | { | 40 | { |
| 41 | struct mlx5_mkey_table *table = &dev->priv.mkey_table; | 41 | xa_init_flags(&dev->priv.mkey_table, XA_FLAGS_LOCK_IRQ); |
| 42 | |||
| 43 | memset(table, 0, sizeof(*table)); | ||
| 44 | rwlock_init(&table->lock); | ||
| 45 | INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); | ||
| 46 | } | 42 | } |
| 47 | 43 | ||
| 48 | void mlx5_cleanup_mkey_table(struct mlx5_core_dev *dev) | 44 | void mlx5_cleanup_mkey_table(struct mlx5_core_dev *dev) |
| 49 | { | 45 | { |
| 46 | WARN_ON(!xa_empty(&dev->priv.mkey_table)); | ||
| 50 | } | 47 | } |
| 51 | 48 | ||
| 52 | int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, | 49 | int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, |
| @@ -56,8 +53,8 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, · 3 unmodified lines | |||
| 56 | mlx5_async_cbk_t callback, | 53 | mlx5_async_cbk_t callback, |
| 57 | struct mlx5_async_work *context) | 54 | struct mlx5_async_work *context) |
| 58 | { | 55 | { |
| 59 | struct mlx5_mkey_table *table = &dev->priv.mkey_table; | ||
| 60 | u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0}; | 56 | u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0}; |
| 57 | struct xarray *mkeys = &dev->priv.mkey_table; | ||
| 61 | u32 mkey_index; | 58 | u32 mkey_index; |
| 62 | void *mkc; | 59 | void *mkc; |
| 63 | int err; | 60 | int err; |
| @@ -88,12 +85,10 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, · 24 unmodified lines | |||
| 88 | mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n", | 85 | mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n", |
| 89 | mkey_index, key, mkey->key); | 86 | mkey_index, key, mkey->key); |
| 90 | 87 | ||
| 91 | /* connect to mkey tree */ | 88 | err = xa_err(xa_store_irq(mkeys, mlx5_base_mkey(mkey->key), mkey, |
| 92 | write_lock_irq(&table->lock); | 89 | GFP_KERNEL)); |
| 93 | err = radix_tree_insert(&table->tree, mlx5_base_mkey(mkey->key), mkey); | ||
| 94 | write_unlock_irq(&table->lock); | ||
| 95 | if (err) { | 90 | if (err) { |
| 96 | mlx5_core_warn(dev, "failed radix tree insert of mkey 0x%x, %d\n", | 91 | mlx5_core_warn(dev, "failed xarray insert of mkey 0x%x, %d\n", |
| 97 | mlx5_base_mkey(mkey->key), err); | 92 | mlx5_base_mkey(mkey->key), err); |
| 98 | mlx5_core_destroy_mkey(dev, mkey); | 93 | mlx5_core_destroy_mkey(dev, mkey); |
| 99 | } | 94 | } |
| @@ -114,17 +109,17 @@ EXPORT_SYMBOL(mlx5_core_create_mkey); · 14 unmodified lines | |||
| 114 | int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, | 109 | int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, |
| 115 | struct mlx5_core_mkey *mkey) | 110 | struct mlx5_core_mkey *mkey) |
| 116 | { | 111 | { |
| 117 | struct mlx5_mkey_table *table = &dev->priv.mkey_table; | ||
| 118 | u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0}; | 112 | u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0}; |
| 119 | u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0}; | 113 | u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0}; |
| 114 | struct xarray *mkeys = &dev->priv.mkey_table; | ||
| 120 | struct mlx5_core_mkey *deleted_mkey; | 115 | struct mlx5_core_mkey *deleted_mkey; |
| 121 | unsigned long flags; | 116 | unsigned long flags; |
| 122 | 117 | ||
| 123 | write_lock_irqsave(&table->lock, flags); | 118 | xa_lock_irqsave(mkeys, flags); |
| 124 | deleted_mkey = radix_tree_delete(&table->tree, mlx5_base_mkey(mkey->key)); | 119 | deleted_mkey = __xa_erase(mkeys, mlx5_base_mkey(mkey->key)); |
| 125 | write_unlock_irqrestore(&table->lock, flags); | 120 | xa_unlock_irqrestore(mkeys, flags); |
| 126 | if (!deleted_mkey) { | 121 | if (!deleted_mkey) { |
| 127 | mlx5_core_dbg(dev, "failed radix tree delete of mkey 0x%x\n", | 122 | mlx5_core_dbg(dev, "failed xarray delete of mkey 0x%x\n", |
| 128 | mlx5_base_mkey(mkey->key)); | 123 | mlx5_base_mkey(mkey->key)); |
| 129 | return -ENOENT; | 124 | return -ENOENT; |
| 130 | } | 125 | } |
| @@ -41,7 +41,7 @@ · 40 unmodified lines | |||
| 41 | #include <linux/semaphore.h> | 41 | #include <linux/semaphore.h> |
| 42 | #include <linux/slab.h> | 42 | #include <linux/slab.h> |
| 43 | #include <linux/vmalloc.h> | 43 | #include <linux/vmalloc.h> |
| 44 | #include <linux/radix-tree.h> | 44 | #include <linux/xarray.h> |
| 45 | #include <linux/workqueue.h> | 45 | #include <linux/workqueue.h> |
| 46 | #include <linux/mempool.h> | 46 | #include <linux/mempool.h> |
| 47 | #include <linux/interrupt.h> | 47 | #include <linux/interrupt.h> |
| @@ -452,13 +452,6 @@ struct mlx5_qp_table { · 404 unmodified lines | |||
| 452 | struct radix_tree_root tree; | 452 | struct radix_tree_root tree; |
| 453 | }; | 453 | }; |
| 454 | 454 | ||
| 455 | struct mlx5_mkey_table { | ||
| 456 | /* protect radix tree | ||
| 457 | */ | ||
| 458 | rwlock_t lock; | ||
| 459 | struct radix_tree_root tree; | ||
| 460 | }; | ||
| 461 | |||
| 462 | struct mlx5_vf_context { | 455 | struct mlx5_vf_context { |
| 463 | int enabled; | 456 | int enabled; |
| 464 | u64 port_guid; | 457 | u64 port_guid; |
| @@ -546,9 +539,7 @@ struct mlx5_priv { · 81 unmodified lines | |||
| 546 | struct dentry *cmdif_debugfs; | 539 | struct dentry *cmdif_debugfs; |
| 547 | /* end: qp staff */ | 540 | /* end: qp staff */ |
| 548 | 541 | ||
| 549 | /* start: mkey staff */ | 542 | struct xarray mkey_table; |
| 550 | struct mlx5_mkey_table mkey_table; | ||
| 551 | /* end: mkey staff */ | ||
| 552 | 543 | ||
| 553 | /* start: alloc staff */ | 544 | /* start: alloc staff */ |
| 554 | /* protect buffer alocation according to numa node */ | 545 | /* protect buffer alocation according to numa node */ |
| @@ -551,11 +551,6 @@ static inline struct mlx5_core_qp *__mlx5_qp_lookup(struct mlx5_core_dev *dev, u · 550 unmodified lines | |||
| 551 | return radix_tree_lookup(&dev->priv.qp_table.tree, qpn); | 551 | return radix_tree_lookup(&dev->priv.qp_table.tree, qpn); |
| 552 | } | 552 | } |
| 553 | 553 | ||
| 554 | static inline struct mlx5_core_mkey *__mlx5_mr_lookup(struct mlx5_core_dev *dev, u32 key) | ||
| 555 | { | ||
| 556 | return radix_tree_lookup(&dev->priv.mkey_table.tree, key); | ||
| 557 | } | ||
| 558 | |||
| 559 | int mlx5_core_create_dct(struct mlx5_core_dev *dev, | 554 | int mlx5_core_create_dct(struct mlx5_core_dev *dev, |
| 560 | struct mlx5_core_dct *qp, | 555 | struct mlx5_core_dct *qp, |
| 561 | u32 *in, int inlen, | 556 | u32 *in, int inlen, |