History

Merge branch 'odp_rework' into rdma.git for-next

Jason Gunthorpe says: ==================== In order to hoist the interval tree code out of the drivers and into the mmu_notifiers it is necessary for the drivers to not use the interval tree for other things. This series replaces the interval tree with an xarray and along the way re-aligns all the locking to use a sensible SRCU model where the 'update' step is done by modifying an xarray. The result is overall much simpler and with less locking in the critical path. Many functions were reworked for clarity and small details like using 'imr' to refer to the implicit MR make the entire code flow here more readable. This also squashes at least two race bugs on its own, and quite possibily more that haven't been identified. ==================== Merge conflicts with the odp statistics patch resolved. * branch 'odp_rework': RDMA/odp: Remove broken debugging call to invalidate_range RDMA/mlx5: Do not race with mlx5_ib_invalidate_range during create and destroy RDMA/mlx5: Do not store implicit children in the odp_mkeys xarray RDMA/mlx5: Rework implicit ODP destroy RDMA/mlx5: Avoid double lookups on the pagefault path RDMA/mlx5: Reduce locking in implicit_mr_get_data() RDMA/mlx5: Use an xarray for the children of an implicit ODP RDMA/mlx5: Split implicit handling from pagefault_mr RDMA/mlx5: Set the HW IOVA of the child MRs to their place in the tree RDMA/mlx5: Lift implicit_mr_alloc() into the two routines that call it RDMA/mlx5: Rework implicit_mr_get_data RDMA/mlx5: Delete struct mlx5_priv->mkey_table RDMA/mlx5: Use a dedicated mkey xarray for ODP RDMA/mlx5: Split sig_err MR data into its own xarray RDMA/mlx5: Use SRCU properly in ODP prefetch

Jason Gunthorpe committed 7 years ago

Commit bb3dba32 parents 0363133 46870b211 files · +624 −663Browse files

Unified · Split

11 files
  1. drivers/infiniband/core/umem_odp.c +19 −19
  2. drivers/infiniband/hw/mlx5/cq.c +16 −17
  3. drivers/infiniband/hw/mlx5/devx.c +4 −4
  4. drivers/infiniband/hw/mlx5/main.c +8 −9
  5. drivers/infiniband/hw/mlx5/mlx5_ib.h +18 −6
  6. drivers/infiniband/hw/mlx5/mr.c +59 −80
  7. drivers/infiniband/hw/mlx5/odp.c +499 −475
  8. drivers/net/ethernet/mellanox/mlx5/core/main.c +0 −4
  9. drivers/net/ethernet/mellanox/mlx5/core/mr.c +1 −27
  10. include/linux/mlx5/driver.h +0 −4
  11. include/rdma/ib_umem_odp.h +0 −18
drivers/infiniband/core/umem_odp.c+19−19
@@ -508,7 +508,6 @@ static int ib_umem_odp_map_dma_single_page( · 507 unmodified lines
508{508{
509 struct ib_device *dev = umem_odp->umem.ibdev;509 struct ib_device *dev = umem_odp->umem.ibdev;
510 dma_addr_t dma_addr;510 dma_addr_t dma_addr;
511 int remove_existing_mapping = 0;
512 int ret = 0;511 int ret = 0;
513 512
514 /*513 /*
@@ -534,28 +533,29 @@ static int ib_umem_odp_map_dma_single_page( · 19 unmodified lines
534 } else if (umem_odp->page_list[page_index] == page) {533 } else if (umem_odp->page_list[page_index] == page) {
535 umem_odp->dma_list[page_index] |= access_mask;534 umem_odp->dma_list[page_index] |= access_mask;
536 } else {535 } else {
537 pr_err("error: got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n",536 /*
538 umem_odp->page_list[page_index], page);537 * This is a race here where we could have done:
539 /* Better remove the mapping now, to prevent any further538 *
540 * damage. */539 * CPU0 CPU1
541 remove_existing_mapping = 1;540 * get_user_pages()
541 * invalidate()
542 * page_fault()
543 * mutex_lock(umem_mutex)
544 * page from GUP != page in ODP
545 *
546 * It should be prevented by the retry test above as reading
547 * the seq number should be reliable under the
548 * umem_mutex. Thus something is really not working right if
549 * things get here.
550 */
551 WARN(true,
552 "Got different pages in IB device and from get_user_pages. IB device page: %p, gup page: %p\n",
553 umem_odp->page_list[page_index], page);
554 ret = -EAGAIN;
542 }555 }
543 556
544out:557out:
545 put_user_page(page);558 put_user_page(page);
546
547 if (remove_existing_mapping) {
548 ib_umem_notifier_start_account(umem_odp);
549 dev->ops.invalidate_range(
550 umem_odp,
551 ib_umem_start(umem_odp) +
552 (page_index << umem_odp->page_shift),
553 ib_umem_start(umem_odp) +
554 ((page_index + 1) << umem_odp->page_shift));
555 ib_umem_notifier_end_account(umem_odp);
556 ret = -EAGAIN;
557 }
558
559 return ret;559 return ret;
560}560}
561 561
drivers/infiniband/hw/mlx5/cq.c+16−17
@@ -423,9 +423,6 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq, · 422 unmodified lines
423 struct mlx5_cqe64 *cqe64;423 struct mlx5_cqe64 *cqe64;
424 struct mlx5_core_qp *mqp;424 struct mlx5_core_qp *mqp;
425 struct mlx5_ib_wq *wq;425 struct mlx5_ib_wq *wq;
426 struct mlx5_sig_err_cqe *sig_err_cqe;
427 struct mlx5_core_mkey *mmkey;
428 struct mlx5_ib_mr *mr;
429 uint8_t opcode;426 uint8_t opcode;
430 uint32_t qpn;427 uint32_t qpn;
431 u16 wqe_ctr;428 u16 wqe_ctr;
@@ -519,27 +516,29 @@ repoll: · 87 unmodified lines
519 }516 }
520 }517 }
521 break;518 break;
522 case MLX5_CQE_SIG_ERR:519 case MLX5_CQE_SIG_ERR: {
523 sig_err_cqe = (struct mlx5_sig_err_cqe *)cqe64;520 struct mlx5_sig_err_cqe *sig_err_cqe =
521 (struct mlx5_sig_err_cqe *)cqe64;
522 struct mlx5_core_sig_ctx *sig;
524 523
525 xa_lock(&dev->mdev->priv.mkey_table);524 xa_lock(&dev->sig_mrs);
526 mmkey = xa_load(&dev->mdev->priv.mkey_table,525 sig = xa_load(&dev->sig_mrs,
527 mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));526 mlx5_base_mkey(be32_to_cpu(sig_err_cqe->mkey)));
528 mr = to_mibmr(mmkey);527 get_sig_err_item(sig_err_cqe, &sig->err_item);
529 get_sig_err_item(sig_err_cqe, &mr->sig->err_item);528 sig->sig_err_exists = true;
530 mr->sig->sig_err_exists = true;529 sig->sigerr_count++;
531 mr->sig->sigerr_count++;
532 530
533 mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",531 mlx5_ib_warn(dev, "CQN: 0x%x Got SIGERR on key: 0x%x err_type %x err_offset %llx expected %x actual %x\n",
534 cq->mcq.cqn, mr->sig->err_item.key,532 cq->mcq.cqn, sig->err_item.key,
535 mr->sig->err_item.err_type,533 sig->err_item.err_type,
536 mr->sig->err_item.sig_err_offset,534 sig->err_item.sig_err_offset,
537 mr->sig->err_item.expected,535 sig->err_item.expected,
538 mr->sig->err_item.actual);536 sig->err_item.actual);
539 537
540 xa_unlock(&dev->mdev->priv.mkey_table);538 xa_unlock(&dev->sig_mrs);
541 goto repoll;539 goto repoll;
542 }540 }
541 }
543 542
544 return 0;543 return 0;
545}544}
drivers/infiniband/hw/mlx5/devx.c+4−4
@@ -1265,8 +1265,8 @@ static int devx_handle_mkey_indirect(struct devx_obj *obj, · 1264 unmodified lines
1265 mkey->pd = MLX5_GET(mkc, mkc, pd);1265 mkey->pd = MLX5_GET(mkc, mkc, pd);
1266 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);1266 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1267 1267
1268 return xa_err(xa_store(&dev->mdev->priv.mkey_table,1268 return xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(mkey->key), mkey,
1269 mlx5_base_mkey(mkey->key), mkey, GFP_KERNEL));1269 GFP_KERNEL));
1270}1270}
1271 1271
1272static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,1272static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
@@ -1345,9 +1345,9 @@ static int devx_obj_cleanup(struct ib_uobject *uobject, · 72 unmodified lines
1345 * the mmkey, we must wait for that to stop before freeing the1345 * the mmkey, we must wait for that to stop before freeing the
1346 * mkey, as another allocation could get the same mkey #.1346 * mkey, as another allocation could get the same mkey #.
1347 */1347 */
1348 xa_erase(&obj->ib_dev->mdev->priv.mkey_table,1348 xa_erase(&obj->ib_dev->odp_mkeys,
1349 mlx5_base_mkey(obj->devx_mr.mmkey.key));1349 mlx5_base_mkey(obj->devx_mr.mmkey.key));
1350 synchronize_srcu(&dev->mr_srcu);1350 synchronize_srcu(&dev->odp_srcu);
1351 }1351 }
1352 1352
1353 if (obj->flags & DEVX_OBJ_FLAGS_DCT)1353 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
drivers/infiniband/hw/mlx5/main.c+8−9
@@ -6133,11 +6133,10 @@ static struct ib_counters *mlx5_ib_create_counters(struct ib_device *device, · 6132 unmodified lines
6133static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)6133static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
6134{6134{
6135 mlx5_ib_cleanup_multiport_master(dev);6135 mlx5_ib_cleanup_multiport_master(dev);
6136 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {6136 WARN_ON(!xa_empty(&dev->odp_mkeys));
6137 srcu_barrier(&dev->mr_srcu);6137 cleanup_srcu_struct(&dev->odp_srcu);
6138 cleanup_srcu_struct(&dev->mr_srcu);
6139 }
6140 6138
6139 WARN_ON(!xa_empty(&dev->sig_mrs));
6141 WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));6140 WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
6142}6141}
6143 6142
@@ -6189,15 +6188,15 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) · 45 unmodified lines
6189 mutex_init(&dev->cap_mask_mutex);6188 mutex_init(&dev->cap_mask_mutex);
6190 INIT_LIST_HEAD(&dev->qp_list);6189 INIT_LIST_HEAD(&dev->qp_list);
6191 spin_lock_init(&dev->reset_flow_resource_lock);6190 spin_lock_init(&dev->reset_flow_resource_lock);
6191 xa_init(&dev->odp_mkeys);
6192 xa_init(&dev->sig_mrs);
6192 6193
6193 spin_lock_init(&dev->dm.lock);6194 spin_lock_init(&dev->dm.lock);
6194 dev->dm.dev = mdev;6195 dev->dm.dev = mdev;
6195 6196
6196 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {6197 err = init_srcu_struct(&dev->odp_srcu);
6197 err = init_srcu_struct(&dev->mr_srcu);6198 if (err)
6198 if (err)6199 goto err_mp;
6199 goto err_mp;
6200 }
6201 6200
6202 return 0;6201 return 0;
6203 6202
drivers/infiniband/hw/mlx5/mlx5_ib.h+18−6
@@ -604,7 +604,6 @@ struct mlx5_ib_mr { · 603 unmodified lines
604 struct mlx5_ib_dev *dev;604 struct mlx5_ib_dev *dev;
605 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];605 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
606 struct mlx5_core_sig_ctx *sig;606 struct mlx5_core_sig_ctx *sig;
607 unsigned int live;
608 void *descs_alloc;607 void *descs_alloc;
609 int access_flags; /* Needed for rereg MR */608 int access_flags; /* Needed for rereg MR */
610 609
@@ -616,12 +615,18 @@ struct mlx5_ib_mr { · 5 unmodified lines
616 u64 data_iova;615 u64 data_iova;
617 u64 pi_iova;616 u64 pi_iova;
618 617
619 atomic_t num_leaf_free;618 /* For ODP and implicit */
620 wait_queue_head_t q_leaf_free;619 atomic_t num_deferred_work;
621 struct mlx5_async_work cb_work;620 struct xarray implicit_children;
622 atomic_t num_pending_prefetch;621 union {
622 struct rcu_head rcu;
623 struct list_head elm;
624 struct work_struct work;
625 } odp_destroy;
623 struct ib_odp_counters odp_stats;626 struct ib_odp_counters odp_stats;
624 bool is_odp_implicit;627 bool is_odp_implicit;
628
629 struct mlx5_async_work cb_work;
625};630};
626 631
627static inline bool is_odp_mr(struct mlx5_ib_mr *mr)632static inline bool is_odp_mr(struct mlx5_ib_mr *mr)
@@ -977,7 +982,9 @@ struct mlx5_ib_dev { · 349 unmodified lines
977 * Sleepable RCU that prevents destruction of MRs while they are still982 * Sleepable RCU that prevents destruction of MRs while they are still
978 * being used by a page fault handler.983 * being used by a page fault handler.
979 */984 */
980 struct srcu_struct mr_srcu;985 struct srcu_struct odp_srcu;
986 struct xarray odp_mkeys;
987
981 u32 null_mkey;988 u32 null_mkey;
982 struct mlx5_ib_flow_db *flow_db;989 struct mlx5_ib_flow_db *flow_db;
983 /* protect resources needed as part of reset flow */990 /* protect resources needed as part of reset flow */
@@ -999,6 +1006,8 @@ struct mlx5_ib_dev { · 15 unmodified lines
999 struct mlx5_srq_table srq_table;1006 struct mlx5_srq_table srq_table;
1000 struct mlx5_async_ctx async_ctx;1007 struct mlx5_async_ctx async_ctx;
1001 struct mlx5_devx_event_table devx_event_table;1008 struct mlx5_devx_event_table devx_event_table;
1009
1010 struct xarray sig_mrs;
1002};1011};
1003 1012
1004static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)1013static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
@@ -1162,6 +1171,7 @@ struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, · 157 unmodified lines
1162 struct ib_udata *udata,1171 struct ib_udata *udata,
1163 int access_flags);1172 int access_flags);
1164void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);1173void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);
1174void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr);
1165int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,1175int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
1166 u64 length, u64 virt_addr, int access_flags,1176 u64 length, u64 virt_addr, int access_flags,
1167 struct ib_pd *pd, struct ib_udata *udata);1177 struct ib_pd *pd, struct ib_udata *udata);
@@ -1223,6 +1233,8 @@ int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev); · 55 unmodified lines
1223 1233
1224struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);1234struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);
1225void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);1235void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
1236int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr);
1237
1226int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,1238int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1227 struct ib_mr_status *mr_status);1239 struct ib_mr_status *mr_status);
1228struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,1240struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
drivers/infiniband/hw/mlx5/mr.c+59−80
@@ -50,7 +50,6 @@ enum { · 49 unmodified lines
50static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);50static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
51static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);51static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
52static int mr_cache_max_order(struct mlx5_ib_dev *dev);52static int mr_cache_max_order(struct mlx5_ib_dev *dev);
53static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
54 53
55static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)54static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
56{55{
@@ -59,13 +58,9 @@ static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) · 2 unmodified lines
59 58
60static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)59static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
61{60{
62 int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);61 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));
63 62
64 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))63 return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
65 /* Wait until all page fault handlers using the mr complete. */
66 synchronize_srcu(&dev->mr_srcu);
67
68 return err;
69}64}
70 65
71static int order2idx(struct mlx5_ib_dev *dev, int order)66static int order2idx(struct mlx5_ib_dev *dev, int order)
@@ -94,8 +89,6 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context) · 22 unmodified lines
94 struct mlx5_cache_ent *ent = &cache->ent[c];89 struct mlx5_cache_ent *ent = &cache->ent[c];
95 u8 key;90 u8 key;
96 unsigned long flags;91 unsigned long flags;
97 struct xarray *mkeys = &dev->mdev->priv.mkey_table;
98 int err;
99 92
100 spin_lock_irqsave(&ent->lock, flags);93 spin_lock_irqsave(&ent->lock, flags);
101 ent->pending--;94 ent->pending--;
@@ -122,13 +115,6 @@ static void reg_mr_callback(int status, struct mlx5_async_work *context) · 20 unmodified lines
122 ent->size++;115 ent->size++;
123 spin_unlock_irqrestore(&ent->lock, flags);116 spin_unlock_irqrestore(&ent->lock, flags);
124 117
125 xa_lock_irqsave(mkeys, flags);
126 err = xa_err(__xa_store(mkeys, mlx5_base_mkey(mr->mmkey.key),
127 &mr->mmkey, GFP_ATOMIC));
128 xa_unlock_irqrestore(mkeys, flags);
129 if (err)
130 pr_err("Error inserting to mkey tree. 0x%x\n", -err);
131
132 if (!completion_done(&ent->compl))118 if (!completion_done(&ent->compl))
133 complete(&ent->compl);119 complete(&ent->compl);
134}120}
@@ -218,9 +204,6 @@ static void remove_keys(struct mlx5_ib_dev *dev, int c, int num) · 83 unmodified lines
218 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);204 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
219 }205 }
220 206
221 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
222 synchronize_srcu(&dev->mr_srcu);
223
224 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {207 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
225 list_del(&mr->list);208 list_del(&mr->list);
226 kfree(mr);209 kfree(mr);
@@ -511,7 +494,7 @@ void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) · 284 unmodified lines
511 c = order2idx(dev, mr->order);494 c = order2idx(dev, mr->order);
512 WARN_ON(c < 0 || c >= MAX_MR_CACHE_ENTRIES);495 WARN_ON(c < 0 || c >= MAX_MR_CACHE_ENTRIES);
513 496
514 if (unreg_umr(dev, mr)) {497 if (mlx5_mr_cache_invalidate(mr)) {
515 mr->allocated_from_cache = false;498 mr->allocated_from_cache = false;
516 destroy_mkey(dev, mr);499 destroy_mkey(dev, mr);
517 ent = &cache->ent[c];500 ent = &cache->ent[c];
@@ -555,10 +538,6 @@ static void clean_keys(struct mlx5_ib_dev *dev, int c) · 37 unmodified lines
555 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);538 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
556 }539 }
557 540
558#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
559 synchronize_srcu(&dev->mr_srcu);
560#endif
561
562 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {541 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) {
563 list_del(&mr->list);542 list_del(&mr->list);
564 kfree(mr);543 kfree(mr);
@@ -1335,10 +1314,15 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, · 770 unmodified lines
1335 1314
1336 if (is_odp_mr(mr)) {1315 if (is_odp_mr(mr)) {
1337 to_ib_umem_odp(mr->umem)->private = mr;1316 to_ib_umem_odp(mr->umem)->private = mr;
1338 atomic_set(&mr->num_pending_prefetch, 0);1317 atomic_set(&mr->num_deferred_work, 0);
1318 err = xa_err(xa_store(&dev->odp_mkeys,
1319 mlx5_base_mkey(mr->mmkey.key), &mr->mmkey,
1320 GFP_KERNEL));
1321 if (err) {
1322 dereg_mr(dev, mr);
1323 return ERR_PTR(err);
1324 }
1339 }1325 }
1340 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1341 smp_store_release(&mr->live, 1);
1342 1326
1343 return &mr->ibmr;1327 return &mr->ibmr;
1344error:1328error:
@@ -1346,22 +1330,29 @@ error: · 1 unmodified line
1346 return ERR_PTR(err);1330 return ERR_PTR(err);
1347}1331}
1348 1332
1349static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)1333/**
1334 * mlx5_mr_cache_invalidate - Fence all DMA on the MR
1335 * @mr: The MR to fence
1336 *
1337 * Upon return the NIC will not be doing any DMA to the pages under the MR,
1338 * and any DMA inprogress will be completed. Failure of this function
1339 * indicates the HW has failed catastrophically.
1340 */
1341int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr)
1350{1342{
1351 struct mlx5_core_dev *mdev = dev->mdev;
1352 struct mlx5_umr_wr umrwr = {};1343 struct mlx5_umr_wr umrwr = {};
1353 1344
1354 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)1345 if (mr->dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
1355 return 0;1346 return 0;
1356 1347
1357 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |1348 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
1358 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;1349 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1359 umrwr.wr.opcode = MLX5_IB_WR_UMR;1350 umrwr.wr.opcode = MLX5_IB_WR_UMR;
1360 umrwr.pd = dev->umrc.pd;1351 umrwr.pd = mr->dev->umrc.pd;
1361 umrwr.mkey = mr->mmkey.key;1352 umrwr.mkey = mr->mmkey.key;
1362 umrwr.ignore_free_state = 1;1353 umrwr.ignore_free_state = 1;
1363 1354
1364 return mlx5_ib_post_send_wait(dev, &umrwr);1355 return mlx5_ib_post_send_wait(mr->dev, &umrwr);
1365}1356}
1366 1357
1367static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,1358static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr,
@@ -1445,7 +1436,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, · 77 unmodified lines
1445 * UMR can't be used - MKey needs to be replaced.1436 * UMR can't be used - MKey needs to be replaced.
1446 */1437 */
1447 if (mr->allocated_from_cache)1438 if (mr->allocated_from_cache)
1448 err = unreg_umr(dev, mr);1439 err = mlx5_mr_cache_invalidate(mr);
1449 else1440 else
1450 err = destroy_mkey(dev, mr);1441 err = destroy_mkey(dev, mr);
1451 if (err)1442 if (err)
@@ -1558,6 +1549,7 @@ static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) · 106 unmodified lines
1558 mr->sig->psv_wire.psv_idx))1549 mr->sig->psv_wire.psv_idx))
1559 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",1550 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n",
1560 mr->sig->psv_wire.psv_idx);1551 mr->sig->psv_wire.psv_idx);
1552 xa_erase(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key));
1561 kfree(mr->sig);1553 kfree(mr->sig);
1562 mr->sig = NULL;1554 mr->sig = NULL;
1563 }1555 }
@@ -1573,54 +1565,20 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) · 9 unmodified lines
1573 int npages = mr->npages;1565 int npages = mr->npages;
1574 struct ib_umem *umem = mr->umem;1566 struct ib_umem *umem = mr->umem;
1575 1567
1576 if (is_odp_mr(mr)) {1568 /* Stop all DMA */
1577 struct ib_umem_odp *umem_odp = to_ib_umem_odp(umem);1569 if (is_odp_mr(mr))
1578 1570 mlx5_ib_fence_odp_mr(mr);
1579 /* Prevent new page faults and1571 else
1580 * prefetch requests from succeeding1572 clean_mr(dev, mr);
1581 */
1582 WRITE_ONCE(mr->live, 0);
1583
1584 /* Wait for all running page-fault handlers to finish. */
1585 synchronize_srcu(&dev->mr_srcu);
1586
1587 /* dequeue pending prefetch requests for the mr */
1588 if (atomic_read(&mr->num_pending_prefetch))
1589 flush_workqueue(system_unbound_wq);
1590 WARN_ON(atomic_read(&mr->num_pending_prefetch));
1591
1592 /* Destroy all page mappings */
1593 if (!umem_odp->is_implicit_odp)
1594 mlx5_ib_invalidate_range(umem_odp,
1595 ib_umem_start(umem_odp),
1596 ib_umem_end(umem_odp));
1597 else
1598 mlx5_ib_free_implicit_mr(mr);
1599 /*
1600 * We kill the umem before the MR for ODP,
1601 * so that there will not be any invalidations in
1602 * flight, looking at the *mr struct.
1603 */
1604 ib_umem_odp_release(umem_odp);
1605 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1606
1607 /* Avoid double-freeing the umem. */
1608 umem = NULL;
1609 }
1610 1573
1611 clean_mr(dev, mr);1574 if (mr->allocated_from_cache)
1575 mlx5_mr_cache_free(dev, mr);
1576 else
1577 kfree(mr);
1612 1578
1613 /*
1614 * We should unregister the DMA address from the HCA before
1615 * remove the DMA mapping.
1616 */
1617 mlx5_mr_cache_free(dev, mr);
1618 ib_umem_release(umem);1579 ib_umem_release(umem);
1619 if (umem)1580 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1620 atomic_sub(npages, &dev->mdev->priv.reg_pages);
1621 1581
1622 if (!mr->allocated_from_cache)
1623 kfree(mr);
1624}1582}
1625 1583
1626int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)1584int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
@@ -1632,6 +1590,11 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) · 5 unmodified lines
1632 dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr);1590 dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr);
1633 }1591 }
1634 1592
1593 if (is_odp_mr(mmr) && to_ib_umem_odp(mmr->umem)->is_implicit_odp) {
1594 mlx5_ib_free_implicit_mr(mmr);
1595 return 0;
1596 }
1597
1635 dereg_mr(to_mdev(ibmr->device), mmr);1598 dereg_mr(to_mdev(ibmr->device), mmr);
1636 1599
1637 return 0;1600 return 0;
@@ -1795,8 +1758,15 @@ static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, · 157 unmodified lines
1795 if (err)1758 if (err)
1796 goto err_free_mtt_mr;1759 goto err_free_mtt_mr;
1797 1760
1761 err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key),
1762 mr->sig, GFP_KERNEL));
1763 if (err)
1764 goto err_free_descs;
1798 return 0;1765 return 0;
1799 1766
1767err_free_descs:
1768 destroy_mkey(dev, mr);
1769 mlx5_free_priv_descs(mr);
1800err_free_mtt_mr:1770err_free_mtt_mr:
1801 dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);1771 dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr);
1802 mr->mtt_mr = NULL;1772 mr->mtt_mr = NULL;
@@ -1949,9 +1919,19 @@ struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, · 146 unmodified lines
1949 }1919 }
1950 }1920 }
1951 1921
1922 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
1923 err = xa_err(xa_store(&dev->odp_mkeys,
1924 mlx5_base_mkey(mw->mmkey.key), &mw->mmkey,
1925 GFP_KERNEL));
1926 if (err)
1927 goto free_mkey;
1928 }
1929
1952 kfree(in);1930 kfree(in);
1953 return &mw->ibmw;1931 return &mw->ibmw;
1954 1932
1933free_mkey:
1934 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
1955free:1935free:
1956 kfree(mw);1936 kfree(mw);
1957 kfree(in);1937 kfree(in);
@@ -1965,13 +1945,12 @@ int mlx5_ib_dealloc_mw(struct ib_mw *mw) · 7 unmodified lines
1965 int err;1945 int err;
1966 1946
1967 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {1947 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
1968 xa_erase(&dev->mdev->priv.mkey_table,1948 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key));
1969 mlx5_base_mkey(mmw->mmkey.key));
1970 /*1949 /*
1971 * pagefault_single_data_segment() may be accessing mmw under1950 * pagefault_single_data_segment() may be accessing mmw under
1972 * SRCU if the user bound an ODP MR to this MW.1951 * SRCU if the user bound an ODP MR to this MW.
1973 */1952 */
1974 synchronize_srcu(&dev->mr_srcu);1953 synchronize_srcu(&dev->odp_srcu);
1975 }1954 }
1976 1955
1977 err = mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey);1956 err = mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey);
drivers/infiniband/hw/mlx5/odp.c+499−475
@@ -93,158 +93,152 @@ struct mlx5_pagefault { · 92 unmodified lines
93 93
94static u64 mlx5_imr_ksm_entries;94static u64 mlx5_imr_ksm_entries;
95 95
96static int check_parent(struct ib_umem_odp *odp,96void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries,
97 struct mlx5_ib_mr *parent)97 struct mlx5_ib_mr *imr, int flags)
98{98{
99 struct mlx5_ib_mr *mr = odp->private;99 struct mlx5_klm *end = pklm + nentries;
100
101 return mr && mr->parent == parent && !odp->dying;
102}
103
104static struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr)
105{
106 if (WARN_ON(!mr || !is_odp_mr(mr)))
107 return NULL;
108
109 return to_ib_umem_odp(mr->umem)->per_mm;
110}
111
112static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp)
113{
114 struct mlx5_ib_mr *mr = odp->private, *parent = mr->parent;
115 struct ib_ucontext_per_mm *per_mm = odp->per_mm;
116 struct rb_node *rb;
117
118 down_read(&per_mm->umem_rwsem);
119 while (1) {
120 rb = rb_next(&odp->interval_tree.rb);
121 if (!rb)
122 goto not_found;
123 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb);
124 if (check_parent(odp, parent))
125 goto end;
126 }
127not_found:
128 odp = NULL;
129end:
130 up_read(&per_mm->umem_rwsem);
131 return odp;
132}
133
134static struct ib_umem_odp *odp_lookup(u64 start, u64 length,
135 struct mlx5_ib_mr *parent)
136{
137 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent);
138 struct ib_umem_odp *odp;
139 struct rb_node *rb;
140
141 down_read(&per_mm->umem_rwsem);
142 odp = rbt_ib_umem_lookup(&per_mm->umem_tree, start, length);
143 if (!odp)
144 goto end;
145
146 while (1) {
147 if (check_parent(odp, parent))
148 goto end;
149 rb = rb_next(&odp->interval_tree.rb);
150 if (!rb)
151 goto not_found;
152 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb);
153 if (ib_umem_start(odp) > start + length)
154 goto not_found;
155 }
156not_found:
157 odp = NULL;
158end:
159 up_read(&per_mm->umem_rwsem);
160 return odp;
161}
162
163void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
164 size_t nentries, struct mlx5_ib_mr *mr, int flags)
165{
166 struct ib_pd *pd = mr->ibmr.pd;
167 struct mlx5_ib_dev *dev = to_mdev(pd->device);
168 struct ib_umem_odp *odp;
169 unsigned long va;
170 int i;
171 100
172 if (flags & MLX5_IB_UPD_XLT_ZAP) {101 if (flags & MLX5_IB_UPD_XLT_ZAP) {
173 for (i = 0; i < nentries; i++, pklm++) {102 for (; pklm != end; pklm++, idx++) {
174 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);103 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
175 pklm->key = cpu_to_be32(dev->null_mkey);104 pklm->key = cpu_to_be32(imr->dev->null_mkey);
176 pklm->va = 0;105 pklm->va = 0;
177 }106 }
178 return;107 return;
179 }108 }
180 109
181 /*110 /*
182 * The locking here is pretty subtle. Ideally the implicit children111 * The locking here is pretty subtle. Ideally the implicit_children
183 * list would be protected by the umem_mutex, however that is not112 * xarray would be protected by the umem_mutex, however that is not
184 * possible. Instead this uses a weaker update-then-lock pattern:113 * possible. Instead this uses a weaker update-then-lock pattern:
185 *114 *
186 * srcu_read_lock()115 * srcu_read_lock()
187 * <change children list>116 * xa_store()
188 * mutex_lock(umem_mutex)117 * mutex_lock(umem_mutex)
189 * mlx5_ib_update_xlt()118 * mlx5_ib_update_xlt()
190 * mutex_unlock(umem_mutex)119 * mutex_unlock(umem_mutex)
191 * destroy lkey120 * destroy lkey
192 *121 *
193 * ie any change the children list must be followed by the locked122 * ie any change the xarray must be followed by the locked update_xlt
194 * update_xlt before destroying.123 * before destroying.
195 *124 *
196 * The umem_mutex provides the acquire/release semantic needed to make125 * The umem_mutex provides the acquire/release semantic needed to make
197 * the children list visible to a racing thread. While SRCU is not126 * the xa_store() visible to a racing thread. While SRCU is not
198 * technically required, using it gives consistent use of the SRCU127 * technically required, using it gives consistent use of the SRCU
199 * locking around the children list.128 * locking around the xarray.
200 */129 */
201 lockdep_assert_held(&to_ib_umem_odp(mr->umem)->umem_mutex);130 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex);
202 lockdep_assert_held(&mr->dev->mr_srcu);131 lockdep_assert_held(&imr->dev->odp_srcu);
203 132
204 odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE,133 for (; pklm != end; pklm++, idx++) {
205 nentries * MLX5_IMR_MTT_SIZE, mr);134 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx);
206 135
207 for (i = 0; i < nentries; i++, pklm++) {
208 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);136 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
209 va = (offset + i) * MLX5_IMR_MTT_SIZE;137 if (mtt) {
210 if (odp && ib_umem_start(odp) == va) {
211 struct mlx5_ib_mr *mtt = odp->private;
212
213 pklm->key = cpu_to_be32(mtt->ibmr.lkey);138 pklm->key = cpu_to_be32(mtt->ibmr.lkey);
214 odp = odp_next(odp);139 pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE);
215 } else {140 } else {
216 pklm->key = cpu_to_be32(dev->null_mkey);141 pklm->key = cpu_to_be32(imr->dev->null_mkey);
142 pklm->va = 0;
217 }143 }
218 mlx5_ib_dbg(dev, "[%d] va %lx key %x\n",
219 i, va, be32_to_cpu(pklm->key));
220 }144 }
221}145}
222 146
223static void mr_leaf_free_action(struct work_struct *work)147static void dma_fence_odp_mr(struct mlx5_ib_mr *mr)
148{
149 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
150
151 /* Ensure mlx5_ib_invalidate_range() will not touch the MR any more */
152 mutex_lock(&odp->umem_mutex);
153 if (odp->npages) {
154 mlx5_mr_cache_invalidate(mr);
155 ib_umem_odp_unmap_dma_pages(odp, ib_umem_start(odp),
156 ib_umem_end(odp));
157 WARN_ON(odp->npages);
158 }
159 odp->private = NULL;
160 mutex_unlock(&odp->umem_mutex);
161
162 if (!mr->allocated_from_cache) {
163 mlx5_core_destroy_mkey(mr->dev->mdev, &mr->mmkey);
164 WARN_ON(mr->descs);
165 }
166}
167
168/*
169 * This must be called after the mr has been removed from implicit_children
170 * and the SRCU synchronized. NOTE: The MR does not necessarily have to be
171 * empty here, parallel page faults could have raced with the free process and
172 * added pages to it.
173 */
174static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt)
224{175{
225 struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work);176 struct mlx5_ib_mr *imr = mr->parent;
226 int idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT;
227 struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent;
228 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);177 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);
178 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
179 unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT;
229 int srcu_key;180 int srcu_key;
230 181
231 mr->parent = NULL;182 /* implicit_child_mr's are not allowed to have deferred work */
232 synchronize_srcu(&mr->dev->mr_srcu);183 WARN_ON(atomic_read(&mr->num_deferred_work));
233 184
234 if (smp_load_acquire(&imr->live)) {185 if (need_imr_xlt) {
235 srcu_key = srcu_read_lock(&mr->dev->mr_srcu);186 srcu_key = srcu_read_lock(&mr->dev->odp_srcu);
236 mutex_lock(&odp_imr->umem_mutex);187 mutex_lock(&odp_imr->umem_mutex);
237 mlx5_ib_update_xlt(imr, idx, 1, 0,188 mlx5_ib_update_xlt(mr->parent, idx, 1, 0,
238 MLX5_IB_UPD_XLT_INDIRECT |189 MLX5_IB_UPD_XLT_INDIRECT |
239 MLX5_IB_UPD_XLT_ATOMIC);190 MLX5_IB_UPD_XLT_ATOMIC);
240 mutex_unlock(&odp_imr->umem_mutex);191 mutex_unlock(&odp_imr->umem_mutex);
241 srcu_read_unlock(&mr->dev->mr_srcu, srcu_key);192 srcu_read_unlock(&mr->dev->odp_srcu, srcu_key);
242 }193 }
243 ib_umem_odp_release(odp);194
195 dma_fence_odp_mr(mr);
196
197 mr->parent = NULL;
244 mlx5_mr_cache_free(mr->dev, mr);198 mlx5_mr_cache_free(mr->dev, mr);
199 ib_umem_odp_release(odp);
200 atomic_dec(&imr->num_deferred_work);
201}
202
203static void free_implicit_child_mr_work(struct work_struct *work)
204{
205 struct mlx5_ib_mr *mr =
206 container_of(work, struct mlx5_ib_mr, odp_destroy.work);
207
208 free_implicit_child_mr(mr, true);
209}
210
211static void free_implicit_child_mr_rcu(struct rcu_head *head)
212{
213 struct mlx5_ib_mr *mr =
214 container_of(head, struct mlx5_ib_mr, odp_destroy.rcu);
245 215
246 if (atomic_dec_and_test(&imr->num_leaf_free))216 /* Freeing a MR is a sleeping operation, so bounce to a work queue */
247 wake_up(&imr->q_leaf_free);217 INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work);
218 queue_work(system_unbound_wq, &mr->odp_destroy.work);
219}
220
221static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr)
222{
223 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
224 unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT;
225 struct mlx5_ib_mr *imr = mr->parent;
226
227 xa_lock(&imr->implicit_children);
228 /*
229 * This can race with mlx5_ib_free_implicit_mr(), the first one to
230 * reach the xa lock wins the race and destroys the MR.
231 */
232 if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_ATOMIC) !=
233 mr)
234 goto out_unlock;
235
236 atomic_inc(&imr->num_deferred_work);
237 call_srcu(&mr->dev->odp_srcu, &mr->odp_destroy.rcu,
238 free_implicit_child_mr_rcu);
239
240out_unlock:
241 xa_unlock(&imr->implicit_children);
248}242}
249 243
250void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,244void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start,
@@ -258,16 +252,15 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, · 7 unmodified lines
258 int in_block = 0;252 int in_block = 0;
259 u64 addr;253 u64 addr;
260 254
261 if (!umem_odp) {255 mutex_lock(&umem_odp->umem_mutex);
262 pr_err("invalidation called on NULL umem or non-ODP umem\n");256 /*
263 return;257 * If npages is zero then umem_odp->private may not be setup yet. This
264 }258 * does not complete until after the first page is mapped for DMA.
265 259 */
260 if (!umem_odp->npages)
261 goto out;
266 mr = umem_odp->private;262 mr = umem_odp->private;
267 263
268 if (!mr || !mr->ibmr.pd)
269 return;
270
271 start = max_t(u64, ib_umem_start(umem_odp), start);264 start = max_t(u64, ib_umem_start(umem_odp), start);
272 end = min_t(u64, ib_umem_end(umem_odp), end);265 end = min_t(u64, ib_umem_end(umem_odp), end);
273 266
@@ -277,7 +270,6 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, · 3 unmodified lines
277 * overwrite the same MTTs. Concurent invalidations might race us,270 * overwrite the same MTTs. Concurent invalidations might race us,
278 * but they will write 0s as well, so no difference in the end result.271 * but they will write 0s as well, so no difference in the end result.
279 */272 */
280 mutex_lock(&umem_odp->umem_mutex);
281 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {273 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {
282 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;274 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
283 /*275 /*
@@ -323,14 +315,9 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, · 39 unmodified lines
323 315
324 ib_umem_odp_unmap_dma_pages(umem_odp, start, end);316 ib_umem_odp_unmap_dma_pages(umem_odp, start, end);
325 317
326 318 if (unlikely(!umem_odp->npages && mr->parent))
327 if (unlikely(!umem_odp->npages && mr->parent &&319 destroy_unused_implicit_child_mr(mr);
328 !umem_odp->dying)) {320out:
329 WRITE_ONCE(mr->live, 0);
330 umem_odp->dying = 1;
331 atomic_inc(&mr->parent->num_leaf_free);
332 schedule_work(&umem_odp->work);
333 }
334 mutex_unlock(&umem_odp->umem_mutex);321 mutex_unlock(&umem_odp->umem_mutex);
335}322}
336 323
@@ -422,239 +409,213 @@ static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, · 85 unmodified lines
422 wq_num, err);409 wq_num, err);
423}410}
424 411
425static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd,412static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
426 struct ib_umem_odp *umem_odp,413 unsigned long idx)
427 bool ksm, int access_flags)
428{414{
429 struct mlx5_ib_dev *dev = to_mdev(pd->device);415 struct ib_umem_odp *odp;
430 struct mlx5_ib_mr *mr;416 struct mlx5_ib_mr *mr;
417 struct mlx5_ib_mr *ret;
431 int err;418 int err;
432 419
433 mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY :420 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem),
434 MLX5_IMR_MTT_CACHE_ENTRY);421 idx * MLX5_IMR_MTT_SIZE,
422 MLX5_IMR_MTT_SIZE);
423 if (IS_ERR(odp))
424 return ERR_CAST(odp);
435 425
426 ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY);
436 if (IS_ERR(mr))427 if (IS_ERR(mr))
437 return mr;428 goto out_umem;
438
439 mr->ibmr.pd = pd;
440
441 mr->dev = dev;
442 mr->access_flags = access_flags;
443 mr->mmkey.iova = 0;
444 mr->umem = &umem_odp->umem;
445
446 if (ksm) {
447 err = mlx5_ib_update_xlt(mr, 0,
448 mlx5_imr_ksm_entries,
449 MLX5_KSM_PAGE_SHIFT,
450 MLX5_IB_UPD_XLT_INDIRECT |
451 MLX5_IB_UPD_XLT_ZAP |
452 MLX5_IB_UPD_XLT_ENABLE);
453
454 } else {
455 err = mlx5_ib_update_xlt(mr, 0,
456 MLX5_IMR_MTT_ENTRIES,
457 PAGE_SHIFT,
458 MLX5_IB_UPD_XLT_ZAP |
459 MLX5_IB_UPD_XLT_ENABLE |
460 MLX5_IB_UPD_XLT_ATOMIC);
461 }
462
463 if (err)
464 goto fail;
465 429
430 mr->ibmr.pd = imr->ibmr.pd;
431 mr->access_flags = imr->access_flags;
432 mr->umem = &odp->umem;
466 mr->ibmr.lkey = mr->mmkey.key;433 mr->ibmr.lkey = mr->mmkey.key;
467 mr->ibmr.rkey = mr->mmkey.key;434 mr->ibmr.rkey = mr->mmkey.key;
468 435 mr->mmkey.iova = idx * MLX5_IMR_MTT_SIZE;
469 mlx5_ib_dbg(dev, "key %x dev %p mr %p\n",436 mr->parent = imr;
470 mr->mmkey.key, dev->mdev, mr);437 odp->private = mr;
471 438
472 return mr;439 err = mlx5_ib_update_xlt(mr, 0,
473 440 MLX5_IMR_MTT_ENTRIES,
474fail:441 PAGE_SHIFT,
475 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err);442 MLX5_IB_UPD_XLT_ZAP |
476 mlx5_mr_cache_free(dev, mr);443 MLX5_IB_UPD_XLT_ENABLE);
477 444 if (err) {
478 return ERR_PTR(err);445 ret = ERR_PTR(err);
479}446 goto out_mr;
480
481static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr,
482 u64 io_virt, size_t bcnt)
483{
484 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.pd->device);
485 struct ib_umem_odp *odp, *result = NULL;
486 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem);
487 u64 addr = io_virt & MLX5_IMR_MTT_MASK;
488 int nentries = 0, start_idx = 0, ret;
489 struct mlx5_ib_mr *mtt;
490
491 mutex_lock(&odp_mr->umem_mutex);
492 odp = odp_lookup(addr, 1, mr);
493
494 mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n",
495 io_virt, bcnt, addr, odp);
496
497next_mr:
498 if (likely(odp)) {
499 if (nentries)
500 nentries++;
501 } else {
502 odp = ib_umem_odp_alloc_child(odp_mr, addr, MLX5_IMR_MTT_SIZE);
503 if (IS_ERR(odp)) {
504 mutex_unlock(&odp_mr->umem_mutex);
505 return ERR_CAST(odp);
506 }
507
508 mtt = implicit_mr_alloc(mr->ibmr.pd, odp, 0,
509 mr->access_flags);
510 if (IS_ERR(mtt)) {
511 mutex_unlock(&odp_mr->umem_mutex);
512 ib_umem_odp_release(odp);
513 return ERR_CAST(mtt);
514 }
515
516 odp->private = mtt;
517 mtt->umem = &odp->umem;
518 mtt->mmkey.iova = addr;
519 mtt->parent = mr;
520 INIT_WORK(&odp->work, mr_leaf_free_action);
521
522 smp_store_release(&mtt->live, 1);
523
524 if (!nentries)
525 start_idx = addr >> MLX5_IMR_MTT_SHIFT;
526 nentries++;
527 }
528
529 /* Return first odp if region not covered by single one */
530 if (likely(!result))
531 result = odp;
532
533 addr += MLX5_IMR_MTT_SIZE;
534 if (unlikely(addr < io_virt + bcnt)) {
535 odp = odp_next(odp);
536 if (odp && ib_umem_start(odp) != addr)
537 odp = NULL;
538 goto next_mr;
539 }447 }
540 448
541 if (unlikely(nentries)) {449 /*
542 ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0,450 * Once the store to either xarray completes any error unwind has to
543 MLX5_IB_UPD_XLT_INDIRECT |451 * use synchronize_srcu(). Avoid this with xa_reserve()
544 MLX5_IB_UPD_XLT_ATOMIC);452 */
545 if (ret) {453 ret = xa_cmpxchg(&imr->implicit_children, idx, NULL, mr,
546 mlx5_ib_err(dev, "Failed to update PAS\n");454 GFP_KERNEL);
547 result = ERR_PTR(ret);455 if (unlikely(ret)) {
456 if (xa_is_err(ret)) {
457 ret = ERR_PTR(xa_err(ret));
458 goto out_mr;
548 }459 }
460 /*
461 * Another thread beat us to creating the child mr, use
462 * theirs.
463 */
464 goto out_mr;
549 }465 }
550 466
551 mutex_unlock(&odp_mr->umem_mutex);467 mlx5_ib_dbg(imr->dev, "key %x mr %p\n", mr->mmkey.key, mr);
552 return result;468 return mr;
469
470out_mr:
471 mlx5_mr_cache_free(imr->dev, mr);
472out_umem:
473 ib_umem_odp_release(odp);
474 return ret;
553}475}
554 476
555struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,477struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
556 struct ib_udata *udata,478 struct ib_udata *udata,
557 int access_flags)479 int access_flags)
558{480{
559 struct mlx5_ib_mr *imr;481 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
560 struct ib_umem_odp *umem_odp;482 struct ib_umem_odp *umem_odp;
483 struct mlx5_ib_mr *imr;
484 int err;
561 485
562 umem_odp = ib_umem_odp_alloc_implicit(udata, access_flags);486 umem_odp = ib_umem_odp_alloc_implicit(udata, access_flags);
563 if (IS_ERR(umem_odp))487 if (IS_ERR(umem_odp))
564 return ERR_CAST(umem_odp);488 return ERR_CAST(umem_odp);
565 489
566 imr = implicit_mr_alloc(&pd->ibpd, umem_odp, 1, access_flags);490 imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY);
567 if (IS_ERR(imr)) {491 if (IS_ERR(imr)) {
568 ib_umem_odp_release(umem_odp);492 err = PTR_ERR(imr);
569 return ERR_CAST(imr);493 goto out_umem;
570 }494 }
571 495
496 imr->ibmr.pd = &pd->ibpd;
497 imr->access_flags = access_flags;
498 imr->mmkey.iova = 0;
499 imr->umem = &umem_odp->umem;
500 imr->ibmr.lkey = imr->mmkey.key;
501 imr->ibmr.rkey = imr->mmkey.key;
572 imr->umem = &umem_odp->umem;502 imr->umem = &umem_odp->umem;
573 init_waitqueue_head(&imr->q_leaf_free);
574 atomic_set(&imr->num_leaf_free, 0);
575 atomic_set(&imr->num_pending_prefetch, 0);
576 smp_store_release(&imr->live, 1);
577
578 imr->is_odp_implicit = true;503 imr->is_odp_implicit = true;
504 atomic_set(&imr->num_deferred_work, 0);
505 xa_init(&imr->implicit_children);
506
507 err = mlx5_ib_update_xlt(imr, 0,
508 mlx5_imr_ksm_entries,
509 MLX5_KSM_PAGE_SHIFT,
510 MLX5_IB_UPD_XLT_INDIRECT |
511 MLX5_IB_UPD_XLT_ZAP |
512 MLX5_IB_UPD_XLT_ENABLE);
513 if (err)
514 goto out_mr;
515
516 err = xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key),
517 &imr->mmkey, GFP_KERNEL));
518 if (err)
519 goto out_mr;
579 520
521 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr);
580 return imr;522 return imr;
523out_mr:
524 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err);
525 mlx5_mr_cache_free(dev, imr);
526out_umem:
527 ib_umem_odp_release(umem_odp);
528 return ERR_PTR(err);
581}529}
582 530
583void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)531void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)
584{532{
585 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr);533 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);
586 struct rb_node *node;534 struct mlx5_ib_dev *dev = imr->dev;
535 struct list_head destroy_list;
536 struct mlx5_ib_mr *mtt;
537 struct mlx5_ib_mr *tmp;
538 unsigned long idx;
587 539
588 down_read(&per_mm->umem_rwsem);540 INIT_LIST_HEAD(&destroy_list);
589 for (node = rb_first_cached(&per_mm->umem_tree); node;
590 node = rb_next(node)) {
591 struct ib_umem_odp *umem_odp =
592 rb_entry(node, struct ib_umem_odp, interval_tree.rb);
593 struct mlx5_ib_mr *mr = umem_odp->private;
594 541
595 if (mr->parent != imr)542 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key));
596 continue;543 /*
544 * This stops the SRCU protected page fault path from touching either
545 * the imr or any children. The page fault path can only reach the
546 * children xarray via the imr.
547 */
548 synchronize_srcu(&dev->odp_srcu);
597 549
598 mutex_lock(&umem_odp->umem_mutex);550 xa_lock(&imr->implicit_children);
599 ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp),551 xa_for_each (&imr->implicit_children, idx, mtt) {
600 ib_umem_end(umem_odp));552 __xa_erase(&imr->implicit_children, idx);
553 list_add(&mtt->odp_destroy.elm, &destroy_list);
554 }
555 xa_unlock(&imr->implicit_children);
601 556
602 if (umem_odp->dying) {557 /*
603 mutex_unlock(&umem_odp->umem_mutex);558 * num_deferred_work can only be incremented inside the odp_srcu, or
604 continue;559 * under xa_lock while the child is in the xarray. Thus at this point
605 }560 * it is only decreasing, and all work holding it is now on the wq.
561 */
562 if (atomic_read(&imr->num_deferred_work)) {
563 flush_workqueue(system_unbound_wq);
564 WARN_ON(atomic_read(&imr->num_deferred_work));
565 }
566
567 /*
568 * Fence the imr before we destroy the children. This allows us to
569 * skip updating the XLT of the imr during destroy of the child mkey
570 * the imr points to.
571 */
572 mlx5_mr_cache_invalidate(imr);
573
574 list_for_each_entry_safe (mtt, tmp, &destroy_list, odp_destroy.elm)
575 free_implicit_child_mr(mtt, false);
576
577 mlx5_mr_cache_free(dev, imr);
578 ib_umem_odp_release(odp_imr);
579}
580
581/**
582 * mlx5_ib_fence_odp_mr - Stop all access to the ODP MR
583 * @mr: to fence
584 *
585 * On return no parallel threads will be touching this MR and no DMA will be
586 * active.
587 */
588void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr)
589{
590 /* Prevent new page faults and prefetch requests from succeeding */
591 xa_erase(&mr->dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key));
592
593 /* Wait for all running page-fault handlers to finish. */
594 synchronize_srcu(&mr->dev->odp_srcu);
606 595
607 umem_odp->dying = 1;596 if (atomic_read(&mr->num_deferred_work)) {
608 atomic_inc(&imr->num_leaf_free);597 flush_workqueue(system_unbound_wq);
609 schedule_work(&umem_odp->work);598 WARN_ON(atomic_read(&mr->num_deferred_work));
610 mutex_unlock(&umem_odp->umem_mutex);
611 }599 }
612 up_read(&per_mm->umem_rwsem);
613 600
614 wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free));601 dma_fence_odp_mr(mr);
615}602}
616 603
617#define MLX5_PF_FLAGS_PREFETCH BIT(0)
618#define MLX5_PF_FLAGS_DOWNGRADE BIT(1)604#define MLX5_PF_FLAGS_DOWNGRADE BIT(1)
619static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,605static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
620 u64 io_virt, size_t bcnt, u32 *bytes_mapped,606 u64 user_va, size_t bcnt, u32 *bytes_mapped,
621 u32 flags)607 u32 flags)
622{608{
623 int npages = 0, current_seq, page_shift, ret, np;609 int current_seq, page_shift, ret, np;
624 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem);
625 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE;610 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE;
626 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH;
627 u64 access_mask;611 u64 access_mask;
628 u64 start_idx, page_mask;612 u64 start_idx, page_mask;
629 struct ib_umem_odp *odp;
630 size_t size;
631
632 if (odp_mr->is_implicit_odp) {
633 odp = implicit_mr_get_data(mr, io_virt, bcnt);
634
635 if (IS_ERR(odp))
636 return PTR_ERR(odp);
637 mr = odp->private;
638 } else {
639 odp = odp_mr;
640 }
641
642next_mr:
643 size = min_t(size_t, bcnt, ib_umem_end(odp) - io_virt);
644 613
645 page_shift = odp->page_shift;614 page_shift = odp->page_shift;
646 page_mask = ~(BIT(page_shift) - 1);615 page_mask = ~(BIT(page_shift) - 1);
647 start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift;616 start_idx = (user_va - (mr->mmkey.iova & page_mask)) >> page_shift;
648 access_mask = ODP_READ_ALLOWED_BIT;617 access_mask = ODP_READ_ALLOWED_BIT;
649 618
650 if (prefetch && !downgrade && !odp->umem.writable) {
651 /* prefetch with write-access must
652 * be supported by the MR
653 */
654 ret = -EINVAL;
655 goto out;
656 }
657
658 if (odp->umem.writable && !downgrade)619 if (odp->umem.writable && !downgrade)
659 access_mask |= ODP_WRITE_ALLOWED_BIT;620 access_mask |= ODP_WRITE_ALLOWED_BIT;
660 621
@@ -665,13 +626,10 @@ next_mr: · 4 unmodified lines
665 */626 */
666 smp_rmb();627 smp_rmb();
667 628
668 ret = ib_umem_odp_map_dma_pages(odp, io_virt, size, access_mask,629 np = ib_umem_odp_map_dma_pages(odp, user_va, bcnt, access_mask,
669 current_seq);630 current_seq);
670 631 if (np < 0)
671 if (ret < 0)632 return np;
672 goto out;
673
674 np = ret;
675 633
676 mutex_lock(&odp->umem_mutex);634 mutex_lock(&odp->umem_mutex);
677 if (!ib_umem_mmu_notifier_retry(odp, current_seq)) {635 if (!ib_umem_mmu_notifier_retry(odp, current_seq)) {
@@ -689,35 +647,19 @@ next_mr: · 11 unmodified lines
689 647
690 if (ret < 0) {648 if (ret < 0) {
691 if (ret != -EAGAIN)649 if (ret != -EAGAIN)
692 mlx5_ib_err(dev, "Failed to update mkey page tables\n");650 mlx5_ib_err(mr->dev,
651 "Failed to update mkey page tables\n");
693 goto out;652 goto out;
694 }653 }
695 654
696 if (bytes_mapped) {655 if (bytes_mapped) {
697 u32 new_mappings = (np << page_shift) -656 u32 new_mappings = (np << page_shift) -
698 (io_virt - round_down(io_virt, 1 << page_shift));657 (user_va - round_down(user_va, 1 << page_shift));
699 *bytes_mapped += min_t(u32, new_mappings, size);
700 }
701
702 npages += np << (page_shift - PAGE_SHIFT);
703 bcnt -= size;
704 658
705 if (unlikely(bcnt)) {659 *bytes_mapped += min_t(u32, new_mappings, bcnt);
706 struct ib_umem_odp *next;
707
708 io_virt += size;
709 next = odp_next(odp);
710 if (unlikely(!next || ib_umem_start(next) != io_virt)) {
711 mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n",
712 io_virt, next);
713 return -EAGAIN;
714 }
715 odp = next;
716 mr = odp->private;
717 goto next_mr;
718 }660 }
719 661
720 return npages;662 return np << (page_shift - PAGE_SHIFT);
721 663
722out:664out:
723 if (ret == -EAGAIN) {665 if (ret == -EAGAIN) {
@@ -726,7 +668,7 @@ out: · 2 unmodified lines
726 if (!wait_for_completion_timeout(&odp->notifier_completion,668 if (!wait_for_completion_timeout(&odp->notifier_completion,
727 timeout)) {669 timeout)) {
728 mlx5_ib_warn(670 mlx5_ib_warn(
729 dev,671 mr->dev,
730 "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n",672 "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n",
731 current_seq, odp->notifiers_seq,673 current_seq, odp->notifiers_seq,
732 odp->notifiers_count);674 odp->notifiers_count);
@@ -736,6 +678,109 @@ out: · 3 unmodified lines
736 return ret;678 return ret;
737}679}
738 680
681static int pagefault_implicit_mr(struct mlx5_ib_mr *imr,
682 struct ib_umem_odp *odp_imr, u64 user_va,
683 size_t bcnt, u32 *bytes_mapped, u32 flags)
684{
685 unsigned long end_idx = (user_va + bcnt - 1) >> MLX5_IMR_MTT_SHIFT;
686 unsigned long upd_start_idx = end_idx + 1;
687 unsigned long upd_len = 0;
688 unsigned long npages = 0;
689 int err;
690 int ret;
691
692 if (unlikely(user_va >= mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE ||
693 mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE - user_va < bcnt))
694 return -EFAULT;
695
696 /* Fault each child mr that intersects with our interval. */
697 while (bcnt) {
698 unsigned long idx = user_va >> MLX5_IMR_MTT_SHIFT;
699 struct ib_umem_odp *umem_odp;
700 struct mlx5_ib_mr *mtt;
701 u64 len;
702
703 mtt = xa_load(&imr->implicit_children, idx);
704 if (unlikely(!mtt)) {
705 mtt = implicit_get_child_mr(imr, idx);
706 if (IS_ERR(mtt)) {
707 ret = PTR_ERR(mtt);
708 goto out;
709 }
710 upd_start_idx = min(upd_start_idx, idx);
711 upd_len = idx - upd_start_idx + 1;
712 }
713
714 umem_odp = to_ib_umem_odp(mtt->umem);
715 len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) -
716 user_va;
717
718 ret = pagefault_real_mr(mtt, umem_odp, user_va, len,
719 bytes_mapped, flags);
720 if (ret < 0)
721 goto out;
722 user_va += len;
723 bcnt -= len;
724 npages += ret;
725 }
726
727 ret = npages;
728
729 /*
730 * Any time the implicit_children are changed we must perform an
731 * update of the xlt before exiting to ensure the HW and the
732 * implicit_children remains synchronized.
733 */
734out:
735 if (likely(!upd_len))
736 return ret;
737
738 /*
739 * Notice this is not strictly ordered right, the KSM is updated after
740 * the implicit_children is updated, so a parallel page fault could
741 * see a MR that is not yet visible in the KSM. This is similar to a
742 * parallel page fault seeing a MR that is being concurrently removed
743 * from the KSM. Both of these improbable situations are resolved
744 * safely by resuming the HW and then taking another page fault. The
745 * next pagefault handler will see the new information.
746 */
747 mutex_lock(&odp_imr->umem_mutex);
748 err = mlx5_ib_update_xlt(imr, upd_start_idx, upd_len, 0,
749 MLX5_IB_UPD_XLT_INDIRECT |
750 MLX5_IB_UPD_XLT_ATOMIC);
751 mutex_unlock(&odp_imr->umem_mutex);
752 if (err) {
753 mlx5_ib_err(imr->dev, "Failed to update PAS\n");
754 return err;
755 }
756 return ret;
757}
758
759/*
760 * Returns:
761 * -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are
762 * not accessible, or the MR is no longer valid.
763 * -EAGAIN/-ENOMEM: The operation should be retried
764 *
765 * -EINVAL/others: General internal malfunction
766 * >0: Number of pages mapped
767 */
768static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt,
769 u32 *bytes_mapped, u32 flags)
770{
771 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
772
773 if (!odp->is_implicit_odp) {
774 if (unlikely(io_virt < ib_umem_start(odp) ||
775 ib_umem_end(odp) - io_virt < bcnt))
776 return -EFAULT;
777 return pagefault_real_mr(mr, odp, io_virt, bcnt, bytes_mapped,
778 flags);
779 }
780 return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped,
781 flags);
782}
783
739struct pf_frame {784struct pf_frame {
740 struct pf_frame *next;785 struct pf_frame *next;
741 u32 key;786 u32 key;
@@ -783,10 +828,9 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, · 41 unmodified lines
783 struct ib_pd *pd, u32 key,828 struct ib_pd *pd, u32 key,
784 u64 io_virt, size_t bcnt,829 u64 io_virt, size_t bcnt,
785 u32 *bytes_committed,830 u32 *bytes_committed,
786 u32 *bytes_mapped, u32 flags)831 u32 *bytes_mapped)
787{832{
788 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0;833 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0;
789 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH;
790 struct pf_frame *head = NULL, *frame;834 struct pf_frame *head = NULL, *frame;
791 struct mlx5_core_mkey *mmkey;835 struct mlx5_core_mkey *mmkey;
792 struct mlx5_ib_mr *mr;836 struct mlx5_ib_mr *mr;
@@ -795,55 +839,39 @@ static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, · 2 unmodified lines
795 size_t offset;839 size_t offset;
796 int ndescs;840 int ndescs;
797 841
798 srcu_key = srcu_read_lock(&dev->mr_srcu);842 srcu_key = srcu_read_lock(&dev->odp_srcu);
799 843
800 io_virt += *bytes_committed;844 io_virt += *bytes_committed;
801 bcnt -= *bytes_committed;845 bcnt -= *bytes_committed;
802 846
803next_mr:847next_mr:
804 mmkey = xa_load(&dev->mdev->priv.mkey_table, mlx5_base_mkey(key));848 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key));
849 if (!mmkey) {
850 mlx5_ib_dbg(
851 dev,
852 "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",
853 key);
854 if (bytes_mapped)
855 *bytes_mapped += bcnt;
856 /*
857 * The user could specify a SGL with multiple lkeys and only
858 * some of them are ODP. Treat the non-ODP ones as fully
859 * faulted.
860 */
861 ret = 0;
862 goto srcu_unlock;
863 }
805 if (!mkey_is_eq(mmkey, key)) {864 if (!mkey_is_eq(mmkey, key)) {
806 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key);865 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key);
807 ret = -EFAULT;866 ret = -EFAULT;
808 goto srcu_unlock;867 goto srcu_unlock;
809 }868 }
810 869
811 if (prefetch && mmkey->type != MLX5_MKEY_MR) {
812 mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n");
813 ret = -EINVAL;
814 goto srcu_unlock;
815 }
816
817 switch (mmkey->type) {870 switch (mmkey->type) {
818 case MLX5_MKEY_MR:871 case MLX5_MKEY_MR:
819 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);872 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
820 if (!smp_load_acquire(&mr->live) || !mr->ibmr.pd) {
821 mlx5_ib_dbg(dev, "got dead MR\n");
822 ret = -EFAULT;
823 goto srcu_unlock;
824 }
825
826 if (prefetch) {
827 if (!is_odp_mr(mr) ||
828 mr->ibmr.pd != pd) {
829 mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n",
830 is_odp_mr(mr) ? "MR is not ODP" :
831 "PD is not of the MR");
832 ret = -EINVAL;
833 goto srcu_unlock;
834 }
835 }
836 873
837 if (!is_odp_mr(mr)) {874 ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0);
838 mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",
839 key);
840 if (bytes_mapped)
841 *bytes_mapped += bcnt;
842 ret = 0;
843 goto srcu_unlock;
844 }
845
846 ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags);
847 if (ret < 0)875 if (ret < 0)
848 goto srcu_unlock;876 goto srcu_unlock;
849 877
@@ -943,7 +971,7 @@ srcu_unlock: · 93 unmodified lines
943 }971 }
944 kfree(out);972 kfree(out);
945 973
946 srcu_read_unlock(&dev->mr_srcu, srcu_key);974 srcu_read_unlock(&dev->odp_srcu, srcu_key);
947 *bytes_committed = 0;975 *bytes_committed = 0;
948 return ret ? ret : npages;976 return ret ? ret : npages;
949}977}
@@ -1024,7 +1052,7 @@ static int pagefault_data_segments(struct mlx5_ib_dev *dev, · 74 unmodified lines
1024 ret = pagefault_single_data_segment(dev, NULL, key,1052 ret = pagefault_single_data_segment(dev, NULL, key,
1025 io_virt, bcnt,1053 io_virt, bcnt,
1026 &pfault->bytes_committed,1054 &pfault->bytes_committed,
1027 bytes_mapped, 0);1055 bytes_mapped);
1028 if (ret < 0)1056 if (ret < 0)
1029 break;1057 break;
1030 npages += ret;1058 npages += ret;
@@ -1307,8 +1335,7 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, · 276 unmodified lines
1307 }1335 }
1308 1336
1309 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length,1337 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length,
1310 &pfault->bytes_committed, NULL,1338 &pfault->bytes_committed, NULL);
1311 0);
1312 if (ret == -EAGAIN) {1339 if (ret == -EAGAIN) {
1313 /* We're racing with an invalidation, don't prefetch */1340 /* We're racing with an invalidation, don't prefetch */
1314 prefetch_activated = 0;1341 prefetch_activated = 0;
@@ -1335,8 +1362,7 @@ static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, · 20 unmodified lines
1335 1362
1336 ret = pagefault_single_data_segment(dev, NULL, rkey, address,1363 ret = pagefault_single_data_segment(dev, NULL, rkey, address,
1337 prefetch_len,1364 prefetch_len,
1338 &bytes_committed, NULL,1365 &bytes_committed, NULL);
1339 0);
1340 if (ret < 0 && ret != -EAGAIN) {1366 if (ret < 0 && ret != -EAGAIN) {
1341 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",1367 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",
1342 ret, pfault->token, address, prefetch_len);1368 ret, pfault->token, address, prefetch_len);
@@ -1639,114 +1665,128 @@ int mlx5_ib_odp_init(void) · 296 unmodified lines
1639 1665
1640struct prefetch_mr_work {1666struct prefetch_mr_work {
1641 struct work_struct work;1667 struct work_struct work;
1642 struct ib_pd *pd;
1643 u32 pf_flags;1668 u32 pf_flags;
1644 u32 num_sge;1669 u32 num_sge;
1645 struct ib_sge sg_list[0];1670 struct {
1671 u64 io_virt;
1672 struct mlx5_ib_mr *mr;
1673 size_t length;
1674 } frags[];
1646};1675};
1647 1676
1648static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev,1677static void destroy_prefetch_work(struct prefetch_mr_work *work)
1649 struct ib_sge *sg_list, u32 num_sge,
1650 u32 from)
1651{1678{
1652 u32 i;1679 u32 i;
1653 int srcu_key;
1654
1655 srcu_key = srcu_read_lock(&dev->mr_srcu);
1656
1657 for (i = from; i < num_sge; ++i) {
1658 struct mlx5_core_mkey *mmkey;
1659 struct mlx5_ib_mr *mr;
1660
1661 mmkey = xa_load(&dev->mdev->priv.mkey_table,
1662 mlx5_base_mkey(sg_list[i].lkey));
1663 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
1664 atomic_dec(&mr->num_pending_prefetch);
1665 }
1666 1680
1667 srcu_read_unlock(&dev->mr_srcu, srcu_key);1681 for (i = 0; i < work->num_sge; ++i)
1682 atomic_dec(&work->frags[i].mr->num_deferred_work);
1683 kvfree(work);
1668}1684}
1669 1685
1670static bool num_pending_prefetch_inc(struct ib_pd *pd,1686static struct mlx5_ib_mr *
1671 struct ib_sge *sg_list, u32 num_sge)1687get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
1688 u32 lkey)
1672{1689{
1673 struct mlx5_ib_dev *dev = to_mdev(pd->device);1690 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1674 bool ret = true;1691 struct mlx5_core_mkey *mmkey;
1675 u32 i;1692 struct ib_umem_odp *odp;
1693 struct mlx5_ib_mr *mr;
1676 1694
1677 for (i = 0; i < num_sge; ++i) {1695 lockdep_assert_held(&dev->odp_srcu);
1678 struct mlx5_core_mkey *mmkey;
1679 struct mlx5_ib_mr *mr;
1680 1696
1681 mmkey = xa_load(&dev->mdev->priv.mkey_table,1697 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey));
1682 mlx5_base_mkey(sg_list[i].lkey));1698 if (!mmkey || mmkey->key != lkey || mmkey->type != MLX5_MKEY_MR)
1683 if (!mmkey || mmkey->key != sg_list[i].lkey) {1699 return NULL;
1684 ret = false;
1685 break;
1686 }
1687 1700
1688 if (mmkey->type != MLX5_MKEY_MR) {1701 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
1689 ret = false;
1690 break;
1691 }
1692 1702
1693 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);1703 if (mr->ibmr.pd != pd)
1704 return NULL;
1694 1705
1695 if (!smp_load_acquire(&mr->live)) {1706 odp = to_ib_umem_odp(mr->umem);
1696 ret = false;
1697 break;
1698 }
1699 1707
1700 if (mr->ibmr.pd != pd) {1708 /* prefetch with write-access must be supported by the MR */
1701 ret = false;1709 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE &&
1702 break;1710 !odp->umem.writable)
1703 }1711 return NULL;
1704 1712
1705 atomic_inc(&mr->num_pending_prefetch);1713 return mr;
1706 }1714}
1707 1715
1708 if (!ret)1716static void mlx5_ib_prefetch_mr_work(struct work_struct *w)
1709 num_pending_prefetch_dec(dev, sg_list, i, 0);1717{
1718 struct prefetch_mr_work *work =
1719 container_of(w, struct prefetch_mr_work, work);
1720 u32 bytes_mapped = 0;
1721 u32 i;
1710 1722
1711 return ret;1723 for (i = 0; i < work->num_sge; ++i)
1724 pagefault_mr(work->frags[i].mr, work->frags[i].io_virt,
1725 work->frags[i].length, &bytes_mapped,
1726 work->pf_flags);
1727
1728 destroy_prefetch_work(work);
1712}1729}
1713 1730
1714static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags,1731static bool init_prefetch_work(struct ib_pd *pd,
1715 struct ib_sge *sg_list, u32 num_sge)1732 enum ib_uverbs_advise_mr_advice advice,
1733 u32 pf_flags, struct prefetch_mr_work *work,
1734 struct ib_sge *sg_list, u32 num_sge)
1716{1735{
1717 u32 i;1736 u32 i;
1718 int ret = 0;1737
1719 struct mlx5_ib_dev *dev = to_mdev(pd->device);1738 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work);
1739 work->pf_flags = pf_flags;
1720 1740
1721 for (i = 0; i < num_sge; ++i) {1741 for (i = 0; i < num_sge; ++i) {
1722 struct ib_sge *sg = &sg_list[i];1742 work->frags[i].io_virt = sg_list[i].addr;
1723 int bytes_committed = 0;1743 work->frags[i].length = sg_list[i].length;
1744 work->frags[i].mr =
1745 get_prefetchable_mr(pd, advice, sg_list[i].lkey);
1746 if (!work->frags[i].mr) {
1747 work->num_sge = i - 1;
1748 if (i)
1749 destroy_prefetch_work(work);
1750 return false;
1751 }
1724 1752
1725 ret = pagefault_single_data_segment(dev, pd, sg->lkey, sg->addr,1753 /* Keep the MR pointer will valid outside the SRCU */
1726 sg->length,1754 atomic_inc(&work->frags[i].mr->num_deferred_work);
1727 &bytes_committed, NULL,
1728 pf_flags);
1729 if (ret < 0)
1730 break;
1731 }1755 }
1732 1756 work->num_sge = num_sge;
1733 return ret < 0 ? ret : 0;1757 return true;
1734}1758}
1735 1759
1736static void mlx5_ib_prefetch_mr_work(struct work_struct *work)1760static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd,
1761 enum ib_uverbs_advise_mr_advice advice,
1762 u32 pf_flags, struct ib_sge *sg_list,
1763 u32 num_sge)
1737{1764{
1738 struct prefetch_mr_work *w =1765 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1739 container_of(work, struct prefetch_mr_work, work);1766 u32 bytes_mapped = 0;
1767 int srcu_key;
1768 int ret = 0;
1769 u32 i;
1770
1771 srcu_key = srcu_read_lock(&dev->odp_srcu);
1772 for (i = 0; i < num_sge; ++i) {
1773 struct mlx5_ib_mr *mr;
1740 1774
1741 if (ib_device_try_get(w->pd->device)) {1775 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey);
1742 mlx5_ib_prefetch_sg_list(w->pd, w->pf_flags, w->sg_list,1776 if (!mr) {
1743 w->num_sge);1777 ret = -ENOENT;
1744 ib_device_put(w->pd->device);1778 goto out;
1779 }
1780 ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length,
1781 &bytes_mapped, pf_flags);
1782 if (ret < 0)
1783 goto out;
1745 }1784 }
1785 ret = 0;
1746 1786
1747 num_pending_prefetch_dec(to_mdev(w->pd->device), w->sg_list,1787out:
1748 w->num_sge, 0);1788 srcu_read_unlock(&dev->odp_srcu, srcu_key);
1749 kvfree(w);1789 return ret;
1750}1790}
1751 1791
1752int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,1792int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
@@ -1754,43 +1794,27 @@ int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, · 1 unmodified line
1754 u32 flags, struct ib_sge *sg_list, u32 num_sge)1794 u32 flags, struct ib_sge *sg_list, u32 num_sge)
1755{1795{
1756 struct mlx5_ib_dev *dev = to_mdev(pd->device);1796 struct mlx5_ib_dev *dev = to_mdev(pd->device);
1757 u32 pf_flags = MLX5_PF_FLAGS_PREFETCH;1797 u32 pf_flags = 0;
1758 struct prefetch_mr_work *work;1798 struct prefetch_mr_work *work;
1759 bool valid_req;
1760 int srcu_key;1799 int srcu_key;
1761 1800
1762 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH)1801 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH)
1763 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE;1802 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE;
1764 1803
1765 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH)1804 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH)
1766 return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list,1805 return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list,
1767 num_sge);1806 num_sge);
1768 1807
1769 work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL);1808 work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL);
1770 if (!work)1809 if (!work)
1771 return -ENOMEM;1810 return -ENOMEM;
1772 1811
1773 memcpy(work->sg_list, sg_list, num_sge * sizeof(struct ib_sge));1812 srcu_key = srcu_read_lock(&dev->odp_srcu);
1774 1813 if (!init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge)) {
1775 /* It is guaranteed that the pd when work is executed is the pd when1814 srcu_read_unlock(&dev->odp_srcu, srcu_key);
1776 * work was queued since pd can't be destroyed while it holds MRs and1815 return -EINVAL;
1777 * destroying a MR leads to flushing the workquque1816 }
1778 */1817 queue_work(system_unbound_wq, &work->work);
1779 work->pd = pd;1818 srcu_read_unlock(&dev->odp_srcu, srcu_key);
1780 work->pf_flags = pf_flags;1819 return 0;
1781 work->num_sge = num_sge;
1782
1783 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work);
1784
1785 srcu_key = srcu_read_lock(&dev->mr_srcu);
1786
1787 valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge);
1788 if (valid_req)
1789 queue_work(system_unbound_wq, &work->work);
1790 else
1791 kvfree(work);
1792
1793 srcu_read_unlock(&dev->mr_srcu, srcu_key);
1794
1795 return valid_req ? 0 : -EINVAL;
1796}1820}
drivers/net/ethernet/mellanox/mlx5/core/main.c+0−4
@@ -837,8 +837,6 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) · 836 unmodified lines
837 837
838 mlx5_init_qp_table(dev);838 mlx5_init_qp_table(dev);
839 839
840 mlx5_init_mkey_table(dev);
841
842 mlx5_init_reserved_gids(dev);840 mlx5_init_reserved_gids(dev);
843 841
844 mlx5_init_clock(dev);842 mlx5_init_clock(dev);
@@ -896,7 +894,6 @@ err_rl_cleanup: · 51 unmodified lines
896err_tables_cleanup:894err_tables_cleanup:
897 mlx5_geneve_destroy(dev->geneve);895 mlx5_geneve_destroy(dev->geneve);
898 mlx5_vxlan_destroy(dev->vxlan);896 mlx5_vxlan_destroy(dev->vxlan);
899 mlx5_cleanup_mkey_table(dev);
900 mlx5_cleanup_qp_table(dev);897 mlx5_cleanup_qp_table(dev);
901 mlx5_cq_debugfs_cleanup(dev);898 mlx5_cq_debugfs_cleanup(dev);
902 mlx5_events_cleanup(dev);899 mlx5_events_cleanup(dev);
@@ -924,7 +921,6 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) · 21 unmodified lines
924 mlx5_vxlan_destroy(dev->vxlan);921 mlx5_vxlan_destroy(dev->vxlan);
925 mlx5_cleanup_clock(dev);922 mlx5_cleanup_clock(dev);
926 mlx5_cleanup_reserved_gids(dev);923 mlx5_cleanup_reserved_gids(dev);
927 mlx5_cleanup_mkey_table(dev);
928 mlx5_cleanup_qp_table(dev);924 mlx5_cleanup_qp_table(dev);
929 mlx5_cq_debugfs_cleanup(dev);925 mlx5_cq_debugfs_cleanup(dev);
930 mlx5_events_cleanup(dev);926 mlx5_events_cleanup(dev);
drivers/net/ethernet/mellanox/mlx5/core/mr.c+1−27
@@ -36,16 +36,6 @@ · 35 unmodified lines
36#include <linux/mlx5/cmd.h>36#include <linux/mlx5/cmd.h>
37#include "mlx5_core.h"37#include "mlx5_core.h"
38 38
39void mlx5_init_mkey_table(struct mlx5_core_dev *dev)
40{
41 xa_init_flags(&dev->priv.mkey_table, XA_FLAGS_LOCK_IRQ);
42}
43
44void mlx5_cleanup_mkey_table(struct mlx5_core_dev *dev)
45{
46 WARN_ON(!xa_empty(&dev->priv.mkey_table));
47}
48
49int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,39int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
50 struct mlx5_core_mkey *mkey,40 struct mlx5_core_mkey *mkey,
51 struct mlx5_async_ctx *async_ctx, u32 *in,41 struct mlx5_async_ctx *async_ctx, u32 *in,
@@ -54,7 +44,6 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, · 2 unmodified lines
54 struct mlx5_async_work *context)44 struct mlx5_async_work *context)
55{45{
56 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0};46 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0};
57 struct xarray *mkeys = &dev->priv.mkey_table;
58 u32 mkey_index;47 u32 mkey_index;
59 void *mkc;48 void *mkc;
60 int err;49 int err;
@@ -84,16 +73,7 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, · 23 unmodified lines
84 73
85 mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n",74 mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n",
86 mkey_index, key, mkey->key);75 mkey_index, key, mkey->key);
87 76 return 0;
88 err = xa_err(xa_store_irq(mkeys, mlx5_base_mkey(mkey->key), mkey,
89 GFP_KERNEL));
90 if (err) {
91 mlx5_core_warn(dev, "failed xarray insert of mkey 0x%x, %d\n",
92 mlx5_base_mkey(mkey->key), err);
93 mlx5_core_destroy_mkey(dev, mkey);
94 }
95
96 return err;
97}77}
98EXPORT_SYMBOL(mlx5_core_create_mkey_cb);78EXPORT_SYMBOL(mlx5_core_create_mkey_cb);
99 79
@@ -111,12 +91,6 @@ int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, · 11 unmodified lines
111{91{
112 u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0};92 u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0};
113 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0};93 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0};
114 struct xarray *mkeys = &dev->priv.mkey_table;
115 unsigned long flags;
116
117 xa_lock_irqsave(mkeys, flags);
118 __xa_erase(mkeys, mlx5_base_mkey(mkey->key));
119 xa_unlock_irqrestore(mkeys, flags);
120 94
121 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);95 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
122 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));96 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key));
include/linux/mlx5/driver.h+0−4
@@ -556,8 +556,6 @@ struct mlx5_priv { · 555 unmodified lines
556 struct dentry *cmdif_debugfs;556 struct dentry *cmdif_debugfs;
557 /* end: qp staff */557 /* end: qp staff */
558 558
559 struct xarray mkey_table;
560
561 /* start: alloc staff */559 /* start: alloc staff */
562 /* protect buffer alocation according to numa node */560 /* protect buffer alocation according to numa node */
563 struct mutex alloc_mutex;561 struct mutex alloc_mutex;
@@ -942,8 +940,6 @@ struct mlx5_cmd_mailbox *mlx5_alloc_cmd_mailbox_chain(struct mlx5_core_dev *dev, · 378 unmodified lines
942 gfp_t flags, int npages);940 gfp_t flags, int npages);
943void mlx5_free_cmd_mailbox_chain(struct mlx5_core_dev *dev,941void mlx5_free_cmd_mailbox_chain(struct mlx5_core_dev *dev,
944 struct mlx5_cmd_mailbox *head);942 struct mlx5_cmd_mailbox *head);
945void mlx5_init_mkey_table(struct mlx5_core_dev *dev);
946void mlx5_cleanup_mkey_table(struct mlx5_core_dev *dev);
947int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,943int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
948 struct mlx5_core_mkey *mkey,944 struct mlx5_core_mkey *mkey,
949 struct mlx5_async_ctx *async_ctx, u32 *in,945 struct mlx5_async_ctx *async_ctx, u32 *in,
include/rdma/ib_umem_odp.h+0−18
@@ -78,9 +78,7 @@ struct ib_umem_odp { · 77 unmodified lines
78 bool is_implicit_odp;78 bool is_implicit_odp;
79 79
80 struct completion notifier_completion;80 struct completion notifier_completion;
81 int dying;
82 unsigned int page_shift;81 unsigned int page_shift;
83 struct work_struct work;
84};82};
85 83
86static inline struct ib_umem_odp *to_ib_umem_odp(struct ib_umem *umem)84static inline struct ib_umem_odp *to_ib_umem_odp(struct ib_umem *umem)
@@ -156,22 +154,6 @@ int rbt_ib_umem_for_each_in_range(struct rb_root_cached *root, · 69 unmodified lines
156 umem_call_back cb,154 umem_call_back cb,
157 bool blockable, void *cookie);155 bool blockable, void *cookie);
158 156
159/*
160 * Find first region intersecting with address range.
161 * Return NULL if not found
162 */
163static inline struct ib_umem_odp *
164rbt_ib_umem_lookup(struct rb_root_cached *root, u64 addr, u64 length)
165{
166 struct interval_tree_node *node;
167
168 node = interval_tree_iter_first(root, addr, addr + length - 1);
169 if (!node)
170 return NULL;
171 return container_of(node, struct ib_umem_odp, interval_tree);
172
173}
174
175static inline int ib_umem_mmu_notifier_retry(struct ib_umem_odp *umem_odp,157static inline int ib_umem_mmu_notifier_retry(struct ib_umem_odp *umem_odp,
176 unsigned long mmu_seq)158 unsigned long mmu_seq)
177{159{

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t